tdpeuter/lucida-queue
tdpeuter/lucida-queue
Archived
1
Fork 0
Code Issues 10 Pull requests Releases Packages Activity Actions

chore(lucida): Improve download flow

Browse source
This commit is contained in:
Tibo De Peuter 2025-02-02 12:25:18 +01:00
parent 81b18df1f4
commit b5dcaaa515
Signed by: tdpeuter
GPG key ID: 38297DE43F75FFE2
1 changed files with 30 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

60
backend/services/lucida.ts
View file

@ -2,7 +2,6 @@ import {DownloadResult, LucidaOptions} from "../types/queue";
import {Browser, BrowserContext, Download, firefox, Locator, Page, Response as PResponse} from "playwright"; import {Browser, BrowserContext, Download, firefox, Locator, Page, Response as PResponse} from "playwright";
import path from "node:path"; import path from "node:path";
import {fetchBooklet} from "./booklet"; import {fetchBooklet} from "./booklet";
import * as fs from "node:fs";
class Lucida { class Lucida {
private browser: Browser | null; private browser: Browser | null;
@ -77,18 +76,17 @@ class Lucida {
private async saveDownload(download: Download, downloadPath: string): Promise<string> { private async saveDownload(download: Download, downloadPath: string): Promise<string> {
const pathName: string = path.join(downloadPath, download.suggestedFilename()); const pathName: string = path.join(downloadPath, download.suggestedFilename());
// await download.saveAs(pathName); await download.saveAs(pathName);
// return pathName;
const stream = fs.createWriteStream(pathName); // const stream = fs.createWriteStream(pathName);
const response = await download.createReadStream(); // const response = await download.createReadStream();
if (response) { // if (response) {
response.pipe(stream); // response.pipe(stream);
await new Promise((resolve, reject) => { // await new Promise((resolve, reject) => {
stream.on('finish', resolve); // stream.on('finish', resolve);
stream.on('error', reject); // stream.on('error', reject);
}); // });
} // }
return pathName; return pathName;
} }
@ -97,6 +95,7 @@ class Lucida {
status: string status: string
}): Promise<DownloadResult> { }): Promise<DownloadResult> {
let finished: boolean = false; let finished: boolean = false;
let started: boolean = true;
let close: boolean = false; let close: boolean = false;
if (this.context === null) { if (this.context === null) {
@ -123,7 +122,7 @@ class Lucida {
// Check for errors // Check for errors
const error: string | undefined = await this.getError(downloadPage); const error: string | undefined = await this.getError(downloadPage);
if (error) { if (error !== undefined) {
await downloadPage.close(); await downloadPage.close();
return { return {
success: false, success: false,
@ -138,42 +137,38 @@ class Lucida {
// Listen for download // Listen for download
downloadPage.once('download', async (download) => { downloadPage.once('download', async (download) => {
started = true;
await this.saveDownload(download, downloadPath); await this.saveDownload(download, downloadPath);
finished = true;
await fetchBooklet(url, downloadPath, this.context);
await downloadPage.close(); await downloadPage.close();
await fetchBooklet(url, downloadPath, this.context);
finished = true;
if (close) { if (close) {
await this.destruct(); await this.destruct();
} }
process.stdout.clearLine(0);
return {success: true}; return {success: true};
}); });
// Start download // Start download
await downloadPage.getByText('download full album').click(); await downloadPage.getByText('download full album').click();
// Retry file download if it fails // Retry file download if it fails
const maxRetries: number = 10;
const progressElement: Locator = downloadPage.locator('div[class="loader svelte-1ipdo3f"]'); const progressElement: Locator = downloadPage.locator('div[class="loader svelte-1ipdo3f"]');
while (!finished && retryCount < 10 && (Date.now() - start) < timeout) { while (!started && retryCount < maxRetries && (Date.now() - start) < timeout) {
const retry: Locator = downloadPage.getByText('Retry'); const retry: Locator = downloadPage.getByText('Retry');
const error: string | undefined = finished ? undefined : await this.getError(downloadPage); const error: string | undefined = started ? undefined : await this.getError(downloadPage);
if (error !== undefined) { if (error !== undefined) {
console.log('Retrying download because of error:', error); console.log('Retrying download because of error:', error);
await retry.click(); await retry.click();
retryCount++; retryCount++;
} else if (!finished && await progressElement.count() !== 0 && await progressElement.isVisible()) { } else if (!started && await progressElement.count() !== 0 && await progressElement.isVisible()) {
const progress: string = (await progressElement.locator('p').innerText()).trim(); status.status = (await progressElement.locator('p').innerText()).trim();
status.status = progress;
process.stdout.clearLine(0);
process.stdout.write(`\r${progress}`);
} }
} }
process.stdout.clearLine(0); if (!started) {
if (!finished) {
// Cleanup // Cleanup
await downloadPage.close(); await downloadPage.close();
if (close) { if (close) {
@ -183,8 +178,8 @@ class Lucida {
let errorMessage: string; let errorMessage: string;
if (timeout <= (Date.now() - start)) { if (timeout <= (Date.now() - start)) {
errorMessage = 'Download timed out'; errorMessage = 'Download timed out';
} else if (retryCount >= 10) { } else if (retryCount >= maxRetries) {
errorMessage = 'Download failed after 10 retries'; errorMessage = `Download failed after ${maxRetries} retries`;
} else { } else {
errorMessage = 'Unknown error'; errorMessage = 'Unknown error';
} }
@ -194,7 +189,12 @@ class Lucida {
}; };
} }
return {success: true}; // Wait for download to finish
await new Promise(resolve => setTimeout(resolve, 30000));
return {
success: finished,
error: finished ? undefined : 'Save to disk did not finish.'
};
} }
} }