(Proper) Initial commit

backend/services/lucida.ts

@@ -0,0 +1,201 @@
+import {DownloadResult, LucidaOptions} from "../types/queue";
+import {Browser, BrowserContext, Download, firefox, Locator, Page, Response as PResponse} from "playwright";
+import path from "node:path";
+import {fetchBooklet} from "./booklet";
+import * as fs from "node:fs";
+
+class Lucida {
+    private browser: Browser | null;
+    private context: BrowserContext | null;
+    private options: LucidaOptions;
+
+    public constructor(lucidaOptions: LucidaOptions) {
+        this.browser = null;
+        this.context = null;
+        this.options = lucidaOptions
+    }
+
+    public async construct(): Promise<void> {
+        console.log('Lucida: Launching browser');
+
+        const proxy: {server: string} | undefined = this.options.proxy ? {server: this.options.proxy} : undefined;
+        this.browser = await firefox.launch({
+            headless: this.options.headless,
+            proxy: proxy,
+            args: ['--no-sandbox', '--no-gpu']
+        });
+        this.context = await this.browser.newContext({
+            acceptDownloads: true,
+            baseURL: this.options.baseUrl
+        });
+    }
+
+    public async destruct(reason: string | undefined = undefined): Promise<void> {
+        if (this.browser === null) {
+            return;
+        }
+
+        console.log('Lucida: Closing browser');
+        if (this.context !== null) {
+            await this.context.close({reason: reason});
+        }
+
+        await this.browser.close();
+    }
+
+    private async getError(page: Page): Promise<string | undefined> {
+        const errorElement: Locator = page.locator('div[class="error svelte-40348f"]');
+        const hasError: boolean = !page.isClosed() && await errorElement.count() !== 0 && await errorElement.isVisible();
+
+        if (!hasError) {
+            return undefined;
+        }
+
+        let errorText: string = (await errorElement.innerText()).trim();
+        errorText += (await errorElement.locator('details').locator('p').allTextContents()).map(e => e.trim());
+        return errorText;
+    }
+
+    /**
+     * Check the 'Hide my download from recently downloaded' checkbox
+     * @param page The page to check
+     * @private
+     */
+    private async hideDownload(page: Page): Promise<void> {
+        const hideCheckbox: Locator = page.locator('input[id="hide-from-ticker"]');
+        if (await hideCheckbox.count() === 0 || !await hideCheckbox.isVisible()) {
+            console.error('Lucida: No hide checkbox found!');
+            return;
+        }
+
+        try {
+            await hideCheckbox.setChecked(true, {force: true});
+        } catch (err) {
+            console.error('Lucida: Could not check hide checkbox:', err instanceof Error ? err.message : 'Unknown error');
+        }
+    }
+
+    private async saveDownload(download: Download, downloadPath: string): Promise<string> {
+        const pathName: string = path.join(downloadPath, download.suggestedFilename());
+        // await download.saveAs(pathName);
+        // return pathName;
+
+        const stream = fs.createWriteStream(pathName);
+        const response = await download.createReadStream();
+        if (response) {
+            response.pipe(stream);
+            await new Promise((resolve, reject) => {
+                stream.on('finish', resolve);
+                stream.on('error', reject);
+            });
+        }
+
+        return pathName;
+    }
+
+    public async download(url: string, downloadPath: string, timeout: number, status: {
+        status: string
+    }): Promise<DownloadResult> {
+        let finished: boolean = false;
+
+        let close: boolean = false;
+        if (this.context === null) {
+            await this.construct();
+            close = true;
+        }
+
+        if (this.context === null) {
+            return {
+                success: false,
+                error: 'Could not create browser context'
+            };
+        }
+        const downloadPage: Page = await this.context.newPage();
+
+        const downloadPageUrl: string = this.options.baseUrl + '?url=' + encodeURIComponent(url);
+        const downloadPageResp: PResponse | null = await downloadPage.goto(downloadPageUrl);
+        if (downloadPageResp === null || !downloadPageResp.ok()) {
+            return {
+                success: false,
+                error: 'Could not load page'
+            };
+        }
+
+        // Check for errors
+        const error: string | undefined = await this.getError(downloadPage);
+        if (error) {
+            await downloadPage.close();
+            return {
+                success: false,
+                error: error
+            };
+        }
+
+        await this.hideDownload(downloadPage);
+
+        const start: number = Date.now();
+        let retryCount: number = 0;
+
+        // Listen for download
+        downloadPage.once('download', async (download) => {
+            await this.saveDownload(download, downloadPath);
+            finished = true;
+            await fetchBooklet(url, downloadPath, this.context);
+
+            await downloadPage.close();
+            if (close) {
+                await this.destruct();
+            }
+
+            process.stdout.clearLine(0);
+            return {success: true};
+        });
+        // Start download
+        await downloadPage.getByText('download full album').click();
+
+        // Retry file download if it fails
+        const progressElement: Locator = downloadPage.locator('div[class="loader svelte-1ipdo3f"]');
+        while (!finished && retryCount < 10 && (Date.now() - start) < timeout) {
+            const retry: Locator = downloadPage.getByText('Retry');
+            const error: string | undefined = finished ? undefined : await this.getError(downloadPage);
+
+            if (error !== undefined) {
+                console.log('Retrying download because of error:', error);
+                await retry.click();
+                retryCount++;
+            } else if (!finished && await progressElement.count() !== 0 && await progressElement.isVisible()) {
+                const progress: string = (await progressElement.locator('p').innerText()).trim();
+                status.status = progress;
+                process.stdout.clearLine(0);
+                process.stdout.write(`\r${progress}`);
+            }
+        }
+
+        process.stdout.clearLine(0);
+
+        if (!finished) {
+            // Cleanup
+            await downloadPage.close();
+            if (close) {
+                await this.destruct();
+            }
+
+            let errorMessage: string;
+            if (timeout <= (Date.now() - start)) {
+                errorMessage = 'Download timed out';
+            } else if (retryCount >= 10) {
+                errorMessage = 'Download failed after 10 retries';
+            } else {
+                errorMessage = 'Unknown error';
+            }
+            return {
+                success: false,
+                error: errorMessage
+            };
+        }
+
+        return {success: true};
+    }
+}
+
+export default Lucida;