From 48c005986095e740624e024fd809226d9a63b8d9 Mon Sep 17 00:00:00 2001 From: Tibo De Peuter Date: Sun, 5 Jan 2025 23:56:04 +0100 Subject: [PATCH] chore: Clean slate --- .gitignore | 1 - src/index.ts | 61 ------------------------ src/lucida.ts | 112 -------------------------------------------- src/queueManager.ts | 112 -------------------------------------------- 4 files changed, 286 deletions(-) delete mode 100644 src/index.ts delete mode 100644 src/lucida.ts delete mode 100644 src/queueManager.ts diff --git a/.gitignore b/.gitignore index 3077801..4afb56c 100644 --- a/.gitignore +++ b/.gitignore @@ -103,7 +103,6 @@ dist # vuepress v2.x temp and cache directory .temp -.cache # Docusaurus cache and generated files .docusaurus diff --git a/src/index.ts b/src/index.ts deleted file mode 100644 index 0e56264..0000000 --- a/src/index.ts +++ /dev/null @@ -1,61 +0,0 @@ -import express from 'express'; -import bodyParser from "body-parser"; -import QueueManager from './queueManager'; - -const port = 3000; -const api = '/api/v1'; - -const app = express(); -const qm = new QueueManager(); - - app.use(bodyParser.urlencoded({ extended: true })); -app.use(bodyParser.json()); - - -app.get(api + '/queue', (req, res) => { - res.json(qm.getQueue().map(qi => qi.album.href)); -}); - -app.post(api + '/queue', (req, res) => { - const album: string = req.body.album; - - if (!album) { - res.status(400).send('No album URL provided'); // Bad request - return; - } - - const result: boolean | string = qm.addToQueue(album); - if (result === true) { - res.status(201).send('Album added to queue'); // Created - } else { - res.status(409).send(result); // Conflict - } -}); - -app.delete(api + '/queue', (req, res) => { - qm.clearQueue(); - res.status(205).send(); // Client should reset content. -}); - -app.delete(api + '/queue/:index', (req, res) => { - const index = parseInt(req.params.index, 10); - if (isNaN(index)) { - res.status(400).send('Invalid index'); - return; - } - const result: boolean | string = qm.removeFromQueue(index); - if (result === true) { - res.status(205).send(); // Client should reset content. - } else { - res.status(404).send(result); // Not found - } -}); - -app.listen(port, () => { - console.log(`Server running on port ${port}`); -}); - -process.on('SIGINT', async () => { - await qm.forceStop(); - process.exit(); -}); \ No newline at end of file diff --git a/src/lucida.ts b/src/lucida.ts deleted file mode 100644 index ca77d94..0000000 --- a/src/lucida.ts +++ /dev/null @@ -1,112 +0,0 @@ -import {BrowserContext, Download, Locator, Page} from 'playwright'; - -export async function openAlbum(album: URL, context: BrowserContext): Promise { - const page: Page = await context.newPage(); - await page.goto('/'); - - // Fill in the album URL - await page.locator('input[id="download"]').first().fill(album.href); - - // Wait for the XHR request to complete - await Promise.all([ - page.waitForResponse(res => res.url().includes('/api/load') && res.status() == 200), - page.locator('input[id="go"]').first().click(), - page.waitForLoadState('domcontentloaded') - ]); - - return page; -} - -export async function lucida(album: URL, baseTimeout: number, context: BrowserContext): Promise { - const page: Page = await openAlbum(album, context); - - // Check 'Hide my download from recently downloaded' checkbox - await page.locator('input[id="hide-from-ticker"]').first().setChecked(true); - - // Parse info - const albumName: string = (await page.locator('h1[class="svelte-6pt9ji"]').last().innerText()).trim(); - const trackCount: number = parseInt((await page.locator('h3[class="svelte-6pt9ji"]').first().innerText()).trim().split(' ')[0]); - - console.log(`Downloading ${albumName} (${trackCount} tracks) from ${album.href}...`); - console.log(`Setting timeout to ${baseTimeout * trackCount} ms...`); - - // Start download - await page.getByText('download full album').click(); - - let download: Download | null = null; - const timeout: number = baseTimeout * trackCount; - const start: number = Date.now(); - let retryCount: number = 0; - - // Retry file download if it fails - while (!download && (Date.now() - start) < timeout && retryCount < trackCount) { - try { - download = await page.waitForEvent('download'); - } catch { - const retry: Locator = page.getByText('Retry'); - if (await retry.count() !== 0 && await retry.isVisible() && await retry.isEnabled()) { - await page.getByText('Retry').click(); - console.log('Retrying download...'); - retryCount++; - } - } - } - - if (download === null) { - await page.close(); - return 'Download timed out'; - } - - // Save the download to the Downloads folder - // TODO Set path (configurable) - await download.saveAs('/home/tdpeuter/Downloads/lucida/' + download.suggestedFilename()); - - // Check if the album has a booklet - const bookPath: File | null = await booklet(album, context); - if (bookPath !== null) { - console.log(`Downloaded booklet ${bookPath.name}`); - // TODO Add booklet to ZIP - } - - await page.close(); - return true; -} - -async function booklet(album: URL, context: BrowserContext): Promise { - const page: Page = await context.newPage(); - - const bookletURL: URL = new URL('http://audiofil.hostronavt.ru/booklet.php?name=' + encodeURIComponent(album.href)); - await page.goto(bookletURL.href); - - // Find link with goodies - const link: Locator = page.locator('a').filter({hasText: '/goodies/'}); - const linkCount: number = await link.count(); - - if (0 <= linkCount) { - console.log('No goodies link found'); - await page.close(); - return null; - } - - console.log(`Found goodies: ${await link.innerHTML()}`) - let result: File | null = null; - - try { - // Go to the goodies link - await link.dispatchEvent('click'); - - // Download the booklet - const download: Download = await page.waitForEvent('download'); - const filename: string = download.suggestedFilename(); - await download.saveAs('/home/tdpeuter/Downloads/lucida/' + filename); - - result = new File([await download.path()], filename); - - console.log(`Downloaded booklet ${filename}`); - } catch (e) { - console.log('Could not download booklet:', e); - } - - await page.close(); - return result; -} diff --git a/src/queueManager.ts b/src/queueManager.ts deleted file mode 100644 index 73f4956..0000000 --- a/src/queueManager.ts +++ /dev/null @@ -1,112 +0,0 @@ -import {firefox, Browser, BrowserContext, Page} from "playwright"; -import {lucida} from "./lucida"; - -type QueueItem = { - album: URL, - timeout: number, - retries?: number -} - -const DEFAULT_TIMEOUT: number = 60000; -const MAX_RETRIES: number = 3; - -class QueueManager { - private browser: Browser | null; - private queue: QueueItem[]; - - constructor(browser: Browser | null = null) { - this.browser = browser; - this.queue = []; - } - - getQueue(): QueueItem[] { - return this.queue; - } - - addToQueue(link: string): boolean | string { - // Check if the URL is a valid URL - try { - const url: URL = new URL(link); - const qi: QueueItem = { - album: url, - timeout: DEFAULT_TIMEOUT - }; - - // Check if the URL was already added to the queue - if (this.queue.some(q => q.album.href === qi.album.href)) { - return "Album already in queue"; - } - - this.queue.push(qi); - this.processQueue(); - return true; - } catch { - return "Invalid URL"; - } - } - - removeFromQueue(index: number): boolean | string { - if (index < 0 || index >= this.queue.length) { - return "Index out of bounds"; - } - - this.queue.splice(index, 1); - return true; - } - - clearQueue(): void { - this.queue = []; - } - - forceStop(): Promise { - if (this.browser === null) { - return Promise.resolve(); - } - - return this.browser.close(); - } - - private async processQueue(): Promise { - if (this.browser !== null) { - return; - } - - console.log('Starting browser'); - this.browser = await firefox.launch({headless: false}); - const context: BrowserContext = await this.browser.newContext({ - acceptDownloads: true, - baseURL: 'https://lucida.to' - }); - - while (this.queue.length > 0) { - const qi: QueueItem | undefined = this.queue.shift(); - if (qi === undefined) { - continue; - } - - const album: URL = qi.album; - console.log(`Processing ${album.href}`); - const result: boolean | string = await lucida(album, qi.timeout, context); - if (result === true) { - console.log(`Finished processing ${album.href}`); - } else { - if (qi.retries === undefined) { - qi.retries = 0; - } - qi.retries++; - qi.timeout *= 2; - if (qi.retries < MAX_RETRIES) { - this.queue.push(qi); - } else { - console.error(`Failed to process ${album.href} after 3 retries: ${result}`); - } - } - } - - console.log('Shutting down browser'); - await this.browser.close(); - this.browser = null; - } -} - -export default QueueManager;