(Proper) Initial commit

backend/app.ts

@@ -0,0 +1,76 @@
+import express, {Express} from 'express';
+import cors from 'cors';
+import queueRoutes from "./routes/queue.routes";
+import bodyParser from 'body-parser';
+import pingRoutes from "./routes/ping.routes";
+import queueManager from "./queueManager";
+import {DEFAULT_LUCIDA_OPTIONS} from "./types/queue";
+
+const app: Express = express();
+
+app.use(cors());
+app.use(express.json());
+app.use(bodyParser.urlencoded({extended: true}));
+
+/* Routes */
+app.use('/api/v1/ping', pingRoutes);
+app.use('/api/v1/queue', queueRoutes);
+
+/* Setup */
+let port: number = 3000;
+let baseUrl: string = DEFAULT_LUCIDA_OPTIONS.baseUrl;
+let headless: boolean = DEFAULT_LUCIDA_OPTIONS.headless;
+let proxy: string | undefined = DEFAULT_LUCIDA_OPTIONS.proxy;
+
+// Parse command line arguments
+let i: number = 0;
+while (i < process.argv.length) {
+    switch (process.argv[i]) {
+        case '--port':
+            i++;
+            port = parseInt(process.argv[i]);
+            break;
+        case '--lucida':
+            i++;
+            baseUrl = process.argv[i];
+            break;
+        case '--headless':
+            i++;
+            headless = process.argv[i] === 'true';
+            break;
+        case '--proxy':
+            i++;
+            proxy = process.argv[i];
+            break;
+        case '--help':
+            console.log(`
+            Options:
+            --port      Specify the port to run the server on. Default is 3000.
+            --lucida    Specify the url to the Lucida server. Default is 'https://lucida.to/'
+            --headless  Run the Lucida browser in headless mode.
+            --proxy     Specify a proxy to use for requests. Example: --proxy 'socks5://localhost:9050' (Tor)
+            --help      Show this help message and exit.
+            `);
+            process.exit();
+        default:
+            console.error(`App: Unknown option ${process.argv[i]}`);
+            break;
+    }
+    i++;
+}
+const lucidaOptions = {
+    baseUrl: baseUrl,
+    headless: headless,
+    proxy: proxy
+};
+queueManager.setLucidaOptions(lucidaOptions);
+
+process.on('SIGINT', async () => {
+    console.log('App: Received SIGINT');
+    await queueManager.forceStop();
+    process.exit();
+});
+
+app.listen(port, () => {
+    console.log(`App: Server running on port ${port}`);
+});

backend/controllers/ping.controllers.ts

@@ -0,0 +1,5 @@
+import {Request, Response} from 'express';
+
+export function ping(_req: Request, res: Response): void {
+    res.status(200).send('pong');
+}

backend/controllers/queue.controllers.ts

@@ -0,0 +1,133 @@
+import {add, clear, get, history, insertAt, list, move, processing, remove, retry} from "../models/queue.models";
+import {ProcessingQueue, Queue, QueueItem} from "../types/queue";
+import {Request, Response} from "express";
+
+export function listQueue(_req: Request, res: Response): void {
+    try {
+        const resp: Queue = list();
+        res.status(200).json(resp);
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}
+
+export function getQueueItem(req: Request, res: Response): void {
+    try {
+        const index = parseInt(req.params.index, 10);
+        if (isNaN(index)) {
+            res.status(400).send('Invalid index');
+            return;
+        }
+        const resp: QueueItem = get(index);
+        res.status(200).json(resp);
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}
+
+export async function addQueueItem(req: Request, res: Response): Promise<void> {
+    try {
+        const song: string = req.body.song;
+        if (!song) {
+            res.status(400).send('No song provided');
+            return;
+        }
+        const resp: QueueItem = await add(song);
+        res.status(201).json(resp); // Created
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}
+
+export async function insertQueueItem(req: Request, res: Response): Promise<void> {
+    try {
+        const song: string = req.body.song;
+        if (!song) {
+            res.status(400).send('No song provided');
+            return;
+        }
+        const index: number = parseInt(req.params.index, 10);
+        if (isNaN(index)) {
+            res.status(400).send('Invalid index');
+            return;
+        }
+        const resp: QueueItem = await insertAt(song, index);
+        res.status(201).json(resp); // Created
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}
+
+export async function removeQueueItem(req: Request, res: Response): Promise<void> {
+    try {
+        const song: string = req.params.song;
+        if (!song) {
+            res.status(400).send('No song provided');
+            return;
+        }
+        const resp: QueueItem = await remove(song);
+        res.status(205).json(resp); // Client should reset content.
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}
+
+export function clearQueue(_req: Request, res: Response): void {
+    try {
+        clear();
+        res.status(205).send(); // Client should reset content.
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}
+
+export function moveQueueItem(req: Request, res: Response): void {
+    try {
+        const song: string = req.body.song;
+        if (!song) {
+            res.status(400).send('No song provided');
+            return;
+        }
+        const to: number = parseInt(req.body.to, 10);
+        if (isNaN(to)) {
+            res.status(400).send('Invalid index');
+            return;
+        }
+        move(song, to);
+        res.status(205).send(); // Client should reset content.
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}
+
+export function retryQueueItem(req: Request, res: Response): void {
+    try {
+        const song: string = req.params.song;
+        if (!song) {
+            res.status(400).send('No song provided');
+            return;
+        }
+        retry(song);
+        res.status(205).send(); // Client should reset content.
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}
+
+export function getProcessing(_req: Request, res: Response): void {
+    try {
+        const resp: ProcessingQueue = processing();
+        res.status(200).json(resp);
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}
+
+export function getHistory(_req: Request, res: Response): void {
+    try {
+        const resp: Queue = history();
+        res.status(200).json(resp);
+    } catch (err) {
+        res.status(500).send(err instanceof Error ? err.message : 'Unknown error');
+    }
+}

backend/models/queue.models.ts

@@ -0,0 +1,49 @@
+import QM from "../queueManager";
+import {ProcessingQueue, Queue, QueueItem} from "../types/queue";
+
+export function list(): Queue {
+    return QM.getQueue();
+}
+
+export function get(index: number): QueueItem {
+    return QM.get(index);
+}
+
+export async function add(song: string): Promise<QueueItem> {
+    return await QM.add(song);
+}
+
+export async function insertAt(song: string, index: number): Promise<QueueItem> {
+    return await QM.insertAt(song, index);
+}
+
+export async function remove(song: string): Promise<QueueItem> {
+    return await QM.remove(song);
+}
+
+export function clear(): void {
+    QM.clear();
+}
+
+/**
+ * Moves an item from one index to another.
+ * @param song the song to move
+ * @param to the index to move the song to.
+ * If an item already exists at this index, the song will be inserted before it.
+ * If the index is out of bounds, the song will be moved to the end of the queue.
+ */
+export function move(song: string, to: number): void {
+    QM.move(song, to);
+}
+
+export function retry(song: string): void {
+    QM.retry(song);
+}
+
+export function processing(): ProcessingQueue {
+    return QM.getProcessing();
+}
+
+export function history(): Queue {
+    return QM.getHistory();
+}

backend/queueManager.ts

@@ -0,0 +1,189 @@
+import {fetchMetadata} from "./services/metadata";
+import {DEFAULT_LUCIDA_OPTIONS, DownloadResult, LucidaOptions, ProcessingQueue, Queue, QueueItem} from "./types/queue";
+import Lucida from "./services/lucida";
+
+const TIMEOUT: number = 120000;
+const RETRIES: number = 5;
+
+class QueueManager {
+    private readonly queue: Queue;
+    private readonly processing: ProcessingQueue;
+    private readonly history: Queue;
+
+    private lucida: Lucida | null;
+    private lucidaOptions: LucidaOptions;
+
+    public constructor() {
+        this.queue = [];
+        this.processing = [];
+        this.history = [];
+
+        this.lucida = null;
+        this.lucidaOptions = DEFAULT_LUCIDA_OPTIONS;
+    }
+
+    public setLucidaOptions(options: LucidaOptions): void {
+        console.log('QueueManager: Setting Lucida options:', options);
+        this.lucidaOptions = options;
+    }
+
+    public getQueue(): Queue {
+        return this.queue;
+    }
+
+    public getProcessing(): ProcessingQueue {
+        return this.processing;
+    }
+
+    public getHistory(): Queue {
+        return this.history;
+    }
+
+    public get(index: number): QueueItem {
+        if (index < 0 || index >= this.queue.length) {
+            throw new Error('Index out of bounds');
+        }
+
+        return this.queue[index];
+    }
+
+    public async add(song: string): Promise<QueueItem> {
+        // Check if the URL is already in the queue.
+        if (this.queue.some(q => q.id === song)) {
+            throw new Error('Song already in queue');
+        }
+
+        const item: QueueItem = {
+            id: song,
+            song: await fetchMetadata(song)
+        };
+        this.queue.push(item);
+        this.processQueue();
+
+        return item;
+    }
+
+    public async insertAt(song: string, index: number): Promise<QueueItem> {
+        // Check if the URL is already in the queue.
+        if (this.queue.some(q => q.id === song)) {
+            throw new Error('Song already in queue');
+        }
+
+        const item: QueueItem = {
+            id: song,
+            song: await fetchMetadata(song)
+        };
+        this.queue.splice(index, 0, item);
+        this.processQueue();
+
+        return item;
+    }
+
+    public async remove(song: string): Promise<QueueItem> {
+        const index: number = this.queue.findIndex(q => q.id === song);
+        if (index === -1) {
+            throw new Error('Song not found');
+        }
+
+        const item: QueueItem = this.queue[index];
+        this.queue.splice(index, 1);
+        return item;
+    }
+
+    public clear(): void {
+        this.queue.length = 0;
+    }
+
+    public move(song: string, to: number): QueueItem {
+        const index: number = this.queue.findIndex(q => q.id === song);
+        if (index === -1) {
+            throw new Error('Song not found');
+        }
+
+        const item: QueueItem = this.queue.splice(index, 1)[0];
+        this.queue.splice(to, 0, item);
+        return item;
+    }
+
+    public retry(song: string): QueueItem {
+        const index: number = this.history.findIndex(q => q.id === song);
+        if (index === -1) {
+            throw new Error('Song not found');
+        }
+
+        const item: QueueItem = this.history.splice(index, 1)[0];
+        item.retries = 0;
+        this.queue.push(item);
+        this.processQueue();
+
+        return item;
+    }
+
+    public async processQueue(): Promise<void> {
+        if (this.lucida !== null) {
+            return;
+        }
+
+        this.lucida = new Lucida(this.lucidaOptions);
+        await this.lucida.construct();
+
+        while (0 < this.queue.length) {
+            const item: QueueItem | undefined = this.queue.shift();
+            if (!item) {
+                console.log('QueueManager: Detected undefined item');
+                continue;
+            }
+
+            const current: {item: QueueItem, status: string} = {
+                item: item,
+                status: 'Starting'
+            }
+            this.processing.push(current);
+
+            try {
+                const timeout: number = (item.timeout ?? TIMEOUT) * item.song.trackCount;
+                const result: DownloadResult = await this.lucida.download(item.song.url, '/tmp', timeout, current);
+
+                if (!result.success && (item.retries ?? 0) + 1 < RETRIES) {
+                    item.retries = (item.retries ?? 0) + 1;
+                    item.timeout = (item.timeout ?? TIMEOUT) * 2;
+                    item.result = result;
+                    this.queue.push(item);
+                } else {
+                    item.result = result;
+                    this.history.push(item);
+                }
+            } catch (err) {
+                item.result = {
+                    success: false,
+                    error: err instanceof Error ? err.message : 'Unknown error'
+                };
+                this.history.push(item);
+            }
+
+            this.processing.splice(this.processing.indexOf(current), 1);
+        }
+
+        await this.lucida.destruct();
+        this.lucida = null;
+    }
+
+    public async forceStop(): Promise<void> {
+        this.processing.forEach(item => {
+            item.item.result = {
+                success: false,
+                error: 'Forced stop'
+            };
+            this.history.push(item.item);
+        })
+
+        if (this.lucida === null) {
+            return;
+        }
+
+        await this.lucida.destruct('Forced stop');
+        this.lucida = null;
+    }
+}
+
+export default new QueueManager();

backend/routes/ping.routes.ts

@@ -0,0 +1,9 @@
+import {Router} from "express";
+import {ping} from "../controllers/ping.controllers";
+
+const router: Router = Router();
+
+router.get('/', ping);
+
+
+export default router;

backend/routes/queue.routes.ts

@@ -0,0 +1,29 @@
+import {Router} from "express";
+import {
+    addQueueItem,
+    clearQueue,
+    getHistory,
+    getProcessing,
+    getQueueItem,
+    insertQueueItem,
+    listQueue,
+    moveQueueItem,
+    removeQueueItem, retryQueueItem
+} from "../controllers/queue.controllers";
+
+const router: Router = Router();
+
+router.get('/', listQueue);
+router.get('/processing', getProcessing);
+router.get('/history', getHistory);
+router.get('/:index', getQueueItem);
+
+router.post('/', addQueueItem);
+router.post('/move', moveQueueItem);
+router.post('/retry/:song', retryQueueItem);
+router.post('/:index', insertQueueItem);
+
+router.delete('/', clearQueue);
+router.delete('/:song', removeQueueItem);
+
+export default router;

backend/services/booklet.ts

@@ -0,0 +1,41 @@
+import {BrowserContext, Download, firefox, Locator} from "playwright";
+import path from "node:path";
+
+export async function fetchBooklet(url: string, downloadPath: string, context: BrowserContext | null): Promise<string | null> {
+    const browser = await firefox.launch({
+        headless: true
+    });
+    context = await browser.newContext({
+        acceptDownloads: true
+    });
+    const page = await context.newPage();
+
+    const bookletPage: string = 'http://audiofil.hostronavt.ru/booklet.php?name=' + encodeURIComponent(url);
+    await page.goto(bookletPage);
+
+    // Find link with goodies
+    const link: Locator = page.locator('a').filter({hasText: '/goodies/'});
+    const linkCount: number = await link.count();
+    if (0 <= linkCount) {
+        await page.close();
+        return null;
+    }
+
+    console.log(`Booklet: Found goodies: ${await link.innerHTML()}`);
+
+    let filename: string | null = null;
+    try {
+        const downloadPromise: Promise<Download> = page.waitForEvent('download');
+        await link.dispatchEvent('click');
+        const download: Download = await downloadPromise;
+
+        filename = download.suggestedFilename();
+        await download.saveAs(path.join(downloadPath, filename));
+    } catch (err) {
+        console.log('Booklet: Could not download booklet:', err instanceof Error ? err.message : 'Unknown error');
+        filename = null;
+    }
+
+    await page.close();
+    return filename;
+}

backend/services/lucida.ts

@@ -0,0 +1,201 @@
+import {DownloadResult, LucidaOptions} from "../types/queue";
+import {Browser, BrowserContext, Download, firefox, Locator, Page, Response as PResponse} from "playwright";
+import path from "node:path";
+import {fetchBooklet} from "./booklet";
+import * as fs from "node:fs";
+
+class Lucida {
+    private browser: Browser | null;
+    private context: BrowserContext | null;
+    private options: LucidaOptions;
+
+    public constructor(lucidaOptions: LucidaOptions) {
+        this.browser = null;
+        this.context = null;
+        this.options = lucidaOptions
+    }
+
+    public async construct(): Promise<void> {
+        console.log('Lucida: Launching browser');
+
+        const proxy: {server: string} | undefined = this.options.proxy ? {server: this.options.proxy} : undefined;
+        this.browser = await firefox.launch({
+            headless: this.options.headless,
+            proxy: proxy,
+            args: ['--no-sandbox', '--no-gpu']
+        });
+        this.context = await this.browser.newContext({
+            acceptDownloads: true,
+            baseURL: this.options.baseUrl
+        });
+    }
+
+    public async destruct(reason: string | undefined = undefined): Promise<void> {
+        if (this.browser === null) {
+            return;
+        }
+
+        console.log('Lucida: Closing browser');
+        if (this.context !== null) {
+            await this.context.close({reason: reason});
+        }
+
+        await this.browser.close();
+    }
+
+    private async getError(page: Page): Promise<string | undefined> {
+        const errorElement: Locator = page.locator('div[class="error svelte-40348f"]');
+        const hasError: boolean = !page.isClosed() && await errorElement.count() !== 0 && await errorElement.isVisible();
+
+        if (!hasError) {
+            return undefined;
+        }
+
+        let errorText: string = (await errorElement.innerText()).trim();
+        errorText += (await errorElement.locator('details').locator('p').allTextContents()).map(e => e.trim());
+        return errorText;
+    }
+
+    /**
+     * Check the 'Hide my download from recently downloaded' checkbox
+     * @param page The page to check
+     * @private
+     */
+    private async hideDownload(page: Page): Promise<void> {
+        const hideCheckbox: Locator = page.locator('input[id="hide-from-ticker"]');
+        if (await hideCheckbox.count() === 0 || !await hideCheckbox.isVisible()) {
+            console.error('Lucida: No hide checkbox found!');
+            return;
+        }
+
+        try {
+            await hideCheckbox.setChecked(true, {force: true});
+        } catch (err) {
+            console.error('Lucida: Could not check hide checkbox:', err instanceof Error ? err.message : 'Unknown error');
+        }
+    }
+
+    private async saveDownload(download: Download, downloadPath: string): Promise<string> {
+        const pathName: string = path.join(downloadPath, download.suggestedFilename());
+        // await download.saveAs(pathName);
+        // return pathName;
+
+        const stream = fs.createWriteStream(pathName);
+        const response = await download.createReadStream();
+        if (response) {
+            response.pipe(stream);
+            await new Promise((resolve, reject) => {
+                stream.on('finish', resolve);
+                stream.on('error', reject);
+            });
+        }
+
+        return pathName;
+    }
+
+    public async download(url: string, downloadPath: string, timeout: number, status: {
+        status: string
+    }): Promise<DownloadResult> {
+        let finished: boolean = false;
+
+        let close: boolean = false;
+        if (this.context === null) {
+            await this.construct();
+            close = true;
+        }
+
+        if (this.context === null) {
+            return {
+                success: false,
+                error: 'Could not create browser context'
+            };
+        }
+        const downloadPage: Page = await this.context.newPage();
+
+        const downloadPageUrl: string = this.options.baseUrl + '?url=' + encodeURIComponent(url);
+        const downloadPageResp: PResponse | null = await downloadPage.goto(downloadPageUrl);
+        if (downloadPageResp === null || !downloadPageResp.ok()) {
+            return {
+                success: false,
+                error: 'Could not load page'
+            };
+        }
+
+        // Check for errors
+        const error: string | undefined = await this.getError(downloadPage);
+        if (error) {
+            await downloadPage.close();
+            return {
+                success: false,
+                error: error
+            };
+        }
+
+        await this.hideDownload(downloadPage);
+
+        const start: number = Date.now();
+        let retryCount: number = 0;
+
+        // Listen for download
+        downloadPage.once('download', async (download) => {
+            await this.saveDownload(download, downloadPath);
+            finished = true;
+            await fetchBooklet(url, downloadPath, this.context);
+
+            await downloadPage.close();
+            if (close) {
+                await this.destruct();
+            }
+
+            process.stdout.clearLine(0);
+            return {success: true};
+        });
+        // Start download
+        await downloadPage.getByText('download full album').click();
+
+        // Retry file download if it fails
+        const progressElement: Locator = downloadPage.locator('div[class="loader svelte-1ipdo3f"]');
+        while (!finished && retryCount < 10 && (Date.now() - start) < timeout) {
+            const retry: Locator = downloadPage.getByText('Retry');
+            const error: string | undefined = finished ? undefined : await this.getError(downloadPage);
+
+            if (error !== undefined) {
+                console.log('Retrying download because of error:', error);
+                await retry.click();
+                retryCount++;
+            } else if (!finished && await progressElement.count() !== 0 && await progressElement.isVisible()) {
+                const progress: string = (await progressElement.locator('p').innerText()).trim();
+                status.status = progress;
+                process.stdout.clearLine(0);
+                process.stdout.write(`\r${progress}`);
+            }
+        }
+
+        process.stdout.clearLine(0);
+
+        if (!finished) {
+            // Cleanup
+            await downloadPage.close();
+            if (close) {
+                await this.destruct();
+            }
+
+            let errorMessage: string;
+            if (timeout <= (Date.now() - start)) {
+                errorMessage = 'Download timed out';
+            } else if (retryCount >= 10) {
+                errorMessage = 'Download failed after 10 retries';
+            } else {
+                errorMessage = 'Unknown error';
+            }
+            return {
+                success: false,
+                error: errorMessage
+            };
+        }
+
+        return {success: true};
+    }
+}
+
+export default Lucida;

backend/services/metadata.ts

@@ -0,0 +1,50 @@
+import {JSDOM} from 'jsdom';
+import {Song, SongSource} from '../types/queue';
+
+const TITLE_SELECTOR: string = 'h1[class="svelte-6pt9ji"]';
+const ARTIST_SELECTOR: string = 'h2[class="svelte-6pt9ji"]';
+const TRACK_COUNT_SELECTOR: string = 'h3[class="svelte-6pt9ji"]';
+const COVER_SELECTOR: string = 'a[title="Click for full quality cover (unproxied)."]';
+
+export async function fetchMetadata(url: string): Promise<Song> {
+    const resp: Response = await fetch('https://lucida.to/?url=' + encodeURIComponent(url));
+    if (!resp.ok) {
+        return {
+            title: '<unknown>',
+            artist: '<unknown>',
+            trackCount: -1,
+            url: url,
+            source: getSource(url)
+        }
+    }
+
+    const doc: Document = new JSDOM(await resp.text()).window.document;
+    const title: string = doc.querySelector(TITLE_SELECTOR)?.textContent?.trim() ?? '<unknown>';
+    const artist: string = doc.querySelector(ARTIST_SELECTOR)?.textContent?.trim() ?? '<unknown>';
+    const trackCount: number = parseInt(doc.querySelector(TRACK_COUNT_SELECTOR)?.innerHTML.trim().split(' ')[0] ?? '-1');
+    const cover: string | undefined = doc.querySelector(COVER_SELECTOR)?.attributes.getNamedItem('href')?.value ?? undefined;
+
+    return {
+        title: title,
+        artist: artist.substring(artist.indexOf(' ') + 1),
+        trackCount: trackCount,
+        url: url,
+        source: getSource(url),
+        cover: cover
+    }
+}
+
+function getSource(url: string): SongSource {
+    const urlToSource: Map<string, SongSource> = new Map([
+        ['qobuz.com', SongSource.Qobuz],
+        ['spotify.com', SongSource.Spotify]
+    ]);
+
+    for (const [key, value] of urlToSource) {
+        if (url.includes(key)) {
+            return value;
+        }
+    }
+
+    return SongSource.Unknown;
+}

backend/types/queue.ts

@@ -0,0 +1,44 @@
+export type Song = {
+    title: string,
+    artist: string,
+    trackCount: number,
+    url: string,
+    source: SongSource,
+    cover?: string
+}
+
+export enum SongSource {
+    Spotify = 'spotify',
+    Qobuz = 'qobuz',
+    Unknown = 'unknown'
+}
+
+export type QueueItem = {
+    id: string,
+    song: Song,
+    timeout?: number,
+    retries?: number,
+    result?: DownloadResult
+}
+
+export type Queue = QueueItem[];
+export type ProcessingQueue = {
+    item: QueueItem,
+    status: string
+}[];
+
+export type DownloadResult = {
+    success: boolean,
+    error?: string
+}
+
+export type LucidaOptions = {
+    baseUrl: string,
+    headless: boolean,
+    proxy?: string
+}
+
+export const DEFAULT_LUCIDA_OPTIONS: LucidaOptions = {
+    baseUrl: 'https://lucida.to',
+    headless: true
+}