tdpeuter/lucida-queue
tdpeuter/lucida-queue
Archived
1
Fork 0
Code Issues 10 Pull requests Releases Packages Activity Actions

Compare commits

base: tdpeuter:15091444bea63047561603361c3dea853aa0c239
Branches Tags
tdpeuter:main
tdpeuter:ci/containers
tdpeuter:v0.1.0
..
compare: tdpeuter:212c544eb18a6509bc5462babcaa30c85c48b913
Branches Tags
tdpeuter:ci/containers
tdpeuter:main
tdpeuter:v0.1.0

No commits in common. "15091444bea63047561603361c3dea853aa0c239" and "212c544eb18a6509bc5462babcaa30c85c48b913" have entirely different histories.
15091444be ... 212c544eb1

10 changed files with 795 additions and 1072 deletions
Download patch file Download diff file Expand all files Collapse all files

8
.dockerignore
View file

@ -1,8 +0,0 @@
.idea
node_modules
backend/Dockerfile
dist
.dockerignore
test
.gitignore
.git

15
backend/Dockerfile
View file

@ -1,15 +0,0 @@
FROM node:22-bookworm
WORKDIR /app
COPY package*.json ./
RUN npm install
RUN npx playwright install --with-deps
COPY . .
RUN npm run build
EXPOSE 3000
CMD ["node", "dist/app.js", "--", "--port", "3000", "--proxy", "socks5://tor:9050"]

48
backend/queueManager.ts
View file

@ -1,13 +1,10 @@
import {fetchMetadata} from "./services/metadata";
import {DEFAULT_LUCIDA_OPTIONS, DownloadResult, LucidaOptions, ProcessingQueue, Queue, QueueItem} from "./types/queue";
import Lucida from "./services/lucida";
import * as fs from "node:fs";
const TIMEOUT: number = 120000;
const RETRIES: number = 5;
const STATE_FILE: string = '/data/state.json';
class QueueManager {
private readonly queue: Queue;
private readonly processing: ProcessingQueue;
@ -23,42 +20,6 @@ class QueueManager {
this.lucida = null;
this.lucidaOptions = DEFAULT_LUCIDA_OPTIONS;
this.loadState();
}
private saveState(): void {
const state = {
queue: this.processing.map(pi => pi.item).concat(this.queue),
history: this.history
};
fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 4));
}
private loadState(): void {
if (!fs.existsSync(STATE_FILE)) {
return;
}
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
console.log('QueueManager: Restoring state:', state);
this.queue.push(...state.queue);
state.history.forEach((item: QueueItem) => {
if (!this.queue.some(q => q.id === item.id)) {
if (item.result?.success) {
// Add successful items to the history
this.history.push(item);
} else {
// Add failed items to the queue
this.queue.push(item);
}
}
});
this.processQueue();
}
public setLucidaOptions(options: LucidaOptions): void {
@ -97,7 +58,6 @@ class QueueManager {
song: await fetchMetadata(song)
};
this.queue.push(item);
this.saveState();
this.processQueue();
return item;
@ -114,7 +74,6 @@ class QueueManager {
song: await fetchMetadata(song)
};
this.queue.splice(index, 0, item);
this.saveState();
this.processQueue();
return item;
@ -128,13 +87,11 @@ class QueueManager {
const item: QueueItem = this.queue[index];
this.queue.splice(index, 1);
this.saveState();
return item;
}
public clear(): void {
this.queue.length = 0;
this.saveState();
}
public move(song: string, to: number): QueueItem {
@ -145,7 +102,6 @@ class QueueManager {
const item: QueueItem = this.queue.splice(index, 1)[0];
this.queue.splice(to, 0, item);
this.saveState();
return item;
}
@ -158,7 +114,6 @@ class QueueManager {
const item: QueueItem = this.history.splice(index, 1)[0];
item.retries = 0;
this.queue.push(item);
this.saveState();
this.processQueue();
return item;
@ -194,11 +149,9 @@ class QueueManager {
item.timeout = (item.timeout ?? TIMEOUT) * 2;
item.result = result;
this.queue.push(item);
this.saveState();
} else {
item.result = result;
this.history.push(item);
this.saveState();
}
} catch (err) {
item.result = {
@ -206,7 +159,6 @@ class QueueManager {
error: err instanceof Error ? err.message : 'Unknown error'
};
this.history.push(item);
this.saveState();
}
this.processing.splice(this.processing.indexOf(current), 1);

58
backend/services/lucida.ts
View file

@ -2,6 +2,7 @@ import {DownloadResult, LucidaOptions} from "../types/queue";
import {Browser, BrowserContext, Download, firefox, Locator, Page, Response as PResponse} from "playwright";
import path from "node:path";
import {fetchBooklet} from "./booklet";
import * as fs from "node:fs";
class Lucida {
private browser: Browser | null;
@ -76,17 +77,18 @@ class Lucida {
private async saveDownload(download: Download, downloadPath: string): Promise<string> {
const pathName: string = path.join(downloadPath, download.suggestedFilename());
await download.saveAs(pathName);
// await download.saveAs(pathName);
// return pathName;
// const stream = fs.createWriteStream(pathName);
// const response = await download.createReadStream();
// if (response) {
// response.pipe(stream);
// await new Promise((resolve, reject) => {
// stream.on('finish', resolve);
// stream.on('error', reject);
// });
// }
const stream = fs.createWriteStream(pathName);
const response = await download.createReadStream();
if (response) {
response.pipe(stream);
await new Promise((resolve, reject) => {
stream.on('finish', resolve);
stream.on('error', reject);
});
}
return pathName;
}
@ -95,7 +97,6 @@ class Lucida {
status: string
}): Promise<DownloadResult> {
let finished: boolean = false;
let started: boolean = true;
let close: boolean = false;
if (this.context === null) {
@ -122,7 +123,7 @@ class Lucida {
// Check for errors
const error: string | undefined = await this.getError(downloadPage);
if (error !== undefined) {
if (error) {
await downloadPage.close();
return {
success: false,
@ -137,38 +138,42 @@ class Lucida {
// Listen for download
downloadPage.once('download', async (download) => {
started = true;
await this.saveDownload(download, downloadPath);
await downloadPage.close();
await fetchBooklet(url, downloadPath, this.context);
finished = true;
await fetchBooklet(url, downloadPath, this.context);
await downloadPage.close();
if (close) {
await this.destruct();
}
process.stdout.clearLine(0);
return {success: true};
});
// Start download
await downloadPage.getByText('download full album').click();
// Retry file download if it fails
const maxRetries: number = 10;
const progressElement: Locator = downloadPage.locator('div[class="loader svelte-1ipdo3f"]');
while (!started && retryCount < maxRetries && (Date.now() - start) < timeout) {
while (!finished && retryCount < 10 && (Date.now() - start) < timeout) {
const retry: Locator = downloadPage.getByText('Retry');
const error: string | undefined = started ? undefined : await this.getError(downloadPage);
const error: string | undefined = finished ? undefined : await this.getError(downloadPage);
if (error !== undefined) {
console.log('Retrying download because of error:', error);
await retry.click();
retryCount++;
} else if (!started && await progressElement.count() !== 0 && await progressElement.isVisible()) {
status.status = (await progressElement.locator('p').innerText()).trim();
} else if (!finished && await progressElement.count() !== 0 && await progressElement.isVisible()) {
const progress: string = (await progressElement.locator('p').innerText()).trim();
status.status = progress;
process.stdout.clearLine(0);
process.stdout.write(`\r${progress}`);
}
}
if (!started) {
process.stdout.clearLine(0);
if (!finished) {
// Cleanup
await downloadPage.close();
if (close) {
@ -178,8 +183,8 @@ class Lucida {
let errorMessage: string;
if (timeout <= (Date.now() - start)) {
errorMessage = 'Download timed out';
} else if (retryCount >= maxRetries) {
errorMessage = `Download failed after ${maxRetries} retries`;
} else if (retryCount >= 10) {
errorMessage = 'Download failed after 10 retries';
} else {
errorMessage = 'Unknown error';
}
@ -189,12 +194,7 @@ class Lucida {
};
}
// Wait for download to finish
await new Promise(resolve => setTimeout(resolve, 30000));
return {
success: finished,
error: finished ? undefined : 'Save to disk did not finish.'
};
return {success: true};
}
}

30
docker-compose.yml
View file

@ -1,30 +0,0 @@
services:
frontend:
build:
context: ./frontend
dockerfile: Dockerfile
ports:
- "8080:80"
depends_on:
- backend
restart: always
backend:
build:
context: ./backend
dockerfile: Dockerfile
ports:
- "3000:3000"
volumes:
- "/srv/lucida-queue:/data"
depends_on:
- tor
restart: always
tor:
container_name: tor
image: osminogin/tor-simple
ports:
- "9050:9050"
stop_grace_period: 1m
restart: always

7
frontend/Dockerfile
View file

@ -1,7 +0,0 @@
FROM nginx:alpine
COPY . /usr/share/nginx/html
EXPOSE 80
CMD ["nginx", "-g", "daemon off;"]

2
frontend/script.js
View file

@ -19,7 +19,7 @@ function setApiUrl() {
if (url === null || url.value === '') {
console.log('Using default API URL');
apiBaseUrl = `http://${window.location.hostname}:3000/api/v1`;
apiBaseUrl = 'http://127.0.0.1:3000/api/v1';
} else {
apiBaseUrl = url.value;
}

1682
backend/package-lock.json → package-lock.json generated
View file

File diff suppressed because it is too large Load diff

4
backend/package.json → package.json
View file

@ -1,8 +1,8 @@
{
"name": "lucida-queue",
"version": "0.1.0",
"version": "1.0.0",
"description": "Queue for downloading from lucida.to",
"main": "app.ts",
"main": "backend/app.ts",
"scripts": {
"build": "tsc",
"test": "echo \"Error: no test specified\" && exit 1",

13
backend/tsconfig.json → tsconfig.json
View file

@ -58,7 +58,7 @@
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "noEmit": true, /* Disable emitting files from a compilation. */
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
"outDir": "dist", /* Specify an output folder for all emitted files. */
"outDir": "./dist", /* Specify an output folder for all emitted files. */
// "removeComments": true, /* Disable emitting comments. */
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
@ -109,9 +109,12 @@
"skipLibCheck": true /* Skip type checking all .d.ts files. */
},
"include": [
"./**/*.ts",
"backend/**/*.ts",
"backend/**/*.ts",
"frontend/**/*.ts",
"backend/services/**/*.ts",
"backend/types/**/*.ts",
"backend/app.ts"
],
"exclude": [
"node_modules"
]
"exclude": ["node_modules"]
}