From 86ba4ea11e5184b57b3da84324de32b18f2a9356 Mon Sep 17 00:00:00 2001 From: Gerald Schmittinger Date: Mon, 5 May 2025 23:15:22 +0200 Subject: [PATCH] feat(backend): Verwerking van leerobjecten in ZIP-formaat. --- backend/package.json | 4 + .../learning-object-service.ts | 23 +++- .../learning-object-zip-processing-service.ts | 63 +++++++++++ package-lock.json | 103 ++++++++++++++++++ 4 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 backend/src/services/learning-objects/learning-object-zip-processing-service.ts diff --git a/backend/package.json b/backend/package.json index 7943d61d..7b2ab878 100644 --- a/backend/package.json +++ b/backend/package.json @@ -23,6 +23,8 @@ "@mikro-orm/postgresql": "6.4.12", "@mikro-orm/reflection": "6.4.12", "@mikro-orm/sqlite": "6.4.12", + "@types/mime-types": "^2.1.4", + "@types/unzipper": "^0.10.11", "axios": "^1.8.2", "cors": "^2.8.5", "cross": "^1.0.0", @@ -37,8 +39,10 @@ "jwks-rsa": "^3.1.0", "loki-logger-ts": "^1.0.2", "marked": "^15.0.7", + "mime-types": "^3.0.1", "response-time": "^2.3.3", "swagger-ui-express": "^5.0.1", + "unzipper": "^0.12.3", "uuid": "^11.1.0", "winston": "^3.17.0", "winston-loki": "^6.1.3" diff --git a/backend/src/services/learning-objects/learning-object-service.ts b/backend/src/services/learning-objects/learning-object-service.ts index 7b4f47fc..4f5409fd 100644 --- a/backend/src/services/learning-objects/learning-object-service.ts +++ b/backend/src/services/learning-objects/learning-object-service.ts @@ -2,7 +2,13 @@ import dwengoApiLearningObjectProvider from './dwengo-api-learning-object-provid import { LearningObjectProvider } from './learning-object-provider.js'; import { envVars, getEnvVar } from '../../util/envVars.js'; import databaseLearningObjectProvider from './database-learning-object-provider.js'; -import { FilteredLearningObject, LearningObjectIdentifierDTO, LearningPathIdentifier } from '@dwengo-1/common/interfaces/learning-content'; +import { + FilteredLearningObject, + LearningObjectIdentifierDTO, + LearningPathIdentifier +} from '@dwengo-1/common/interfaces/learning-content'; +import {getLearningObjectRepository} from "../../data/repositories"; +import {processLearningObjectZip} from "./learning-object-zip-processing-service"; function getProvider(id: LearningObjectIdentifierDTO): LearningObjectProvider { if (id.hruid.startsWith(getEnvVar(envVars.UserContentPrefix))) { @@ -42,6 +48,21 @@ const learningObjectService = { async getLearningObjectHTML(id: LearningObjectIdentifierDTO): Promise { return getProvider(id).getLearningObjectHTML(id); }, + + + /** + * Store the learning object in the given zip file in the database. + */ + async storeLearningObject(learningObjectPath: string): Promise { + const learningObjectRepository = getLearningObjectRepository(); + const learningObject = await processLearningObjectZip(learningObjectPath); + + if (!learningObject.hruid.startsWith(getEnvVar(envVars.UserContentPrefix))) { + throw Error("Learning object name must start with the user content prefix!"); + } + + await learningObjectRepository.save(learningObject, {preventOverwrite: true}); + } }; export default learningObjectService; diff --git a/backend/src/services/learning-objects/learning-object-zip-processing-service.ts b/backend/src/services/learning-objects/learning-object-zip-processing-service.ts new file mode 100644 index 00000000..3dbd5915 --- /dev/null +++ b/backend/src/services/learning-objects/learning-object-zip-processing-service.ts @@ -0,0 +1,63 @@ +import unzipper from 'unzipper'; +import mime from 'mime-types'; +import {LearningObjectMetadata} from "@dwengo-1/common/dist/interfaces/learning-content"; +import {LearningObject} from "../../entities/content/learning-object.entity"; +import {getAttachmentRepository, getLearningObjectRepository} from "../../data/repositories"; + +/** + * Process an uploaded zip file and construct a LearningObject from its contents. + * @param filePath Path of the zip file to process. + */ +export async function processLearningObjectZip(filePath: string): Promise { + const learningObjectRepo = getLearningObjectRepository(); + const attachmentRepo = getAttachmentRepository(); + + const zip = await unzipper.Open.file(filePath); + + let metadata: LearningObjectMetadata | null = null; + const attachments: {name: string, content: Buffer}[] = []; + let content: Buffer | null = null; + + for (const file of zip.files) { + if (file.type === "Directory") { + throw Error("The learning object zip file should not contain directories."); + } else if (file.path === "metadata.json") { + metadata = await processMetadataJson(file); + } else if (file.path.startsWith("index.")) { + content = await processFile(file); + } else { + attachments.push({ + name: file.path, + content: await processFile(file) + }); + } + } + + if (!metadata) { + throw Error("Missing metadata.json file"); + } + if (!content) { + throw Error("Missing index file"); + } + + const learningObject = learningObjectRepo.create(metadata); + const attachmentEntities = attachments.map(it => attachmentRepo.create({ + name: it.name, + content: it.content, + mimeType: mime.lookup(it.name) || "text/plain", + learningObject + })) + learningObject.attachments.push(...attachmentEntities); + + return learningObject; +} + +async function processMetadataJson(file: unzipper.File): LearningObjectMetadata { + const buf = await file.buffer(); + const content = buf.toString(); + return JSON.parse(content); +} + +async function processFile(file: unzipper.File): Promise { + return await file.buffer(); +} diff --git a/package-lock.json b/package-lock.json index d1a3b3a7..221d9fb5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,8 @@ "@mikro-orm/postgresql": "6.4.12", "@mikro-orm/reflection": "6.4.12", "@mikro-orm/sqlite": "6.4.12", + "@types/mime-types": "^2.1.4", + "@types/unzipper": "^0.10.11", "axios": "^1.8.2", "cors": "^2.8.5", "cross": "^1.0.0", @@ -50,8 +52,10 @@ "jwks-rsa": "^3.1.0", "loki-logger-ts": "^1.0.2", "marked": "^15.0.7", + "mime-types": "^3.0.1", "response-time": "^2.3.3", "swagger-ui-express": "^5.0.1", + "unzipper": "^0.12.3", "uuid": "^11.1.0", "winston": "^3.17.0", "winston-loki": "^6.1.3" @@ -1716,6 +1720,12 @@ "version": "1.3.5", "license": "MIT" }, + "node_modules/@types/mime-types": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz", + "integrity": "sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==", + "license": "MIT" + }, "node_modules/@types/ms": { "version": "2.1.0", "license": "MIT" @@ -1784,6 +1794,15 @@ "license": "MIT", "optional": true }, + "node_modules/@types/unzipper": { + "version": "0.10.11", + "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.11.tgz", + "integrity": "sha512-D25im2zjyMCcgL9ag6N46+wbtJBnXIr7SI4zHf9eJD2Dw2tEB5e+p5MYkrxKIVRscs5QV0EhtU9rgXSPx90oJg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/web-bluetooth": { "version": "0.0.21", "license": "MIT" @@ -2711,6 +2730,12 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "2.2.0", "license": "MIT", @@ -3259,6 +3284,12 @@ "url": "https://github.com/sponsors/mesqueeb" } }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, "node_modules/cors": { "version": "2.8.5", "license": "MIT", @@ -3515,6 +3546,45 @@ "node": ">= 0.4" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/dwengo-1-docs": { "resolved": "docs", "link": true @@ -5053,6 +5123,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, "node_modules/isexe": { "version": "2.0.0", "license": "ISC" @@ -5820,6 +5896,8 @@ }, "node_modules/mime-types": { "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz", + "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==", "license": "MIT", "dependencies": { "mime-db": "^1.54.0" @@ -6165,6 +6243,12 @@ "node": ">=6" } }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "license": "MIT" + }, "node_modules/node-releases": { "version": "2.0.19", "dev": true, @@ -6843,6 +6927,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/promise-inflight": { "version": "1.0.1", "license": "ISC", @@ -8411,6 +8501,19 @@ "node": ">= 0.8" } }, + "node_modules/unzipper": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.12.3.tgz", + "integrity": "sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA==", + "license": "MIT", + "dependencies": { + "bluebird": "~3.7.2", + "duplexer2": "~0.1.4", + "fs-extra": "^11.2.0", + "graceful-fs": "^4.2.2", + "node-int64": "^0.4.0" + } + }, "node_modules/update-browserslist-db": { "version": "1.1.3", "dev": true,