Merge pull request #140 from SELab-2/chore/development-workflow

chore: Development workflow verbeteringen

README.md

@@ -21,14 +21,16 @@ Alternatief kan je één van de volgende methodes gebruiken om de applicatie lok
 
 ### Quick start
 
+Om de applicatie lokaal te draaien als kant-en-klare Docker-containers:
+
 1. Installeer Docker en Docker Compose op je systeem (zie [Docker](https://docs.docker.com/get-docker/)
    en [Docker Compose](https://docs.docker.com/compose/)).
 2. Clone deze repository.
-3. In de backend, kopieer `.env.example` (of `.env.development.example`) naar `.env` en pas de variabelen aan waar
-   nodig.
-4. Voer `docker compose up` uit in de root van de repository.
+3. In de backend, kopieer `.env.example` naar `.env` en pas de variabelen aan waar nodig.
+4. Voer `docker compose -f compose.staging.yml up --build` uit in de root van de repository.
 5. Optioneel: Configureer de applicatie aan de hand van
    de [configuratiehandleiding](https://github.com/SELab-2/Dwengo-1/wiki/Administrator:-Productie-omgeving#dwengo-1-configuratie).
+6. De applicatie is nu beschikbaar op [`http://localhost/`](http://localhost/) en [`http://localhost/api`](http://localhost/api).
 
 ```bash
 docker compose version
@@ -38,14 +40,13 @@ cp .env.example .env
 # Pas .env aan
 nano .env
 cd ..
-docker compose up
-# Configureer de applicatie
+docker compose -f compose.staging.yml up --build
 ```
 
-### Handmatige installatie
+### Handmatige installatie en ontwikkeling
 
 Zie de submappen voor de installatie-instructies van de [frontend](./frontend/README.md)
-en [backend](./backend/README.md).
+en [backend](./backend/README.md) en instructies voor het opzetten van een ontwikkelomgeving.
 
 ## Architectuur
 

backend/.env.development.example

@@ -1,15 +1,24 @@
 #
-# Basic configuration
+# Development environment configuration
+#
+# You probably don't need to change these values, as this configuration takes
+# the docker services and their default ports into account.
 #
 
-DWENGO_PORT=3000 # The port the backend will listen on
+### Dwengo ###
+
+#DWENGO_PORT=3000
+#DWENGO_LEARNING_CONTENT_REPO_API_BASE_URL=https://dwengo.org/backend/api
+#DWENGO_FALLBACK_LANGUAGE=nl
+#DWENGO_RUN_MODE=dev
+
 DWENGO_DB_HOST=localhost
 DWENGO_DB_PORT=5431
+#DWENGO_DB_NAME=dwengo
 DWENGO_DB_USERNAME=postgres
 DWENGO_DB_PASSWORD=postgres
 DWENGO_DB_UPDATE=true
-
-# Auth
+#DWENGO_DB_CONTENT_PREFIX=u_
 
 DWENGO_AUTH_STUDENT_URL=http://localhost:7080/realms/student
 DWENGO_AUTH_STUDENT_CLIENT_ID=dwengo
@@ -17,12 +26,12 @@ DWENGO_AUTH_STUDENT_JWKS_ENDPOINT=http://localhost:7080/realms/student/protocol/
 DWENGO_AUTH_TEACHER_URL=http://localhost:7080/realms/teacher
 DWENGO_AUTH_TEACHER_CLIENT_ID=dwengo
 DWENGO_AUTH_TEACHER_JWKS_ENDPOINT=http://localhost:7080/realms/teacher/protocol/openid-connect/certs
+#DWENGO_AUTH_AUDIENCE=account
 
-# Allow Vite dev-server to access the backend (for testing purposes). Don't forget to remove this in production!
 DWENGO_CORS_ALLOWED_ORIGINS=http://localhost:5173
+#DWENGO_CORS_ALLOWED_HEADERS=Authorization,Content-Type
 
-#
-# Advanced configuration
-#
+### Advanced configuration ###
 
-# LOKI_HOST=http://localhost:9001      # The address of the Loki instance, used for logging
+DWENGO_LOGGING_LEVEL=debug
+#DWENGO_LOGGING_LOKI_HOST=http://localhost:3102

backend/.env.example

@@ -1,27 +1,68 @@
 #
 # Basic configuration
 #
+# Change the values of the variables below to match your environment!
+# Default values are commented out.
+#
 
-DWENGO_PORT=3000 # The port the backend will listen on
+### Dwengo ###
+
+# Port the backend will listen on
+#DWENGO_PORT=3000
+# The hostname or IP address of the remote learning content API.
+#DWENGO_LEARNING_CONTENT_REPO_API_BASE_URL=https://dwengo.org/backend/api
+# The default fallback language.
+#DWENGO_FALLBACK_LANGUAGE=nl
+# Whether running in production mode or not. Possible values are "prod" or "dev".
+#DWENGO_RUN_MODE=dev
+
+# ! Change this! The hostname or IP address of the database
+# If running your stack in docker, this should use the docker service name.
 DWENGO_DB_HOST=domain-or-ip-of-database
-DWENGO_DB_PORT=5431
-
-# Change this to the actual credentials of the user Dwengo should use in the backend
-DWENGO_DB_USERNAME=postgres
-DWENGO_DB_PASSWORD=postgres
-
+# The port of the database.
+#DWENGO_DB_PORT=5432
+# The name of the database.
+#DWENGO_DB_NAME=dwengo
+# ! Change this! The username of the database user.
+DWENGO_DB_USERNAME=username
+# ! Change this! The password of the database user.
+DWENGO_DB_PASSWORD=password
+# Whether the database scheme needs to be updated.
 # Set this to true when the database scheme needs to be updated. In that case, take a backup first.
-DWENGO_DB_UPDATE=false
+#DWENGO_DB_UPDATE=false
+# The prefix used for custom user content.
+#DWENGO_DB_CONTENT_PREFIX=u_
 
-# Data for the identity provider via which the students authenticate.
-DWENGO_AUTH_STUDENT_URL=http://localhost:7080/realms/student
+# ! Change this! The external URL for student authentication. Should be reachable by the client.
+# E.g. https://sel2-1.ugent.be/idp/realms/student
+DWENGO_AUTH_STUDENT_URL=http://hostname/idp/realms/student
+# ! Change this! The client ID for student authentication.
 DWENGO_AUTH_STUDENT_CLIENT_ID=dwengo
-DWENGO_AUTH_STUDENT_JWKS_ENDPOINT=http://localhost:7080/realms/student/protocol/openid-connect/certs
-
-# Data for the identity provider via which the teachers authenticate.
-DWENGO_AUTH_TEACHER_URL=http://localhost:7080/realms/teacher
+# ! Change this! The internal URL for retrieving the JWKS for student authentication.
+# Should be reachable by the backend. If running your stack in docker, this should use the docker service name.
+# E.g. http://idp:7080/realms/student/protocol/openid-connect/certs
+DWENGO_AUTH_STUDENT_JWKS_ENDPOINT=http://hostname/realms/student/protocol/openid-connect/certs
+# ! Change this! The external URL for teacher authentication. Should be reachable by the client.
+# E.g. https://sel2-1.ugent.be/idp/realms/teacher
+DWENGO_AUTH_TEACHER_URL=http://hostname/idp/realms/teacher
+# ! Change this! The client ID for teacher authentication.
 DWENGO_AUTH_TEACHER_CLIENT_ID=dwengo
-DWENGO_AUTH_TEACHER_JWKS_ENDPOINT=http://localhost:7080/realms/teacher/protocol/openid-connect/certs
+# ! Change this! The internal URL for retrieving the JWKS for teacher authentication.
+# Should be reachable by the backend. If running your stack in docker, this should use the docker service name.
+# E.g. http://idp:7080/realms/teacher/protocol/openid-connect/certs
+DWENGO_AUTH_TEACHER_JWKS_ENDPOINT=http://hostname/realms/teacher/protocol/openid-connect/certs
+# The IDP audience
+#DWENGO_AUTH_AUDIENCE=account
 
-# The address of the Lokiinstance, used for logging
-# LOKI_HOST=http://localhost:3102
+# Allowed origins for CORS requests. Separate multiple origins with a comma.
+#DWENGO_CORS_ALLOWED_ORIGINS=
+# Allowed headers for CORS requests. Separate multiple headers with a comma.
+#DWENGO_CORS_ALLOWED_HEADERS=Authorization,Content-Type
+
+### Advanced configuration ###
+
+# The logging level. Possible values are "debug", "info", "warn", "error".
+#DWENGO_LOGGING_LEVEL=info
+# The address of the Loki instance, a log aggregation system.
+# If running your stack in docker, this should use the docker service name.
+#DWENGO_LOGGING_LOKI_HOST=http://localhost:3102

backend/.env.production.example

@@ -1,28 +1,37 @@
-DWENGO_PORT=3000 # The port the backend will listen on
-DWENGO_DB_HOST=db # Name of the database container
-DWENGO_DB_PORT=5431
+#
+# Production environment configuration
+#
+# Change the values of the variables below to match your production environment!
+# See .env.example for more information.
+#
 
-# Change this to the actual credentials of the user Dwengo should use in the backend
+### Dwengo ###
+
+DWENGO_PORT=3000
+#DWENGO_LEARNING_CONTENT_REPO_API_BASE_URL=https://dwengo.org/backend/api
+#DWENGO_FALLBACK_LANGUAGE=nl
+DWENGO_RUN_MODE=prod
+
+DWENGO_DB_HOST=db
+DWENGO_DB_PORT=5432
 DWENGO_DB_NAME=postgres
 DWENGO_DB_USERNAME=postgres
 DWENGO_DB_PASSWORD=postgres
-
-# Set this to true when the database scheme needs to be updated. In that case, take a backup first.
 DWENGO_DB_UPDATE=false
+#DWENGO_DB_CONTENT_PREFIX=u_
 
-# Data for the identity provider via which the students authenticate.
 DWENGO_AUTH_STUDENT_URL=https://sel2-1.ugent.be/idp/realms/student
 DWENGO_AUTH_STUDENT_CLIENT_ID=dwengo
 DWENGO_AUTH_STUDENT_JWKS_ENDPOINT=http://idp:7080/idp/realms/student/protocol/openid-connect/certs # Name of the idp container
-# Data for the identity provider via which the teachers authenticate.
 DWENGO_AUTH_TEACHER_URL=https://sel2-1.ugent.be/idp/realms/teacher
 DWENGO_AUTH_TEACHER_CLIENT_ID=dwengo
 DWENGO_AUTH_TEACHER_JWKS_ENDPOINT=http://idp:7080/idp/realms/teacher/protocol/openid-connect/certs # Name of the idp container
+#DWENGO_AUTH_AUDIENCE=account
 
-#
-# Advanced configuration
-#
+#DWENGO_CORS_ALLOWED_ORIGINS=
+#DWENGO_CORS_ALLOWED_HEADERS=Authorization,Content-Type
 
-# Logging and monitoring
+### Advanced configuration ###
 
-# LOKI_HOST=http://logging:3102 # The address of the Loki instance, used for logging
+DWENGO_LOGGING_LEVEL=info
+DWENGO_LOGGING_LOKI_HOST=http://logging:3102

backend/.env.test.example

@@ -1,3 +1,13 @@
-PORT=3000
-DWENGO_DB_UPDATE=true
+#
+# Test environment configuration
+#
+# Should not need to be modified.
+# See .env.example for more information.
+#
+
+### Dwengo ###
+
+DWENGO_PORT=3000
+
 DWENGO_DB_NAME=":memory:"
+DWENGO_DB_UPDATE=true

backend/Dockerfile

@@ -30,6 +30,7 @@ COPY package-lock.json backend/package.json ./
 RUN npm install --silent --only=production
 
 COPY ./docs /docs
+COPY ./backend/i18n /app/i18n
 COPY --from=build-stage /app/backend/dist ./dist/
 
 EXPOSE 3000

backend/README.md

@@ -4,23 +4,24 @@
 
 ```shell
 npm install
+
+# Start de nodige services voor ontwikkeling
+cd ../ # Ga naar de root van de repository
+docker compose up -d
 ```
 
-Setup the environment variables in a `.env` file in the root of the project. You can use the `.env.example` file as a template.
+Zet de omgevingsvariabelen in een `.env` bestand in de root van het project.
+Je kan het `.env.example` bestand als template gebruiken.
 
-### Development
+### Ontwikkeling
 
 ```shell
+# Omgevingsvariabelen
+cp .env.development.example .env.development.local
+
 npm run dev
 ```
 
-### Production
-
-```shell
-npm run build
-npm run start
-```
-
 ### Tests
 
 Voer volgend commando uit om de unit tests uit te voeren:
@@ -29,6 +30,18 @@ Voer volgend commando uit om de unit tests uit te voeren:
 npm run test:unit
 ```
 
+### Productie
+
+```shell
+# Omgevingsvariabelen
+cp .env.development.example .env
+
+npm run build
+npm run start
+```
+
+Zie ook de [installatiehandleiding](https://github.com/SELab-2/Dwengo-1/wiki/Administrator:-Productie-omgeving).
+
 ## Keycloak configuratie
 
 Tijdens development is het voldoende om gebruik te maken van de keycloak configuratie die automatisch ingeladen wordt.

backend/src/config.ts

@@ -1,12 +1,7 @@
 import { EnvVars, getEnvVar } from './util/envvars.js';
-import { Language } from './entities/content/language.js';
 
 // API
 export const DWENGO_API_BASE = getEnvVar(EnvVars.LearningContentRepoApiBaseUrl);
 export const FALLBACK_LANG = getEnvVar(EnvVars.FallbackLanguage);
 
-// Logging
-export const LOG_LEVEL: string = 'development' === process.env.NODE_ENV ? 'debug' : 'info';
-export const LOKI_HOST: string = process.env.LOKI_HOST || 'http://localhost:3102';
-
 export const FALLBACK_SEQ_NUM = 1;

backend/src/logging/initalize.ts

@@ -1,7 +1,7 @@
 import { createLogger, format, Logger as WinstonLogger, transports } from 'winston';
 import LokiTransport from 'winston-loki';
 import { LokiLabels } from 'loki-logger-ts';
-import { LOG_LEVEL, LOKI_HOST } from '../config.js';
+import { EnvVars, getEnvVar } from '../util/envvars.js';
 
 export class Logger extends WinstonLogger {
     constructor() {
@@ -22,10 +22,25 @@ function initializeLogger(): Logger {
         return logger;
     }
 
+    const logLevel = getEnvVar(EnvVars.LogLevel);
+
+    const consoleTransport = new transports.Console({
+        level: getEnvVar(EnvVars.LogLevel),
+        format: format.combine(format.cli(), format.colorize()),
+    });
+
+    if (getEnvVar(EnvVars.RunMode) === 'dev') {
+        return createLogger({
+            transports: [consoleTransport],
+        });
+    }
+
+    const lokiHost = getEnvVar(EnvVars.LokiHost);
+
     const lokiTransport: LokiTransport = new LokiTransport({
-        host: LOKI_HOST,
+        host: lokiHost,
         labels: Labels,
-        level: LOG_LEVEL,
+        level: logLevel,
         json: true,
         format: format.combine(format.timestamp(), format.json()),
         onConnectionError: (err) => {
@@ -34,16 +49,11 @@ function initializeLogger(): Logger {
         },
     });
 
-    const consoleTransport = new transports.Console({
-        level: LOG_LEVEL,
-        format: format.combine(format.cli(), format.colorize()),
-    });
-
     logger = createLogger({
         transports: [lokiTransport, consoleTransport],
     });
 
-    logger.debug(`Logger initialized with level ${LOG_LEVEL}, Loki host ${LOKI_HOST}`);
+    logger.debug(`Logger initialized with level ${logLevel} to Loki host ${lokiHost}`);
     return logger;
 }
 

backend/src/mikro-orm.config.ts

@@ -3,7 +3,6 @@ import { PostgreSqlDriver } from '@mikro-orm/postgresql';
 import { EnvVars, getEnvVar, getNumericEnvVar } from './util/envvars.js';
 import { SqliteDriver } from '@mikro-orm/sqlite';
 import { MikroOrmLogger } from './logging/mikroOrmLogger.js';
-import { LOG_LEVEL } from './config.js';
 
 // Import alle entity-bestanden handmatig
 import { User } from './entities/users/user.entity.js';
@@ -69,7 +68,7 @@ function config(testingMode: boolean = false): Options {
         // EntitiesTs: entitiesTs,
 
         // Logging
-        debug: LOG_LEVEL === 'debug',
+        debug: getEnvVar(EnvVars.LogLevel) === 'debug',
         loggerFactory: (options: LoggerOptions) => new MikroOrmLogger(options),
     };
 }

backend/src/util/envvars.ts

@@ -4,20 +4,24 @@ const IDP_PREFIX = PREFIX + 'AUTH_';
 const STUDENT_IDP_PREFIX = IDP_PREFIX + 'STUDENT_';
 const TEACHER_IDP_PREFIX = IDP_PREFIX + 'TEACHER_';
 const CORS_PREFIX = PREFIX + 'CORS_';
+const LOGGING_PREFIX = PREFIX + 'LOGGING_';
 
 type EnvVar = { key: string; required?: boolean; defaultValue?: any };
 
 export const EnvVars: { [key: string]: EnvVar } = {
     Port: { key: PREFIX + 'PORT', defaultValue: 3000 },
+    LearningContentRepoApiBaseUrl: { key: PREFIX + 'LEARNING_CONTENT_REPO_API_BASE_URL', defaultValue: 'https://dwengo.org/backend/api' },
+    FallbackLanguage: { key: PREFIX + 'FALLBACK_LANGUAGE', defaultValue: 'nl' },
+    RunMode: { key: PREFIX + 'RUN_MODE', defaultValue: 'dev' },
+
     DbHost: { key: DB_PREFIX + 'HOST', required: true },
     DbPort: { key: DB_PREFIX + 'PORT', defaultValue: 5432 },
     DbName: { key: DB_PREFIX + 'NAME', defaultValue: 'dwengo' },
     DbUsername: { key: DB_PREFIX + 'USERNAME', required: true },
     DbPassword: { key: DB_PREFIX + 'PASSWORD', required: true },
     DbUpdate: { key: DB_PREFIX + 'UPDATE', defaultValue: false },
-    LearningContentRepoApiBaseUrl: { key: PREFIX + 'LEARNING_CONTENT_REPO_API_BASE_URL', defaultValue: 'https://dwengo.org/backend/api' },
-    FallbackLanguage: { key: PREFIX + 'FALLBACK_LANGUAGE', defaultValue: 'nl' },
     UserContentPrefix: { key: DB_PREFIX + 'USER_CONTENT_PREFIX', defaultValue: 'u_' },
+
     IdpStudentUrl: { key: STUDENT_IDP_PREFIX + 'URL', required: true },
     IdpStudentClientId: { key: STUDENT_IDP_PREFIX + 'CLIENT_ID', required: true },
     IdpStudentJwksEndpoint: { key: STUDENT_IDP_PREFIX + 'JWKS_ENDPOINT', required: true },
@@ -25,8 +29,12 @@ export const EnvVars: { [key: string]: EnvVar } = {
     IdpTeacherClientId: { key: TEACHER_IDP_PREFIX + 'CLIENT_ID', required: true },
     IdpTeacherJwksEndpoint: { key: TEACHER_IDP_PREFIX + 'JWKS_ENDPOINT', required: true },
     IdpAudience: { key: IDP_PREFIX + 'AUDIENCE', defaultValue: 'account' },
+
     CorsAllowedOrigins: { key: CORS_PREFIX + 'ALLOWED_ORIGINS', defaultValue: '' },
     CorsAllowedHeaders: { key: CORS_PREFIX + 'ALLOWED_HEADERS', defaultValue: 'Authorization,Content-Type' },
+
+    LogLevel: { key: LOGGING_PREFIX + 'LEVEL', defaultValue: 'info' },
+    LokiHost: { key: LOGGING_PREFIX + 'LOKI_HOST', defaultValue: 'http://localhost:3102' },
 } as const;
 
 /**

backend/src/util/translation-helper.ts

@@ -8,12 +8,12 @@ const logger: Logger = getLogger();
 
 export function loadTranslations<T>(language: string): T {
     try {
-        const filePath = path.join(process.cwd(), '_i18n', `${language}.yml`);
+        const filePath = path.join(process.cwd(), 'i18n', `${language}.yml`);
         const yamlFile = fs.readFileSync(filePath, 'utf8');
         return yaml.load(yamlFile) as T;
     } catch (error) {
         logger.warn(`Cannot load translation for ${language}, fallen back to dutch`, error);
-        const fallbackPath = path.join(process.cwd(), '_i18n', `${FALLBACK_LANG}.yml`);
+        const fallbackPath = path.join(process.cwd(), 'i18n', `${FALLBACK_LANG}.yml`);
         return yaml.load(fs.readFileSync(fallbackPath, 'utf8')) as T;
     }
 }

compose.production.yml

@@ -1,7 +1,8 @@
 #
-# This file is used to define the production environment for the project.
-# It is used to deploy the project on a server.
-# Should not be used for local development.
+# Use this configuration to deploy the project on a server.
+#
+# This configuration builds the frontend and backend services as Docker images,
+# and uses the paths for the services, instead of ports, and enables SSL.
 #
 services:
     web:
@@ -35,12 +36,16 @@ services:
             - 'traefik.http.services.api.loadbalancer.server.port=3000'
 
     db:
-        # Also see compose.yml
+        extends:
+            file: ./compose.yml
+            service: db
         networks:
             - dwengo-1
 
     idp:
-        # Also see compose.yml
+        extends:
+            file: ./compose.yml
+            service: idp
         # TODO Replace with proper production command
         command: ['start-dev', '--http-port', '7080', '--https-port', '7443', '--import-realm']
         networks:
@@ -92,7 +97,15 @@ services:
             - dwengo-1
 
     logging:
-        # Also see compose.yml
+        image: grafana/loki:latest
+        ports:
+            - '9001:3102'
+            - '9095:9095'
+        command: -config.file=/etc/loki/config.yaml
+        restart: unless-stopped
+        volumes:
+            - ./config/loki/config.yml:/etc/loki/config.yaml
+            - dwengo_loki_data:/loki
         networks:
             - dwengo-1
 
@@ -107,6 +120,7 @@ services:
 volumes:
     dwengo_grafana_data:
     dwengo_letsencrypt:
+    dwengo_loki_data:
 
 networks:
     dwengo-1:

compose.staging.yml

@@ -32,8 +32,15 @@ services:
             - 'traefik.http.routers.api.rule=PathPrefix(`/api`)'
             - 'traefik.http.services.api.loadbalancer.server.port=3000'
 
+    db:
+        extends:
+            file: ./compose.yml
+            service: db
+
     idp:
-        # Also see compose.yml
+        extends:
+            file: ./compose.yml
+            service: idp
         labels:
             - 'traefik.http.routers.idp.rule=PathPrefix(`/idp`)'
             - 'traefik.http.services.idp.loadbalancer.server.port=7080'
@@ -60,6 +67,17 @@ services:
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock:ro
 
+    logging:
+        image: grafana/loki:latest
+        ports:
+            - '9001:3102'
+            - '9095:9095'
+        command: -config.file=/etc/loki/config.yaml
+        restart: unless-stopped
+        volumes:
+            - ./config/loki/config.yml:/etc/loki/config.yaml
+            - dwengo_loki_data:/loki
+
     dashboards:
         image: grafana/grafana:latest
         ports:
@@ -70,3 +88,5 @@ services:
 
 volumes:
     dwengo_grafana_data:
+    dwengo_loki_data:
+    dwengo_postgres_data:

compose.yml

@@ -36,17 +36,5 @@ services:
             KC_HEALTH_ENABLED: 'true'
             KC_LOG_LEVEL: info
 
-    logging:
-        image: grafana/loki:latest
-        ports:
-            - '9001:3102'
-            - '9095:9095'
-        command: -config.file=/etc/loki/config.yaml
-        restart: unless-stopped
-        volumes:
-            - ./config/loki/config.yml:/etc/loki/config.yaml
-            - dwengo_loki_data:/loki
-
 volumes:
-    dwengo_loki_data:
     dwengo_postgres_data:

frontend/README.md

@@ -19,7 +19,16 @@ See [Vite Configuration Reference](https://vite.dev/config/).
 ## Project Setup
 
 ```sh
+# Install dependencies
 npm install
+
+# Start necessary services for development
+cd ../ # Go to the root of the repository
+docker compose up -d
+# Start the backend
+cd backend
+cp .env.development.example .env.development.local
+npm run dev # or npm run build && npm run start
 ```
 
 ### Compile and Hot-Reload for Development