From 77a0673e03cc12b8bbeb179ca046b273a08a6950 Mon Sep 17 00:00:00 2001 From: Max <53796487+dyedwiper@users.noreply.github.com> Date: Mon, 27 Jan 2025 16:41:55 +0100 Subject: [PATCH 1/5] BC-8440 Remove dev dependencies from docker image (#5460) --- Dockerfile | 55 ++++-- .../api-h5p-library-management-cronjob.yml.j2 | 2 +- .../templates/vidis-sync-cronjob.yml.j2 | 2 +- .../media-metadata-sync-cronjob.yml.j2 | 2 +- ...ule-users-deletion-queueing-cronjob.yml.j2 | 2 +- .../moin-schule-users-sync-cronjob.yml.j2 | 2 +- .../api-delete-s3-files-cronjob.yml.j2 | 2 +- .../data-deletion-trigger-cronjob.yml.j2 | 2 +- .../templates/deployment.yml.j2 | 1 + .../templates/migration-job.yml.j2 | 2 +- .../api-ldap-worker-deployment.yml.j2 | 1 + .../templates/deployment.yml.j2 | 1 + .../templates/api-tsp-sync-cronjob.yml.j2 | 2 +- .../templates/api-tsp-sync-deployment.yml.j2 | 1 + .../templates/api-tsp-sync-init.yml.j2 | 2 +- .../media-available-line.uc.spec.ts | 3 +- .../management/management.module.spec.ts | 1 - .../seed-data/factory/base.factory.ts | 164 ++++++++++++++++++ .../factory/county.embeddable.factory.ts | 17 ++ .../factory/federal-state.factory.ts | 18 ++ .../seed-data/factory/role.factory.ts | 9 + .../factory/school.entity.factory.ts | 12 ++ .../seed-data/factory/schoolyear.factory.ts | 32 ++++ .../factory/system.entity.factory.ts | 99 +++++++++++ .../management/seed-data/federalstates.ts | 2 +- .../src/modules/management/seed-data/roles.ts | 2 +- .../modules/management/seed-data/schools.ts | 4 +- .../management/seed-data/schoolyears.ts | 2 +- .../modules/management/seed-data/systems.ts | 2 +- ...ool-external-tool-created.loggable.spec.ts | 2 +- ...onnex-license-provisioning.service.spec.ts | 8 +- ...ulconnex-tool-provisioning.service.spec.ts | 3 +- .../modules/rocketchat-user/domain/index.ts | 1 - apps/server/src/modules/room/index.ts | 1 - .../modules/synchronization/domain/index.ts | 1 - .../tool-configuration-status.service.spec.ts | 3 +- apps/server/src/modules/user-license/index.ts | 1 - apps/server/tsconfig.app.json | 5 +- esbuild/esmodules-bundler.js | 2 +- package-lock.json | 6 +- package.json | 24 ++- scripts/ldapSync.sh | 2 +- sonar-project.properties | 2 +- 43 files changed, 441 insertions(+), 66 deletions(-) create mode 100644 apps/server/src/modules/management/seed-data/factory/base.factory.ts create mode 100644 apps/server/src/modules/management/seed-data/factory/county.embeddable.factory.ts create mode 100644 apps/server/src/modules/management/seed-data/factory/federal-state.factory.ts create mode 100644 apps/server/src/modules/management/seed-data/factory/role.factory.ts create mode 100644 apps/server/src/modules/management/seed-data/factory/school.entity.factory.ts create mode 100644 apps/server/src/modules/management/seed-data/factory/schoolyear.factory.ts create mode 100644 apps/server/src/modules/management/seed-data/factory/system.entity.factory.ts diff --git a/Dockerfile b/Dockerfile index 98a1f7fb8f4..68b6edf2430 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,46 @@ -FROM docker.io/node:22 AS git +FROM docker.io/node:22-alpine AS builder -RUN mkdir /app && chown -R node:node /app WORKDIR /app -COPY .git . -RUN git config --global --add safe.directory /app && echo "{\"sha\": \"$(git rev-parse HEAD)\", \"version\": \"$(git describe --tags --abbrev=0)\", \"commitDate\": \"$(git log -1 --format=%cd --date=format:'%Y-%m-%dT%H:%M:%SZ')\", \"birthdate\": \"$(date +%Y-%m-%dT%H:%M:%SZ)\"}" > /app/serverversion + +COPY package.json package-lock.json tsconfig.json tsconfig.build.json nest-cli.json ./ +COPY apps apps +COPY config config +COPY esbuild esbuild +COPY src src + +RUN apk add --no-cache git +COPY .git ./.git +RUN git config --global --add safe.directory /app \ + && echo "{\"sha\": \"$(git rev-parse HEAD)\", \"version\": \"$(git describe --tags --abbrev=0)\", \"commitDate\": \"$(git log -1 --format=%cd --date=format:'%Y-%m-%dT%H:%M:%SZ')\", \"birthdate\": \"$(date +%Y-%m-%dT%H:%M:%SZ)\"}" > apps/server/static-assets/serverversion + +RUN npm ci && npm run build + FROM docker.io/node:22-alpine + ENV TZ=Europe/Berlin -RUN apk add --no-cache git make python3 -# to run ldap sync as script curl is needed -RUN apk add --no-cache curl +RUN apk add --no-cache python3 curl + WORKDIR /schulcloud-server -COPY tsconfig.json tsconfig.build.json package.json package-lock.json .eslintrc.js .eslintignore nest-cli.json ./ -COPY esbuild ./esbuild -RUN npm ci && npm cache clean --force -COPY config /schulcloud-server/config -COPY backup /schulcloud-server/backup -COPY src /schulcloud-server/src -COPY apps /schulcloud-server/apps -COPY --from=git /app/serverversion /schulcloud-server/apps/server/static-assets -COPY scripts/ldapSync.sh /schulcloud-server/scripts/ -RUN npm run build + +COPY package.json package-lock.json ./ +COPY backup backup +COPY config config +COPY scripts/ldapSync.sh scripts/ +COPY src src + +COPY --from=builder /app/dist dist + +# The postinstall script must be disabled, because esbuild is a dev dependency and not installed here. +RUN npm pkg delete scripts.postinstall \ + && npm ci --omit=dev \ + && npm cache clean --force + +# The modules transpiled by esbuild need to be copied manually from the build stage. +COPY --from=builder /app/node_modules/@keycloak/keycloak-admin-client-cjs node_modules/@keycloak/keycloak-admin-client-cjs +COPY --from=builder /app/node_modules/file-type-cjs node_modules/file-type-cjs ENV NODE_ENV=production ENV NO_COLOR="true" -CMD npm run start + +CMD npm run nest:start:prod diff --git a/ansible/roles/h5p-library-management/templates/api-h5p-library-management-cronjob.yml.j2 b/ansible/roles/h5p-library-management/templates/api-h5p-library-management-cronjob.yml.j2 index cc74146e055..4dc31a84188 100644 --- a/ansible/roles/h5p-library-management/templates/api-h5p-library-management-cronjob.yml.j2 +++ b/ansible/roles/h5p-library-management/templates/api-h5p-library-management-cronjob.yml.j2 @@ -57,7 +57,7 @@ spec: subPath: h5p-libraries.yaml readOnly: true command: ['/bin/sh', '-c'] - args: ['npm run nest:start:h5p:library-management'] + args: ['npm run nest:start:h5p:library-management:prod'] resources: limits: cpu: {{ API_H5P_LIBRARY_MANAGEMENT_CPU_LIMITS|default("2000m", true) }} diff --git a/ansible/roles/media-licenses/templates/vidis-sync-cronjob.yml.j2 b/ansible/roles/media-licenses/templates/vidis-sync-cronjob.yml.j2 index 10540f7d4e6..855bc85a853 100644 --- a/ansible/roles/media-licenses/templates/vidis-sync-cronjob.yml.j2 +++ b/ansible/roles/media-licenses/templates/vidis-sync-cronjob.yml.j2 @@ -40,7 +40,7 @@ spec: - configMapRef: name: vidis-sync-cronjob-configmap command: ['/bin/sh','-c'] - args: ['npm run nest:start:sync:vidis'] + args: ['npm run nest:start:sync:vidis:prod'] resources: limits: cpu: {{ MEDIA_ACTIVATION_CPU_LIMITS|default("2000m", true) }} diff --git a/ansible/roles/media-metadata-sync/templates/media-metadata-sync-cronjob.yml.j2 b/ansible/roles/media-metadata-sync/templates/media-metadata-sync-cronjob.yml.j2 index ec55dbb3a6a..99a5108779c 100644 --- a/ansible/roles/media-metadata-sync/templates/media-metadata-sync-cronjob.yml.j2 +++ b/ansible/roles/media-metadata-sync/templates/media-metadata-sync-cronjob.yml.j2 @@ -40,7 +40,7 @@ spec: - configMapRef: name: media-metadata-sync-cronjob-configmap command: ['/bin/sh','-c'] - args: ['npm run nest:start:sync:media-metadata'] + args: ['npm run nest:start:sync:media-metadata:prod'] resources: limits: cpu: {{ MEDIA_METADATA_SYNC_CPU_LIMITS|default("2000m", true) }} diff --git a/ansible/roles/moin-schule-sync/templates/moin-schule-users-deletion-queueing-cronjob.yml.j2 b/ansible/roles/moin-schule-sync/templates/moin-schule-users-deletion-queueing-cronjob.yml.j2 index 76cdf951ece..a336774e331 100644 --- a/ansible/roles/moin-schule-sync/templates/moin-schule-users-deletion-queueing-cronjob.yml.j2 +++ b/ansible/roles/moin-schule-sync/templates/moin-schule-users-deletion-queueing-cronjob.yml.j2 @@ -30,7 +30,7 @@ spec: - secretRef: name: api-files-secret command: ['/bin/sh','-c'] - args: ['npm run nest:start:deletion-console -- queue unsynced --systemId $SYSTEM_ID'] + args: ['npm run nest:start:deletion-console:prod -- queue unsynced --systemId $SYSTEM_ID'] resources: limits: cpu: {{ API_CPU_LIMITS|default("2000m", true) }} diff --git a/ansible/roles/moin-schule-sync/templates/moin-schule-users-sync-cronjob.yml.j2 b/ansible/roles/moin-schule-sync/templates/moin-schule-users-sync-cronjob.yml.j2 index 341fd87a09f..30f78d57690 100644 --- a/ansible/roles/moin-schule-sync/templates/moin-schule-users-sync-cronjob.yml.j2 +++ b/ansible/roles/moin-schule-sync/templates/moin-schule-users-sync-cronjob.yml.j2 @@ -30,7 +30,7 @@ spec: - secretRef: name: api-files-secret command: ['/bin/sh','-c'] - args: ['npm run nest:start:idp-console -- sync users --systemType moin.schule --systemId $SYSTEM_ID'] + args: ['npm run nest:start:idp-console:prod -- sync users --systemType moin.schule --systemId $SYSTEM_ID'] resources: limits: cpu: {{ API_CPU_LIMITS|default("2000m", true) }} diff --git a/ansible/roles/schulcloud-server-core/templates/api-delete-s3-files-cronjob.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-delete-s3-files-cronjob.yml.j2 index 094296e0930..bda229f166d 100644 --- a/ansible/roles/schulcloud-server-core/templates/api-delete-s3-files-cronjob.yml.j2 +++ b/ansible/roles/schulcloud-server-core/templates/api-delete-s3-files-cronjob.yml.j2 @@ -36,7 +36,7 @@ spec: - secretRef: name: api-files-secret command: ['/bin/sh', '-c'] - args: ['npm run nest:start:console -- files cleanup-job 7'] + args: ['npm run nest:start:console:prod -- files cleanup-job 7'] resources: limits: cpu: {{ API_CPU_LIMITS|default("2000m", true) }} diff --git a/ansible/roles/schulcloud-server-core/templates/data-deletion-trigger-cronjob.yml.j2 b/ansible/roles/schulcloud-server-core/templates/data-deletion-trigger-cronjob.yml.j2 index d94cc135112..b8f4b9a9dbd 100644 --- a/ansible/roles/schulcloud-server-core/templates/data-deletion-trigger-cronjob.yml.j2 +++ b/ansible/roles/schulcloud-server-core/templates/data-deletion-trigger-cronjob.yml.j2 @@ -45,7 +45,7 @@ spec: - secretRef: name: api-files-secret command: ['/bin/sh', '-c'] - args: ['npm run nest:start:deletion-console -- execution trigger'] + args: ['npm run nest:start:deletion-console:prod -- execution trigger'] resources: limits: cpu: {{ API_CPU_LIMITS|default("2000m", true) }} diff --git a/ansible/roles/schulcloud-server-core/templates/deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/deployment.yml.j2 index 4aa30ff41b4..70dde61a101 100644 --- a/ansible/roles/schulcloud-server-core/templates/deployment.yml.j2 +++ b/ansible/roles/schulcloud-server-core/templates/deployment.yml.j2 @@ -65,6 +65,7 @@ spec: name: api-secret - secretRef: name: api-files-secret + command: ['npm', 'run', 'nest:start:prod'] readinessProbe: httpGet: path: /internal/health diff --git a/ansible/roles/schulcloud-server-core/templates/migration-job.yml.j2 b/ansible/roles/schulcloud-server-core/templates/migration-job.yml.j2 index 42edd22f4a0..262aca5d07c 100644 --- a/ansible/roles/schulcloud-server-core/templates/migration-job.yml.j2 +++ b/ansible/roles/schulcloud-server-core/templates/migration-job.yml.j2 @@ -28,7 +28,7 @@ spec: - secretRef: name: api-files-secret command: ['/bin/sh','-c'] - args: ['npm run ensureIndexes && npm run migration:up'] + args: ['npm run ensureIndexes:prod && npm run migration:up:prod'] resources: limits: cpu: {{ API_CPU_LIMITS|default("2000m", true) }} diff --git a/ansible/roles/schulcloud-server-ldapsync/templates/api-ldap-worker-deployment.yml.j2 b/ansible/roles/schulcloud-server-ldapsync/templates/api-ldap-worker-deployment.yml.j2 index 027ce4e0c74..541310806f8 100644 --- a/ansible/roles/schulcloud-server-ldapsync/templates/api-ldap-worker-deployment.yml.j2 +++ b/ansible/roles/schulcloud-server-ldapsync/templates/api-ldap-worker-deployment.yml.j2 @@ -61,6 +61,7 @@ spec: name: api-secret - secretRef: name: api-files-secret + command: ['npm', 'run', 'nest:start:prod'] readinessProbe: httpGet: path: /serverversion diff --git a/ansible/roles/schulcloud-server-migration-system/templates/deployment.yml.j2 b/ansible/roles/schulcloud-server-migration-system/templates/deployment.yml.j2 index ea0b242ee2c..6fefdfebfca 100644 --- a/ansible/roles/schulcloud-server-migration-system/templates/deployment.yml.j2 +++ b/ansible/roles/schulcloud-server-migration-system/templates/deployment.yml.j2 @@ -65,6 +65,7 @@ spec: name: api-secret - secretRef: name: api-files-secret + command: ['npm', 'run', 'nest:start:prod'] readinessProbe: httpGet: path: /internal/health diff --git a/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob.yml.j2 b/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob.yml.j2 index d2495657f72..86d5d13eaa2 100644 --- a/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob.yml.j2 +++ b/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob.yml.j2 @@ -55,7 +55,7 @@ spec: - secretRef: name: api-secret command: ['/bin/sh', '-c'] - args: ['npm run nest:start:sync tsp'] + args: ['npm run nest:start:sync:prod tsp'] resources: limits: cpu: {{ API_CPU_LIMITS|default("2000m", true) }} diff --git a/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-deployment.yml.j2 b/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-deployment.yml.j2 index 38a13ef7629..f2ccf1a49c6 100644 --- a/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-deployment.yml.j2 +++ b/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-deployment.yml.j2 @@ -62,6 +62,7 @@ spec: name: api-secret - secretRef: name: api-files-secret + command: ['npm', 'run', 'nest:start:prod'] readinessProbe: httpGet: path: /serverversion diff --git a/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-init.yml.j2 b/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-init.yml.j2 index 6c126f8b981..bac80cc915b 100644 --- a/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-init.yml.j2 +++ b/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-init.yml.j2 @@ -42,7 +42,7 @@ spec: - secretRef: name: api-secret command: ['/bin/sh', '-c'] - args: ['npm run nest:start:sync tsp'] + args: ['npm run nest:start:sync:prod tsp'] resources: limits: cpu: {{ API_CPU_LIMITS|default("2000m", true) }} diff --git a/apps/server/src/modules/board/uc/media-board/media-available-line.uc.spec.ts b/apps/server/src/modules/board/uc/media-board/media-available-line.uc.spec.ts index 8ca51e04adf..b71543310fc 100644 --- a/apps/server/src/modules/board/uc/media-board/media-available-line.uc.spec.ts +++ b/apps/server/src/modules/board/uc/media-board/media-available-line.uc.spec.ts @@ -7,7 +7,8 @@ import { ExternalTool } from '@modules/tool/external-tool/domain'; import { externalToolFactory } from '@modules/tool/external-tool/testing'; import { SchoolExternalTool } from '@modules/tool/school-external-tool/domain'; import { schoolExternalToolFactory } from '@modules/tool/school-external-tool/testing'; -import { MediaUserLicense, mediaUserLicenseFactory, MediaUserLicenseService } from '@modules/user-license'; +import { MediaUserLicense, MediaUserLicenseService } from '@modules/user-license'; +import { mediaUserLicenseFactory } from '@modules/user-license/testing'; import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { FeatureDisabledLoggableException } from '@shared/common/loggable-exception'; diff --git a/apps/server/src/modules/management/management.module.spec.ts b/apps/server/src/modules/management/management.module.spec.ts index d259134ee4a..ecac3cc9fc8 100644 --- a/apps/server/src/modules/management/management.module.spec.ts +++ b/apps/server/src/modules/management/management.module.spec.ts @@ -1,5 +1,4 @@ import { Test, TestingModule } from '@nestjs/testing'; - import { DatabaseManagementController } from './controller/database-management.controller'; import { ManagementServerModule, ManagementServerTestModule } from './management-server.app.module'; import { DatabaseManagementUc } from './uc/database-management.uc'; diff --git a/apps/server/src/modules/management/seed-data/factory/base.factory.ts b/apps/server/src/modules/management/seed-data/factory/base.factory.ts new file mode 100644 index 00000000000..5db3cd71d5a --- /dev/null +++ b/apps/server/src/modules/management/seed-data/factory/base.factory.ts @@ -0,0 +1,164 @@ +import { ObjectId } from '@mikro-orm/mongodb'; +import type { EntityId } from '@shared/domain/types'; +import { BuildOptions, DeepPartial, Factory, GeneratorFn, HookFn } from 'fishery'; + +/** + * Entity factory based on thoughtbot/fishery + * https://github.com/thoughtbot/fishery + * + * @template T The entity to be built + * @template U The properties interface of the entity + * @template I The transient parameters that your factory supports + * @template C The class of the factory object being created. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export class BaseFactory { + protected readonly propsFactory: Factory; + + constructor(private readonly EntityClass: { new (props: U): T }, propsFactory: Factory) { + this.propsFactory = propsFactory; + } + + /** + * Define a factory + * @template T The entity to be built + * @template U The properties interface of the entity + * @template I The transient parameters that your factory supports + * @template C The class of the factory object being created. + * @param EntityClass The constructor of the entity to be built. + * @param generator Your factory function - see `Factory.define()` in thoughtbot/fishery + * @returns + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + static define>( + this: new (EntityClass: { new (props: U): T }, propsFactory: Factory) => F, + EntityClass: { new (props: U): T }, + generator: GeneratorFn + ): F { + const propsFactory = Factory.define(generator); + const factory = new this(EntityClass, propsFactory); + return factory; + } + + /** + * Build an entity using your factory + * @param params + * @returns an entity + */ + build(params?: DeepPartial, options: BuildOptions = {}): T { + const props = this.propsFactory.build(params, options); + const entity = new this.EntityClass(props); + + return entity; + } + + /** + * Build an entity using your factory and generate a id for it. + * @param params + * @param id + * @returns an entity + */ + buildWithId(params?: DeepPartial, id?: string, options: BuildOptions = {}): T { + const entity = this.build(params, options) as { _id: ObjectId; id: EntityId }; + const generatedId = new ObjectId(id); + const entityWithId = Object.assign(entity, { _id: generatedId, id: generatedId.toHexString() }); + + return entityWithId as T; + } + + /** + * Build a list of entities using your factory + * @param number + * @param params + * @returns a list of entities + */ + buildList(number: number, params?: DeepPartial, options: BuildOptions = {}): T[] { + const list: T[] = []; + for (let i = 0; i < number; i += 1) { + list.push(this.build(params, options)); + } + + return list; + } + + buildListWithId(number: number, params?: DeepPartial, options: BuildOptions = {}): T[] { + const list: T[] = []; + for (let i = 0; i < number; i += 1) { + list.push(this.buildWithId(params, undefined, options)); + } + + return list; + } + + /** + * Extend the factory by adding a function to be called after an object is built. + * @param afterBuildFn - the function to call. It accepts your object of type T. The value this function returns gets returned from "build" + * @returns a new factory + */ + afterBuild(afterBuildFn: HookFn): this { + const newPropsFactory = this.propsFactory.afterBuild(afterBuildFn); + const newFactory = this.clone(newPropsFactory); + + return newFactory; + } + + /** + * Extend the factory by adding default associations to be passed to the factory when "build" is called + * @param associations + * @returns a new factory + */ + associations(associations: Partial): this { + const newPropsFactory = this.propsFactory.associations(associations); + const newFactory = this.clone(newPropsFactory); + + return newFactory; + } + + /** + * Extend the factory by adding default parameters to be passed to the factory when "build" is called + * @param params + * @returns a new factory + */ + params(params: DeepPartial): this { + const newPropsFactory = this.propsFactory.params(params); + const newFactory = this.clone(newPropsFactory); + + return newFactory; + } + + /** + * Extend the factory by adding default transient parameters to be passed to the factory when "build" is called + * @param transient - transient params + * @returns a new factory + */ + transient(transient: Partial): this { + const newPropsFactory = this.propsFactory.transient(transient); + const newFactory = this.clone(newPropsFactory); + + return newFactory; + } + + /** + * Set sequence back to its default value + */ + rewindSequence(): void { + this.propsFactory.rewindSequence(); + } + + protected clone>(this: F, propsFactory: Factory): F { + const copy = new (this.constructor as { + new (EntityClass: { new (props: U): T }, propsOfFactory: Factory): F; + })(this.EntityClass, propsFactory); + + return copy; + } + + /** + * Get the next sequence value + * @returns the next sequence value + */ + protected sequence(): number { + // eslint-disable-next-line @typescript-eslint/dot-notation + return this.propsFactory['sequence'](); + } +} diff --git a/apps/server/src/modules/management/seed-data/factory/county.embeddable.factory.ts b/apps/server/src/modules/management/seed-data/factory/county.embeddable.factory.ts new file mode 100644 index 00000000000..58842c7f61b --- /dev/null +++ b/apps/server/src/modules/management/seed-data/factory/county.embeddable.factory.ts @@ -0,0 +1,17 @@ +import { CountyEmbeddable } from '@shared/domain/entity'; +import { ObjectId } from '@mikro-orm/mongodb'; +import { BaseFactory } from './base.factory'; + +export const countyEmbeddableFactory = BaseFactory.define( + CountyEmbeddable, + ({ sequence }) => { + const county = { + _id: new ObjectId(), + name: `County ${sequence}`, + countyId: sequence, + antaresKey: `antaresKey ${sequence}`, + }; + + return county; + } +); diff --git a/apps/server/src/modules/management/seed-data/factory/federal-state.factory.ts b/apps/server/src/modules/management/seed-data/factory/federal-state.factory.ts new file mode 100644 index 00000000000..cb09d7cfa81 --- /dev/null +++ b/apps/server/src/modules/management/seed-data/factory/federal-state.factory.ts @@ -0,0 +1,18 @@ +import { FederalStateEntity, FederalStateProperties } from '@shared/domain/entity'; +import { BaseFactory } from './base.factory'; +import { countyEmbeddableFactory } from './county.embeddable.factory'; + +export const federalStateFactory = BaseFactory.define( + FederalStateEntity, + () => { + return { + name: 'Hamburg', + abbreviation: 'HH', + logoUrl: + 'https://upload.wikimedia.org/wikipedia/commons/thumb/6/6b/Coat_of_arms_of_Hamburg.svg/1200px-Coat_of_arms_of_Hamburg.svg.png', + counties: countyEmbeddableFactory.buildList(2), + createdAt: new Date(2020, 1), + updatedAt: new Date(2020, 1), + }; + } +); diff --git a/apps/server/src/modules/management/seed-data/factory/role.factory.ts b/apps/server/src/modules/management/seed-data/factory/role.factory.ts new file mode 100644 index 00000000000..ea982edf1e5 --- /dev/null +++ b/apps/server/src/modules/management/seed-data/factory/role.factory.ts @@ -0,0 +1,9 @@ +import { Role, RoleProperties } from '@shared/domain/entity'; +import { RoleName } from '@shared/domain/interface'; +import { BaseFactory } from './base.factory'; + +export const roleFactory = BaseFactory.define(Role, ({ sequence }) => { + return { + name: `role${sequence}` as unknown as RoleName, + }; +}); diff --git a/apps/server/src/modules/management/seed-data/factory/school.entity.factory.ts b/apps/server/src/modules/management/seed-data/factory/school.entity.factory.ts new file mode 100644 index 00000000000..d8fe77a84f4 --- /dev/null +++ b/apps/server/src/modules/management/seed-data/factory/school.entity.factory.ts @@ -0,0 +1,12 @@ +import { SchoolEntity, SchoolProperties } from '@shared/domain/entity'; +import { BaseFactory } from './base.factory'; +import { federalStateFactory } from './federal-state.factory'; +import { schoolYearFactory } from './schoolyear.factory'; + +export const schoolEntityFactory = BaseFactory.define(SchoolEntity, ({ sequence }) => { + return { + name: `school #${sequence}`, + currentYear: schoolYearFactory.build(), + federalState: federalStateFactory.build(), + }; +}); diff --git a/apps/server/src/modules/management/seed-data/factory/schoolyear.factory.ts b/apps/server/src/modules/management/seed-data/factory/schoolyear.factory.ts new file mode 100644 index 00000000000..3c5e77d986a --- /dev/null +++ b/apps/server/src/modules/management/seed-data/factory/schoolyear.factory.ts @@ -0,0 +1,32 @@ +import { SchoolYearEntity, SchoolYearProperties } from '@shared/domain/entity/schoolyear.entity'; +import { BaseFactory } from './base.factory'; + +type SchoolYearTransientParams = { + startYear: number; +}; + +class SchoolYearFactory extends BaseFactory { + public withStartYear(startYear: number): this { + this.rewindSequence(); + return this.transient({ startYear }); + } +} + +export const schoolYearFactory = SchoolYearFactory.define(SchoolYearEntity, ({ transientParams, sequence }) => { + const now = new Date(); + const startYearWithoutSequence = transientParams?.startYear ?? now.getFullYear(); + const sequenceStartingWithZero = sequence - 1; + let correction = 0; + + if (now.getMonth() < 7 && !transientParams?.startYear) { + correction = 1; + } + + const startYear = startYearWithoutSequence + sequenceStartingWithZero - correction; + + const name = `${startYear}/${(startYear + 1).toString().slice(-2)}`; + const startDate = new Date(`${startYear}-08-01`); + const endDate = new Date(`${startYear + 1}-07-31`); + + return { name, startDate, endDate }; +}); diff --git a/apps/server/src/modules/management/seed-data/factory/system.entity.factory.ts b/apps/server/src/modules/management/seed-data/factory/system.entity.factory.ts new file mode 100644 index 00000000000..88e1e9a6b05 --- /dev/null +++ b/apps/server/src/modules/management/seed-data/factory/system.entity.factory.ts @@ -0,0 +1,99 @@ +import { + LdapConfigEntity, + OauthConfigEntity, + OidcConfigEntity, + SystemEntity, + SystemEntityProps, +} from '@modules/system/entity'; +import { SystemProvisioningStrategy } from '@shared/domain/interface/system-provisioning.strategy'; +import { SystemTypeEnum } from '@shared/domain/types'; +import { DeepPartial } from 'fishery'; +import { BaseFactory } from './base.factory'; + +export const systemOauthConfigEntityFactory = BaseFactory.define( + OauthConfigEntity, + () => { + return { + clientId: '12345', + clientSecret: 'mocksecret', + idpHint: 'mock-oauth-idpHint', + tokenEndpoint: 'https://mock.de/mock/auth/public/mockToken', + grantType: 'authorization_code', + redirectUri: 'https://mockhost:3030/api/v3/sso/oauth/', + scope: 'openid uuid', + responseType: 'code', + authEndpoint: 'https://mock.de/auth', + provider: 'mock_type', + logoutEndpoint: 'https://mock.de/logout', + issuer: 'mock_issuer', + jwksEndpoint: 'https://mock.de/jwks', + endSessionEndpoint: 'https://mock.de/logout', + }; + } +); + +export const systemLdapConfigEntityFactory = BaseFactory.define( + LdapConfigEntity, + () => { + return { + url: 'ldaps:mock.de:389', + active: true, + }; + } +); + +export const systemOidcConfigEntityFactory = BaseFactory.define( + OidcConfigEntity, + () => { + return { + clientId: 'mock-client-id', + clientSecret: 'mock-client-secret', + idpHint: 'mock-oidc-idpHint', + defaultScopes: 'openid email userinfo', + authorizationUrl: 'https://mock.tld/auth', + tokenUrl: 'https://mock.tld/token', + userinfoUrl: 'https://mock.tld/userinfo', + logoutUrl: 'https://mock.tld/logout', + }; + } +); + +export class SystemEntityFactory extends BaseFactory { + withOauthConfig(otherParams?: DeepPartial): this { + const params: DeepPartial = { + type: SystemTypeEnum.OAUTH, + oauthConfig: systemOauthConfigEntityFactory.build(otherParams), + }; + + return this.params(params); + } + + withLdapConfig(otherParams?: DeepPartial): this { + const params: DeepPartial = { + type: SystemTypeEnum.LDAP, + ldapConfig: systemLdapConfigEntityFactory.build(otherParams), + }; + + return this.params(params); + } + + withOidcConfig(otherParams?: DeepPartial): this { + const params = { + type: SystemTypeEnum.OIDC, + oidcConfig: systemOidcConfigEntityFactory.build(otherParams), + }; + + return this.params(params); + } +} + +export const systemEntityFactory = SystemEntityFactory.define(SystemEntity, ({ sequence }) => { + return { + type: 'oauth', + url: 'https://mock.de', + alias: `system #${sequence}`, + displayName: `system #${sequence}DisplayName`, + provisioningStrategy: SystemProvisioningStrategy.OIDC, + provisioningUrl: 'https://provisioningurl.de/', + }; +}); diff --git a/apps/server/src/modules/management/seed-data/federalstates.ts b/apps/server/src/modules/management/seed-data/federalstates.ts index c9e69566e6c..ef59e6cf309 100644 --- a/apps/server/src/modules/management/seed-data/federalstates.ts +++ b/apps/server/src/modules/management/seed-data/federalstates.ts @@ -1,7 +1,7 @@ import { ObjectId } from '@mikro-orm/mongodb'; import { CountyEmbeddable, FederalStateProperties } from '@shared/domain/entity/federal-state.entity'; -import { federalStateFactory } from '@testing/factory/federal-state.factory'; import { DeepPartial } from 'fishery'; +import { federalStateFactory } from './factory/federal-state.factory'; type SeedFederalStateProperties = Omit & { id: string; diff --git a/apps/server/src/modules/management/seed-data/roles.ts b/apps/server/src/modules/management/seed-data/roles.ts index 200fc609451..a563fc61f39 100644 --- a/apps/server/src/modules/management/seed-data/roles.ts +++ b/apps/server/src/modules/management/seed-data/roles.ts @@ -2,8 +2,8 @@ import { Role, RoleProperties } from '@shared/domain/entity'; import { Permission, RoleName } from '@shared/domain/interface'; -import { roleFactory } from '@testing/factory/role.factory'; import { DeepPartial } from 'fishery'; +import { roleFactory } from './factory/role.factory'; type SeedRoleProperties = Omit & { id: string; diff --git a/apps/server/src/modules/management/seed-data/schools.ts b/apps/server/src/modules/management/seed-data/schools.ts index 411a80141ec..33c9c627865 100644 --- a/apps/server/src/modules/management/seed-data/schools.ts +++ b/apps/server/src/modules/management/seed-data/schools.ts @@ -5,9 +5,9 @@ import { SystemEntity } from '@modules/system/entity'; import { FederalStateEntity, SchoolProperties, SchoolRoles, SchoolYearEntity } from '@shared/domain/entity'; import { LanguageType } from '@shared/domain/interface'; import { SchoolFeature, SchoolPurpose } from '@shared/domain/types'; -import { federalStateFactory } from '@testing/factory/federal-state.factory'; -import { schoolEntityFactory } from '@testing/factory/school-entity.factory'; import { DeepPartial } from 'fishery'; +import { federalStateFactory } from './factory/federal-state.factory'; +import { schoolEntityFactory } from './factory/school.entity.factory'; import { EFederalState } from './federalstates'; import { SeedSchoolYearEnum } from './schoolyears'; diff --git a/apps/server/src/modules/management/seed-data/schoolyears.ts b/apps/server/src/modules/management/seed-data/schoolyears.ts index 228bccf60c8..3561bcc5157 100644 --- a/apps/server/src/modules/management/seed-data/schoolyears.ts +++ b/apps/server/src/modules/management/seed-data/schoolyears.ts @@ -1,6 +1,6 @@ import { SchoolYearProperties } from '@shared/domain/entity'; -import { schoolYearFactory } from '@testing/factory/schoolyear.factory'; import { DeepPartial } from 'fishery'; +import { schoolYearFactory } from './factory/schoolyear.factory'; type SeedSchoolYearProperties = Pick & { id: string; diff --git a/apps/server/src/modules/management/seed-data/systems.ts b/apps/server/src/modules/management/seed-data/systems.ts index b0831b8a6e9..f50ff4d0837 100644 --- a/apps/server/src/modules/management/seed-data/systems.ts +++ b/apps/server/src/modules/management/seed-data/systems.ts @@ -1,8 +1,8 @@ /* eslint-disable no-template-curly-in-string */ import { SystemEntityProps } from '@modules/system/entity'; -import { systemEntityFactory } from '@modules/system/testing'; import { SystemProvisioningStrategy } from '@shared/domain/interface/system-provisioning.strategy'; import { DeepPartial } from 'fishery'; +import { systemEntityFactory } from './factory/system.entity.factory'; type SystemPartial = DeepPartial & { id?: string; diff --git a/apps/server/src/modules/provisioning/loggable/school-external-tool-created.loggable.spec.ts b/apps/server/src/modules/provisioning/loggable/school-external-tool-created.loggable.spec.ts index 8cfbe2a5677..843b6cdfe52 100644 --- a/apps/server/src/modules/provisioning/loggable/school-external-tool-created.loggable.spec.ts +++ b/apps/server/src/modules/provisioning/loggable/school-external-tool-created.loggable.spec.ts @@ -1,5 +1,5 @@ import { schoolExternalToolFactory } from '@modules/tool/school-external-tool/testing'; -import { mediaUserLicenseFactory } from '@modules/user-license'; +import { mediaUserLicenseFactory } from '@modules/user-license/testing'; import { SchoolExternalToolCreatedLoggable } from './school-external-tool-created.loggable'; describe('SchoolExternalToolCreatedLoggable', () => { diff --git a/apps/server/src/modules/provisioning/strategy/schulconnex/service/schulconnex-license-provisioning.service.spec.ts b/apps/server/src/modules/provisioning/strategy/schulconnex/service/schulconnex-license-provisioning.service.spec.ts index c2d34c0dfd1..d97c238446d 100644 --- a/apps/server/src/modules/provisioning/strategy/schulconnex/service/schulconnex-license-provisioning.service.spec.ts +++ b/apps/server/src/modules/provisioning/strategy/schulconnex/service/schulconnex-license-provisioning.service.spec.ts @@ -1,12 +1,8 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { MediaSource, MediaSourceService } from '@modules/media-source'; import { mediaSourceFactory } from '@modules/media-source/testing'; -import { - MediaUserLicense, - mediaUserLicenseFactory, - MediaUserLicenseService, - UserLicenseType, -} from '@modules/user-license'; +import { MediaUserLicense, MediaUserLicenseService, UserLicenseType } from '@modules/user-license'; +import { mediaUserLicenseFactory } from '@modules/user-license/testing'; import { Test, TestingModule } from '@nestjs/testing'; import { User as UserEntity } from '@shared/domain/entity'; import { userFactory } from '@testing/factory/user.factory'; diff --git a/apps/server/src/modules/provisioning/strategy/schulconnex/service/schulconnex-tool-provisioning.service.spec.ts b/apps/server/src/modules/provisioning/strategy/schulconnex/service/schulconnex-tool-provisioning.service.spec.ts index 437cd0aa0b4..19aba65c1ca 100644 --- a/apps/server/src/modules/provisioning/strategy/schulconnex/service/schulconnex-tool-provisioning.service.spec.ts +++ b/apps/server/src/modules/provisioning/strategy/schulconnex/service/schulconnex-tool-provisioning.service.spec.ts @@ -9,7 +9,8 @@ import { customParameterFactory, externalToolFactory } from '@modules/tool/exter import { SchoolExternalToolService } from '@modules/tool/school-external-tool'; import { SchoolExternalTool } from '@modules/tool/school-external-tool/domain'; import { schoolExternalToolFactory } from '@modules/tool/school-external-tool/testing'; -import { MediaUserLicense, mediaUserLicenseFactory, MediaUserLicenseService } from '@modules/user-license'; +import { MediaUserLicense, MediaUserLicenseService } from '@modules/user-license'; +import { mediaUserLicenseFactory } from '@modules/user-license/testing'; import { Test, TestingModule } from '@nestjs/testing'; import { SchulconnexToolProvisioningService } from './schulconnex-tool-provisioning.service'; diff --git a/apps/server/src/modules/rocketchat-user/domain/index.ts b/apps/server/src/modules/rocketchat-user/domain/index.ts index 441baa69c52..0246dd0f0f9 100644 --- a/apps/server/src/modules/rocketchat-user/domain/index.ts +++ b/apps/server/src/modules/rocketchat-user/domain/index.ts @@ -1,2 +1 @@ export * from './rocket-chat-user.do'; -export * from './testing'; diff --git a/apps/server/src/modules/room/index.ts b/apps/server/src/modules/room/index.ts index 31c66bccfac..7be3bdc7474 100644 --- a/apps/server/src/modules/room/index.ts +++ b/apps/server/src/modules/room/index.ts @@ -2,4 +2,3 @@ export * from './domain'; export { RoomConfig } from './room.config'; export * from './room.module'; export * from './repo/entity'; -export { roomFactory } from './testing'; diff --git a/apps/server/src/modules/synchronization/domain/index.ts b/apps/server/src/modules/synchronization/domain/index.ts index ea6c26e614e..2285cbd6829 100644 --- a/apps/server/src/modules/synchronization/domain/index.ts +++ b/apps/server/src/modules/synchronization/domain/index.ts @@ -1,4 +1,3 @@ export * from './do'; export * from './service'; export * from './types'; -export * from './testing'; diff --git a/apps/server/src/modules/tool/context-external-tool/service/tool-configuration-status.service.spec.ts b/apps/server/src/modules/tool/context-external-tool/service/tool-configuration-status.service.spec.ts index 719f9f2d236..b5337e70546 100644 --- a/apps/server/src/modules/tool/context-external-tool/service/tool-configuration-status.service.spec.ts +++ b/apps/server/src/modules/tool/context-external-tool/service/tool-configuration-status.service.spec.ts @@ -5,7 +5,8 @@ import { MediaSchoolLicense } from '@modules/school-license'; import { MediaSchoolLicenseService } from '@modules/school-license/service/media-school-license.service'; import { mediaSchoolLicenseFactory } from '@modules/school-license/testing'; import { UserService } from '@modules/user'; -import { MediaUserLicense, mediaUserLicenseFactory, MediaUserLicenseService } from '@modules/user-license'; +import { MediaUserLicense, MediaUserLicenseService } from '@modules/user-license'; +import { mediaUserLicenseFactory } from '@modules/user-license/testing'; import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { ValidationError } from '@shared/common/error'; diff --git a/apps/server/src/modules/user-license/index.ts b/apps/server/src/modules/user-license/index.ts index 0bff3f1b0c2..efa3fb6d7d3 100644 --- a/apps/server/src/modules/user-license/index.ts +++ b/apps/server/src/modules/user-license/index.ts @@ -1,5 +1,4 @@ export { AnyUserLicense, MediaUserLicense, MediaUserLicenseProps } from './domain'; export { UserLicenseType } from './enum/user-license-type'; export { MediaUserLicenseService } from './service'; -export { mediaUserLicenseEntityFactory, mediaUserLicenseFactory } from './testing'; export { UserLicenseModule } from './user-license.module'; diff --git a/apps/server/tsconfig.app.json b/apps/server/tsconfig.app.json index 14752374a7c..735c8530c91 100644 --- a/apps/server/tsconfig.app.json +++ b/apps/server/tsconfig.app.json @@ -11,6 +11,7 @@ "node_modules", "dist", "test", - "**/*spec.ts" + "**/*spec.ts", + "**/testing" ] -} \ No newline at end of file +} diff --git a/esbuild/esmodules-bundler.js b/esbuild/esmodules-bundler.js index 5774ae47391..e73aab0df47 100644 --- a/esbuild/esmodules-bundler.js +++ b/esbuild/esmodules-bundler.js @@ -3,7 +3,7 @@ const { dtsPlugin } = require('esbuild-plugin-d.ts'); const { build } = require('esbuild'); const fs = require('fs'); const { resolve } = require('path'); -// add files to be transformed from CommonJs to EsModules in the following list +// add files to be transformed from ESModules to CommonJS in the following list const options = [ { name: 'keycloak-admin-client-lib', diff --git a/package-lock.json b/package-lock.json index f8e1b054cdb..091401d6d47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { + "@aws-sdk/client-s3": "^3.731.1", "@aws-sdk/lib-storage": "^3.731.1", "@feathersjs/adapter-commons": "^5.0.31", "@feathersjs/authentication": "^5.0.31", @@ -79,6 +80,7 @@ "feathers-hooks-common": "^8.2.1", "feathers-swagger": "^3.0.0", "file-type": "^20.0.0", + "fishery": "^2.2.2", "gm": "^1.25.0", "html-entities": "^2.5.2", "i18next": "^24.2.1", @@ -135,7 +137,6 @@ "yaml": "^2.7.0" }, "devDependencies": { - "@aws-sdk/client-s3": "^3.731.1", "@faker-js/faker": "^9.4.0", "@feathersjs/adapter-tests": "^5.0.31", "@golevelup/ts-jest": "0.5.0", @@ -181,7 +182,6 @@ "eslint-plugin-no-only-tests": "^3.3.0", "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-promise": "^7.2.1", - "fishery": "^2.2.2", "freeport": "^1.0.5", "jest": "^29.7.0", "mocha": "^11.0.1", @@ -14750,7 +14750,6 @@ }, "node_modules/fishery": { "version": "2.2.2", - "dev": true, "license": "MIT", "dependencies": { "lodash.mergewith": "^4.6.2" @@ -17903,7 +17902,6 @@ }, "node_modules/lodash.mergewith": { "version": "4.6.2", - "dev": true, "license": "MIT" }, "node_modules/lodash.once": { diff --git a/package.json b/package.json index 4d1c2b53844..fa8733efb27 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "mocha-inspect": "cross-env NODE_ENV=test mocha --inspect --no-timeout --exit", "mocha-metrics": "cross-env NODE_ENV=test mocha \"test/routes/*.metrics.js\" --exclude \"{test,src}/**/*.test.{js,ts}\" --no-timeout --exit", "migration:up": "npm run nest:start:console -- database migration --up", + "migration:up:prod": "npm run nest:start:console:prod -- database migration --up", "migration:down": "npm run nest:start:console -- database migration --down", "migration:pending": "npm run nest:start:console -- database migration --pending", "migration:persisted": "npm run nest:start:console -- database export --collection migrations --override", @@ -59,7 +60,6 @@ "nest:start:dev": "nest start server --watch", "nest:start:debug": "nest start server --debug --watch", "nest:start:prod": "node dist/apps/server/apps/server.app", - "nest:start:management": "nest start management", "nest:start:admin-api-server": "nest start admin-api-server", "nest:start:admin-api-server:debug": "nest start admin-api-server --debug --watch", "nest:start:admin-api-server:prod": "node dist/apps/server/apps/admin-api-server.app", @@ -67,6 +67,7 @@ "nest:start:board-collaboration:dev": "nest start board-collaboration --watch --", "nest:start:board-collaboration:debug": "nest start board-collaboration --debug --watch", "nest:start:board-collaboration:prod": "node dist/apps/server/apps/board-collaboration.app", + "nest:start:management": "nest start management", "nest:start:management:dev": "nest start management --watch", "nest:start:management:debug": "nest start management --debug --watch", "nest:start:management:prod": "node dist/apps/server/apps/management.app", @@ -88,22 +89,28 @@ "nest:start:console": "nest start console --", "nest:start:console:dev": "nest start console --watch --", "nest:start:console:debug": "nest start console --debug --watch --", + "nest:start:console:prod": "node dist/apps/server/apps/server-console.app", "nest:start:deletion-console": "nest start deletion-console --", "nest:start:deletion-console:dev": "nest start deletion-console --watch --", "nest:start:deletion-console:debug": "nest start deletion-console --debug --watch --", + "nest:start:deletion-console:prod": "node dist/apps/server/apps/deletion-console.app", "nest:start:idp-console": "nest start idp-console --", "nest:start:idp-console:dev": "nest start idp-console --watch --", "nest:start:idp-console:debug": "nest start idp-console --debug --watch --", + "nest:start:idp-console:prod": "node dist/apps/server/apps/idp-console.app", "nest:start:common-cartridge": "node dist/apps/server/apps/common-cartridge.app", "nest:start:common-cartridge:dev": "nest start common-cartridge --watch --", "nest:start:common-cartridge:debug": "nest start common-cartridge --debug --watch --", "nest:start:sync": "npm run nest:start:console -- sync run", + "nest:start:sync:prod": "npm run nest:start:console:prod -- sync run", "nest:start:sync:vidis": "npm run nest:start:console -- sync run vidis", - "nest:start:sync:vidis:dev": "npm run nest:start:console --watch -- sync run vidis", - "nest:start:sync:vidis:debug": "npm run nest:start:console --watch --debug -- sync run vidis", + "nest:start:sync:vidis:dev": "npm run nest:start:console:dev -- sync run vidis", + "nest:start:sync:vidis:debug": "npm run nest:start:console:debug -- sync run vidis", + "nest:start:sync:vidis:prod": "npm run nest:start:console:prod -- sync run vidis", "nest:start:sync:media-metadata": "npm run nest:start:console -- sync run media-metadata", - "nest:start:sync:media-metadata:dev": "npm run nest:start:console --watch -- sync run media-metadata", - "nest:start:sync:media-metadata:debug": "npm run nest:start:console --watch --debug -- sync run media-metadata", + "nest:start:sync:media-metadata:dev": "npm run nest:start:console:dev -- sync run media-metadata", + "nest:start:sync:media-metadata:debug": "npm run nest:start:console:debug -- sync run media-metadata", + "nest:start:sync:media-metadata:prod": "npm run nest:start:console:prod -- sync run media-metadata", "nest:test": "npm run nest:test:cov && npm run nest:lint", "nest:test:all": "jest \"^((?!(\\.load)\\.spec\\.ts).)*\"", "nest:test:unit": "jest \"^((?!(\\.api|\\.load)\\.spec\\.ts).)*\\.spec\\.ts$\"", @@ -114,6 +121,7 @@ "nest:test:debug": "jest --runInBand", "nest:lint": "eslint apps --ignore-path .gitignore", "ensureIndexes": "npm run nest:start:console -- database sync-indexes", + "ensureIndexes:prod": "npm run nest:start:console:prod -- database sync-indexes", "schoolExport": "node ./scripts/schoolExport.js", "schoolImport": "node ./scripts/schoolImport.js", "generate-client:authorization": "node ./scripts/generate-client.js -u 'http://localhost:3030/api/v3/docs-json/' -p 'apps/server/src/infra/authorization-client/authorization-api-client' -c 'openapitools-config.json' -f 'operationId:AuthorizationReferenceController_authorizeByReference'", @@ -139,6 +147,7 @@ } }, "dependencies": { + "@aws-sdk/client-s3": "^3.731.1", "@aws-sdk/lib-storage": "^3.731.1", "@feathersjs/adapter-commons": "^5.0.31", "@feathersjs/authentication": "^5.0.31", @@ -210,6 +219,7 @@ "feathers-hooks-common": "^8.2.1", "feathers-swagger": "^3.0.0", "file-type": "^20.0.0", + "fishery": "^2.2.2", "gm": "^1.25.0", "html-entities": "^2.5.2", "i18next": "^24.2.1", @@ -266,7 +276,6 @@ "yaml": "^2.7.0" }, "devDependencies": { - "@aws-sdk/client-s3": "^3.731.1", "@faker-js/faker": "^9.4.0", "@feathersjs/adapter-tests": "^5.0.31", "@golevelup/ts-jest": "0.5.0", @@ -312,7 +321,6 @@ "eslint-plugin-no-only-tests": "^3.3.0", "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-promise": "^7.2.1", - "fishery": "^2.2.2", "freeport": "^1.0.5", "jest": "^29.7.0", "mocha": "^11.0.1", @@ -333,4 +341,4 @@ "tsconfig-paths": "^4.2.0", "typescript": "^5.7.3" } -} \ No newline at end of file +} diff --git a/scripts/ldapSync.sh b/scripts/ldapSync.sh index 80b8893d87c..58540f728a8 100755 --- a/scripts/ldapSync.sh +++ b/scripts/ldapSync.sh @@ -8,7 +8,7 @@ default_api_key="example" value_api_key="${SYNC_API_KEY:-$default_api_key}" # Start server in the background and redirect logs to file -nohup npm run nest:start > server.log 2>&1 & +nohup npm run nest:start:prod > server.log 2>&1 & # Show logs and wait for server to start up echo "Server logs:" diff --git a/sonar-project.properties b/sonar-project.properties index f494c23926e..9523dcb7394 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -3,7 +3,7 @@ sonar.projectKey=hpi-schul-cloud_schulcloud-server sonar.sources=. sonar.tests=. sonar.test.inclusions=**/*.spec.ts -sonar.exclusions=**/*.js,jest.config.ts,globalSetup.ts,globalTeardown.ts,**/*.app.ts,**/seed-data/*.ts,**/migrations/mikro-orm/*.ts,**/etherpad-api-client/**/*.ts,**/authorization-api-client/**/*.ts,**/board-api-client/**/*.ts,**/generated/**/*.ts,**/room-api-client/**/*.ts,**/cards-api-client/**/*.ts,**/lessons-api-client/**/*.ts +sonar.exclusions=**/*.js,jest.config.ts,globalSetup.ts,globalTeardown.ts,**/*.app.ts,**/seed-data/**/*.ts,**/migrations/mikro-orm/*.ts,**/etherpad-api-client/**/*.ts,**/authorization-api-client/**/*.ts,**/board-api-client/**/*.ts,**/generated/**/*.ts,**/room-api-client/**/*.ts,**/cards-api-client/**/*.ts,**/lessons-api-client/**/*.ts sonar.coverage.exclusions=**/board-management.uc.ts,**/*.module.ts,**/*.factory.ts,**/testing/**/*.ts,**/migrations/mikro-orm/*.ts,**/globalSetup.ts,**/globalTeardown.ts,**/etherpad-api-client/**/*.ts,**/authorization-api-client/**/*.ts,**/board-api-client/**/*.ts,**/generated/**/*.ts,**/room-api-client/**/*.ts,apps/server/src/console/console.ts sonar.cpd.exclusions=**/controller/dto/**/*.ts,**/api/dto/**/*.ts,**/testing/factory/*.factory.ts,**/modules/common-cartridge/**/*.ts sonar.javascript.lcov.reportPaths=merged-lcov.info From dcb4444c4c2b229dd2ddb9d99516e616f4db1d88 Mon Sep 17 00:00:00 2001 From: Uwe Ilgenstein Date: Mon, 27 Jan 2025 23:54:12 +0100 Subject: [PATCH 2/5] BC-8794 - Escaping for special Char '&' in room names is not correct (#5466) * prevent encoding of html characters --- .../sanitize-html.transformer.spec.ts | 28 ++++++++++++++++++- .../transformer/sanitize-html.transformer.ts | 10 ++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/apps/server/src/shared/controller/transformer/sanitize-html.transformer.spec.ts b/apps/server/src/shared/controller/transformer/sanitize-html.transformer.spec.ts index 1187f2c3166..c55a6440cb4 100644 --- a/apps/server/src/shared/controller/transformer/sanitize-html.transformer.spec.ts +++ b/apps/server/src/shared/controller/transformer/sanitize-html.transformer.spec.ts @@ -25,7 +25,7 @@ describe('SanitizeHtmlTransformer Decorator', () => { contentRichTextCk5Simple!: string; } - describe('when fully sanitizing an input string', () => { + describe('when sanitizing plain text', () => { it('should remove all html', () => { const plainString = { title: 'html text' }; const instance = plainToClass(WithHtmlDto, plainString); @@ -35,6 +35,32 @@ describe('SanitizeHtmlTransformer Decorator', () => { const instance2 = plainToClass(WithHtmlDto, plainString2); expect(instance2.title2).toEqual('html text'); }); + + it('should not encode html entities', () => { + const plainString = { title: 'X & Y < 5' }; + const instance = plainToClass(WithHtmlDto, plainString); + expect(instance.title).toEqual('X & Y < 5'); + + const plainString2 = { title: 'X & Y > 5' }; + const instance2 = plainToClass(WithHtmlDto, plainString2); + expect(instance2.title).toEqual('X & Y > 5'); + }); + + describe('when the text contains a "<" without the closing ">"', () => { + it('should remove all characters after the "<"', () => { + const plainString = { title: 'X"', () => { + it('should remove all characters between "<" and ">"', () => { + const plainString = { title: 'XB' }; + const instance = plainToClass(WithHtmlDto, plainString); + expect(instance.title).toEqual('XB'); + }); + }); }); describe('when sanitizing rich text ck5 formatting', () => { diff --git a/apps/server/src/shared/controller/transformer/sanitize-html.transformer.ts b/apps/server/src/shared/controller/transformer/sanitize-html.transformer.ts index d455422c93a..d36a78a16d3 100644 --- a/apps/server/src/shared/controller/transformer/sanitize-html.transformer.ts +++ b/apps/server/src/shared/controller/transformer/sanitize-html.transformer.ts @@ -1,17 +1,19 @@ import { NotImplementedException } from '@nestjs/common'; import { InputFormat } from '@shared/domain/types/input-format.types'; import { Transform, TransformFnParams } from 'class-transformer'; -import sanitize, { AllowedAttribute } from 'sanitize-html'; +import { decode } from 'html-entities'; +import sanitize, { AllowedAttribute, IOptions } from 'sanitize-html'; export type IInputFormatsConfig = { allowedTags: string[]; // Note: tag names are not case-sensitive allowedAttributes?: Record; }; -const inputFormatsSanitizeConfig: Record = { +const inputFormatsSanitizeConfig: Record = { PlainText: { allowedTags: [], allowedAttributes: {}, + textFilter: (text: string) => decode(text), }, RichTextCk4: { @@ -104,7 +106,7 @@ const inputFormatsSanitizeConfig: Record = { }, }; -export const getSanitizeHtmlOptions = (inputFormat?: InputFormat): IInputFormatsConfig => { +export const getSanitizeHtmlOptions = (inputFormat?: InputFormat): IOptions => { switch (inputFormat) { case InputFormat.RICH_TEXT_CK5_SIMPLE: return inputFormatsSanitizeConfig.RichTextCk5Simple; @@ -119,7 +121,7 @@ export const getSanitizeHtmlOptions = (inputFormat?: InputFormat): IInputFormats }; export const sanitizeRichText = (value: string, inputFormat?: InputFormat): string => { - const sanitizeHtmlOptions: sanitize.IOptions = getSanitizeHtmlOptions(inputFormat); + const sanitizeHtmlOptions = getSanitizeHtmlOptions(inputFormat); const sanitized = sanitize(value, sanitizeHtmlOptions); From 8b5c7fe7319f5cc23cf1449e0db11019c02c6d49 Mon Sep 17 00:00:00 2001 From: Max Bischof <106820326+bischofmax@users.noreply.github.com> Date: Tue, 28 Jan 2025 08:39:01 +0100 Subject: [PATCH 3/5] BC-1591 - Add password_1 and password_2 params to params that don't get sanitized (#5459) --- src/utils/sanitizeHtml.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/sanitizeHtml.js b/src/utils/sanitizeHtml.js index 4f70a457d8c..65ededf941e 100644 --- a/src/utils/sanitizeHtml.js +++ b/src/utils/sanitizeHtml.js @@ -5,7 +5,7 @@ const maxDeep = 12; // enable html for all current editors const keys = ['content', 'text', 'comment', 'gradeComment', 'description']; const paths = ['lessons', 'news', 'homework', 'submissions', 'topics']; -const saveKeys = ['password', 'secret']; +const saveKeys = ['password', 'password_1', 'password_2', 'secret']; const allowedTags = [ 'h1', 'h2', From e2315fe90411bab31cd96a3eb6e0ba9db0965416 Mon Sep 17 00:00:00 2001 From: MBergCap <111343628+MBergCap@users.noreply.github.com> Date: Tue, 28 Jan 2025 10:56:32 +0100 Subject: [PATCH 4/5] N21-2292 adjust ctl tool seed data (#5462) --- backup/setup/external-tools.json | 55 +++++++------------------------- 1 file changed, 12 insertions(+), 43 deletions(-) diff --git a/backup/setup/external-tools.json b/backup/setup/external-tools.json index 68da38a66ab..9d45882e7a9 100644 --- a/backup/setup/external-tools.json +++ b/backup/setup/external-tools.json @@ -63,7 +63,6 @@ ], "isHidden": false, "openNewTab": true, - "version": 1, "isDeactivated": false, "restrictToContexts": [] }, @@ -84,7 +83,6 @@ "parameters": [], "isHidden": false, "openNewTab": true, - "version": 1, "isDeactivated": false, "restrictToContexts": [ "board-element" @@ -116,8 +114,7 @@ } ], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -154,8 +151,7 @@ } ], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -185,8 +181,7 @@ } ], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -205,8 +200,7 @@ "config_baseUrl": "https://google.com", "parameters": [], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": true, "restrictToContexts": [] }, @@ -246,8 +240,7 @@ } ], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -271,7 +264,6 @@ "$date": "2024-03-19T09:19:57.984Z" }, "url": "https://nextcloud-nbc.dbildungscloud.dev/", - "version": 1, "isDeactivated": false, "restrictToContexts": [] }, @@ -315,8 +307,7 @@ ], "isHidden": false, "isDeactivated": false, - "openNewTab": false, - "version": 3, + "openNewTab": true, "restrictToContexts": [] }, { @@ -355,8 +346,7 @@ } ], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -411,7 +401,6 @@ "isHidden": false, "isDeactivated": false, "openNewTab": true, - "version": 2, "restrictToContexts": [] }, { @@ -465,7 +454,6 @@ "isHidden": false, "isDeactivated": false, "openNewTab": true, - "version": 1, "restrictToContexts": [] }, { @@ -499,7 +487,6 @@ "isHidden": false, "isDeactivated": false, "openNewTab": true, - "version": 1, "restrictToContexts": [] }, { @@ -532,7 +519,6 @@ "isHidden": false, "isDeactivated": false, "openNewTab": true, - "version": 1, "restrictToContexts": [] }, { @@ -600,7 +586,6 @@ "isHidden": false, "isDeactivated": false, "openNewTab": true, - "version": 1, "restrictToContexts": [] }, { @@ -668,7 +653,6 @@ "isHidden": false, "isDeactivated": false, "openNewTab": true, - "version": 1, "restrictToContexts": [] }, { @@ -725,7 +709,6 @@ "isHidden": false, "isDeactivated": false, "openNewTab": true, - "version": 2, "restrictToContexts": [] }, { @@ -782,7 +765,6 @@ "isHidden": false, "isDeactivated": false, "openNewTab": true, - "version": 1, "restrictToContexts": [] }, { @@ -800,8 +782,7 @@ "config_baseUrl": "https://google.com/search", "parameters": [], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -841,8 +822,7 @@ } ], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -882,8 +862,7 @@ } ], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -902,8 +881,7 @@ "config_baseUrl": "https://google.com/search", "parameters": [], "isHidden": false, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -922,8 +900,7 @@ "config_baseUrl": "https://google.com/search", "parameters": [], "isHidden": true, - "openNewTab": false, - "version": 1, + "openNewTab": true, "isDeactivated": false, "restrictToContexts": [] }, @@ -944,7 +921,6 @@ "parameters": [], "isHidden": false, "openNewTab": true, - "version": 1, "isDeactivated": false, "restrictToContexts": [ "course" @@ -967,7 +943,6 @@ "parameters": [], "isHidden": false, "openNewTab": true, - "version": 1, "isDeactivated": false, "restrictToContexts": [ "media-board" @@ -990,7 +965,6 @@ "parameters": [], "isHidden": false, "openNewTab": true, - "version": 1, "isDeactivated": false, "restrictToContexts": [ "course", @@ -1015,7 +989,6 @@ "parameters": [], "isHidden": false, "openNewTab": true, - "version": 1, "isDeactivated": false, "restrictToContexts": [], "isPreferred": true, @@ -1049,7 +1022,6 @@ ], "isHidden": false, "openNewTab": true, - "version": 1, "isDeactivated": false, "restrictToContexts": [], "isPreferred": true, @@ -1072,7 +1044,6 @@ "parameters": [], "isHidden": false, "openNewTab": true, - "version": 1, "isDeactivated": false, "restrictToContexts": [ "course" @@ -1097,7 +1068,6 @@ "parameters": [], "isHidden": false, "openNewTab": true, - "version": 1, "isDeactivated": false, "restrictToContexts": [ "board-element" @@ -1185,7 +1155,6 @@ "isHidden": false, "isDeactivated": false, "openNewTab": true, - "version": 2, "restrictToContexts": [] } ] From 29c33a37f09d6f53505123473646a8a2abce72bb Mon Sep 17 00:00:00 2001 From: Simone Radtke <94017602+SimoneRadtke-Cap@users.noreply.github.com> Date: Tue, 28 Jan 2025 14:14:27 +0100 Subject: [PATCH 5/5] EW-1126 Improving performance for tsp sync (#5469) Co-authored-by: Maximilian Kreuzkam Co-authored-by: mkreuzkam-cap <144103168+mkreuzkam-cap@users.noreply.github.com> --- .../api-tsp-sync-cronjob-configmap.yml.j2 | 4 +- ...-data-sync-batch-finished.loggable.spec.ts | 33 ++++ .../tsp-data-sync-batch-finished.loggable.ts | 24 +++ .../infra/sync/tsp/tsp-sync.strategy.spec.ts | 16 +- .../src/infra/sync/tsp/tsp-sync.strategy.ts | 31 ++- .../domain/services/account-db.service.ts | 22 +-- .../account/repo/micro-orm/account.repo.ts | 2 + .../provisioning/provisioning.module.ts | 2 +- .../service/tsp-provisioning.service.spec.ts | 106 ++++++++++- .../service/tsp-provisioning.service.ts | 180 ++++++++++++------ config/default.schema.json | 2 +- 11 files changed, 328 insertions(+), 94 deletions(-) create mode 100644 apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.spec.ts create mode 100644 apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.ts diff --git a/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob-configmap.yml.j2 b/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob-configmap.yml.j2 index 5892f70c54b..7a82014cae5 100644 --- a/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob-configmap.yml.j2 +++ b/ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob-configmap.yml.j2 @@ -21,7 +21,7 @@ data: TSP_API_CLIENT_TOKEN_LIFETIME_MS: "{{ TSP_API_CLIENT_TOKEN_LIFETIME_MS }}" TSP_SYNC_SCHOOL_LIMIT: "{{ TSP_SYNC_SCHOOL_LIMIT }}" TSP_SYNC_DATA_LIMIT: "{{ TSP_SYNC_DATA_LIMIT }}" - TSP_SYNC_SCHOOL_DAYS_TO_FETCH: "{{ TSP_SYNC_SCHOOL_DAYS_TO_FETCH }}" - TSP_SYNC_DATA_DAYS_TO_FETCH: "{{ TSP_SYNC_DATA_DAYS_TO_FETCH }}" + TSP_SYNC_SCHOOL_DAYS_TO_FETCH: "{{ ((ansible_date_time.date | to_datetime('%Y-%m-%d')) - ("1970-01-01" | to_datetime('%Y-%m-%d'))).days }}" + TSP_SYNC_DATA_DAYS_TO_FETCH: "{{ ((ansible_date_time.date | to_datetime('%Y-%m-%d')) - ("1970-01-01" | to_datetime('%Y-%m-%d'))).days }}" FEATURE_TSP_MIGRATION_ENABLED: "{{ FEATURE_TSP_MIGRATION_ENABLED }}" TSP_SYNC_MIGRATION_LIMIT: "{{ TSP_SYNC_MIGRATION_LIMIT }}" diff --git a/apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.spec.ts b/apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.spec.ts new file mode 100644 index 00000000000..7ea2e57c167 --- /dev/null +++ b/apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.spec.ts @@ -0,0 +1,33 @@ +import { faker } from '@faker-js/faker'; +import { TspDataSyncBatchFinishedLoggable } from './tsp-data-sync-batch-finished.loggable'; + +describe(TspDataSyncBatchFinishedLoggable.name, () => { + describe('getLogMessage is called', () => { + const setup = () => { + const processedCount = faker.number.int(); + const batchSize = faker.number.int(); + const batchCount = faker.number.int(); + const batchIndex = faker.number.int(); + + const expected = { + message: `Processed ${processedCount} of ${batchSize} users in batch ${batchIndex} of ${batchCount}.`, + data: { + processedCount, + batchSize, + batchCount, + batchIndex, + }, + }; + + return { processedCount, batchSize, batchCount, batchIndex, expected }; + }; + + it('should return a log message', () => { + const { processedCount, batchSize, batchCount, batchIndex, expected } = setup(); + + const loggable = new TspDataSyncBatchFinishedLoggable(processedCount, batchSize, batchCount, batchIndex); + + expect(loggable.getLogMessage()).toEqual(expected); + }); + }); +}); diff --git a/apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.ts b/apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.ts new file mode 100644 index 00000000000..cf98a903a7c --- /dev/null +++ b/apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.ts @@ -0,0 +1,24 @@ +import { Loggable, LogMessage } from '@core/logger'; + +export class TspDataSyncBatchFinishedLoggable implements Loggable { + constructor( + private readonly processedCount: number, + private readonly batchSize: number, + private readonly batchCount: number, + private readonly batchIndex: number + ) {} + + public getLogMessage(): LogMessage { + const message: LogMessage = { + message: `Processed ${this.processedCount} of ${this.batchSize} users in batch ${this.batchIndex} of ${this.batchCount}.`, + data: { + processedCount: this.processedCount, + batchSize: this.batchSize, + batchCount: this.batchCount, + batchIndex: this.batchIndex, + }, + }; + + return message; + } +} diff --git a/apps/server/src/infra/sync/tsp/tsp-sync.strategy.spec.ts b/apps/server/src/infra/sync/tsp/tsp-sync.strategy.spec.ts index 5588157f378..32af58d2af3 100644 --- a/apps/server/src/infra/sync/tsp/tsp-sync.strategy.spec.ts +++ b/apps/server/src/infra/sync/tsp/tsp-sync.strategy.spec.ts @@ -10,7 +10,8 @@ import { RobjExportSchule, } from '@infra/tsp-client'; import { Account } from '@modules/account'; -import { ExternalUserDto, OauthDataDto, ProvisioningService, ProvisioningSystemDto } from '@modules/provisioning'; +import { ExternalUserDto, OauthDataDto, ProvisioningSystemDto } from '@modules/provisioning'; +import { TspProvisioningService } from '@modules/provisioning/service/tsp-provisioning.service'; import { School } from '@modules/school'; import { schoolFactory } from '@modules/school/testing'; import { System } from '@modules/system'; @@ -33,7 +34,7 @@ describe(TspSyncStrategy.name, () => { let sut: TspSyncStrategy; let tspSyncService: DeepMocked; let tspFetchService: DeepMocked; - let provisioningService: DeepMocked; + let provisioningService: DeepMocked; let tspOauthDataMapper: DeepMocked; let tspLegacyMigrationService: DeepMocked; let tspSyncMigrationService: DeepMocked; @@ -60,8 +61,8 @@ describe(TspSyncStrategy.name, () => { useValue: createMock>(), }, { - provide: ProvisioningService, - useValue: createMock(), + provide: TspProvisioningService, + useValue: createMock(), }, { provide: TspOauthDataMapper, @@ -81,7 +82,7 @@ describe(TspSyncStrategy.name, () => { sut = module.get(TspSyncStrategy); tspSyncService = module.get(TspSyncService); tspFetchService = module.get(TspFetchService); - provisioningService = module.get(ProvisioningService); + provisioningService = module.get(TspProvisioningService); tspOauthDataMapper = module.get(TspOauthDataMapper); tspLegacyMigrationService = module.get(TspLegacyMigrationService); tspSyncMigrationService = module.get(TspSyncMigrationService); @@ -133,6 +134,7 @@ describe(TspSyncStrategy.name, () => { totalUsers: number; totalAccounts: number; }; + processBatchSize?: number; }) => { tspFetchService.fetchTspSchools.mockResolvedValueOnce(params.fetchedSchools ?? []); tspFetchService.fetchTspClasses.mockResolvedValueOnce(params.fetchedClasses ?? []); @@ -156,6 +158,8 @@ describe(TspSyncStrategy.name, () => { totalUsers: faker.number.int(), } ); + + provisioningService.provisionBatch.mockResolvedValueOnce(params.processBatchSize ?? 0); }; describe('sync', () => { @@ -246,7 +250,7 @@ describe(TspSyncStrategy.name, () => { await sut.sync(); - expect(provisioningService.provisionData).toHaveBeenCalledWith(oauthDataDto); + expect(provisioningService.provisionBatch).toHaveBeenCalledWith([oauthDataDto]); }); describe('when feature tsp migration is enabled', () => { diff --git a/apps/server/src/infra/sync/tsp/tsp-sync.strategy.ts b/apps/server/src/infra/sync/tsp/tsp-sync.strategy.ts index 99b545eda05..8c8f926f3b6 100644 --- a/apps/server/src/infra/sync/tsp/tsp-sync.strategy.ts +++ b/apps/server/src/infra/sync/tsp/tsp-sync.strategy.ts @@ -1,5 +1,6 @@ import { Logger } from '@core/logger'; -import { ProvisioningService } from '@modules/provisioning'; +import { OauthDataDto } from '@modules/provisioning'; +import { TspProvisioningService } from '@modules/provisioning/service/tsp-provisioning.service'; import { School } from '@modules/school'; import { System } from '@modules/system'; import { Injectable } from '@nestjs/common'; @@ -8,6 +9,7 @@ import pLimit from 'p-limit'; import { SyncStrategy } from '../strategy/sync-strategy'; import { SyncStrategyTarget } from '../sync-strategy.types'; import { TspDataFetchedLoggable } from './loggable/tsp-data-fetched.loggable'; +import { TspDataSyncBatchFinishedLoggable } from './loggable/tsp-data-sync-batch-finished.loggable'; import { TspSchoolsFetchedLoggable } from './loggable/tsp-schools-fetched.loggable'; import { TspSchoolsSyncedLoggable } from './loggable/tsp-schools-synced.loggable'; import { TspSchulnummerMissingLoggable } from './loggable/tsp-schulnummer-missing.loggable'; @@ -30,7 +32,7 @@ export class TspSyncStrategy extends SyncStrategy { private readonly tspOauthDataMapper: TspOauthDataMapper, private readonly tspLegacyMigrationService: TspLegacyMigrationService, private readonly configService: ConfigService, - private readonly provisioningService: ProvisioningService, + private readonly provisioningService: TspProvisioningService, private readonly tspSyncMigrationService: TspSyncMigrationService ) { super(); @@ -118,16 +120,29 @@ export class TspSyncStrategy extends SyncStrategy { this.logger.info(new TspSyncingUsersLoggable(oauthDataDtos.length)); - const dataLimit = this.configService.getOrThrow('TSP_SYNC_DATA_LIMIT'); - const dataLimitFn = pLimit(dataLimit); + const batchSize = this.configService.getOrThrow('TSP_SYNC_DATA_LIMIT'); - const dataPromises = oauthDataDtos.map((oauthDataDto) => - dataLimitFn(() => this.provisioningService.provisionData(oauthDataDto)) + const batchCount = Math.ceil(oauthDataDtos.length / batchSize); + const batches: OauthDataDto[][] = []; + for (let i = 0; i < batchCount; i += 1) { + const start = i * batchSize; + const end = Math.min((i + 1) * batchSize, oauthDataDtos.length); + batches.push(oauthDataDtos.slice(start, end)); + } + + const batchLimit = pLimit(1); + const batchPromises = batches.map((batch, index) => + batchLimit(async () => { + const processed = await this.provisioningService.provisionBatch(batch); + this.logger.info(new TspDataSyncBatchFinishedLoggable(processed, batchSize, batchCount, index)); + return processed; + }) ); - const results = await Promise.allSettled(dataPromises); + const results = await Promise.all(batchPromises); + const total = results.reduce((previousValue, currentValue) => previousValue + currentValue, 0); - this.logger.info(new TspSyncedUsersLoggable(results.length)); + this.logger.info(new TspSyncedUsersLoggable(total)); } private async runMigration(system: System): Promise { diff --git a/apps/server/src/modules/account/domain/services/account-db.service.ts b/apps/server/src/modules/account/domain/services/account-db.service.ts index bc50003fc3a..f3fbde48b85 100644 --- a/apps/server/src/modules/account/domain/services/account-db.service.ts +++ b/apps/server/src/modules/account/domain/services/account-db.service.ts @@ -22,7 +22,7 @@ export class AccountServiceDb extends AbstractAccountService { } public async findById(id: EntityId): Promise { - const internalId = await this.getInternalId(id); + const internalId = await this.convertExternalToInternalId(id); return this.accountRepo.findById(internalId); } @@ -52,7 +52,7 @@ export class AccountServiceDb extends AbstractAccountService { public async save(accountSave: AccountSave): Promise { let account: Account; if (accountSave.id) { - const internalId = await this.getInternalId(accountSave.id); + const internalId = await this.convertExternalToInternalId(accountSave.id); account = await this.accountRepo.findById(internalId); } else { @@ -67,7 +67,7 @@ export class AccountServiceDb extends AbstractAccountService { accountSaves.map(async (accountSave) => { let account: Account; if (accountSave.id) { - const internalId = await this.getInternalId(accountSave.id); + const internalId = await this.convertExternalToInternalId(accountSave.id); account = await this.accountRepo.findById(internalId); } else { @@ -84,7 +84,7 @@ export class AccountServiceDb extends AbstractAccountService { } public async updateUsername(accountId: EntityId, username: string): Promise { - const internalId = await this.getInternalId(accountId); + const internalId = await this.convertExternalToInternalId(accountId); const account = await this.accountRepo.findById(internalId); account.username = username; await this.accountRepo.save(account); @@ -92,7 +92,7 @@ export class AccountServiceDb extends AbstractAccountService { } public async updateLastLogin(accountId: EntityId, lastLogin: Date): Promise { - const internalId = await this.getInternalId(accountId); + const internalId = await this.convertExternalToInternalId(accountId); const account = await this.accountRepo.findById(internalId); account.lastLogin = lastLogin; @@ -102,7 +102,7 @@ export class AccountServiceDb extends AbstractAccountService { } public async updateLastTriedFailedLogin(accountId: EntityId, lastTriedFailedLogin: Date): Promise { - const internalId = await this.getInternalId(accountId); + const internalId = await this.convertExternalToInternalId(accountId); const account = await this.accountRepo.findById(internalId); account.lasttriedFailedLogin = lastTriedFailedLogin; @@ -112,7 +112,7 @@ export class AccountServiceDb extends AbstractAccountService { } public async updatePassword(accountId: EntityId, password: string): Promise { - const internalId = await this.getInternalId(accountId); + const internalId = await this.convertExternalToInternalId(accountId); const account = await this.accountRepo.findById(internalId); account.password = await this.encryptPassword(password); @@ -122,7 +122,7 @@ export class AccountServiceDb extends AbstractAccountService { } public async delete(id: EntityId): Promise { - const internalId = await this.getInternalId(id); + const internalId = await this.convertExternalToInternalId(id); return this.accountRepo.deleteById(internalId); } @@ -151,12 +151,6 @@ export class AccountServiceDb extends AbstractAccountService { return passwordCompare; } - private async getInternalId(id: EntityId | ObjectId): Promise { - const internalId = await this.convertExternalToInternalId(id); - - return internalId; - } - private async convertExternalToInternalId(id: EntityId | ObjectId): Promise { if (id instanceof ObjectId || ObjectId.isValid(id)) { return new ObjectId(id); diff --git a/apps/server/src/modules/account/repo/micro-orm/account.repo.ts b/apps/server/src/modules/account/repo/micro-orm/account.repo.ts index e1cefcc6d8d..662d79ec68d 100644 --- a/apps/server/src/modules/account/repo/micro-orm/account.repo.ts +++ b/apps/server/src/modules/account/repo/micro-orm/account.repo.ts @@ -43,6 +43,8 @@ export class AccountRepo extends BaseDomainObjectRepo { } public async saveAll(accounts: Account[]): Promise { + // Testing showed that there are significant performance gains after clearing the entity manager here + this.em.clear(); const savedAccounts = await Promise.all(accounts.map((account) => this.saveWithoutFlush(account))); await this.flush(); diff --git a/apps/server/src/modules/provisioning/provisioning.module.ts b/apps/server/src/modules/provisioning/provisioning.module.ts index 0ce21c72e53..87dce761206 100644 --- a/apps/server/src/modules/provisioning/provisioning.module.ts +++ b/apps/server/src/modules/provisioning/provisioning.module.ts @@ -65,6 +65,6 @@ import { TspProvisioningStrategy } from './strategy/tsp'; TspProvisioningStrategy, TspProvisioningService, ], - exports: [ProvisioningService], + exports: [ProvisioningService, TspProvisioningService], }) export class ProvisioningModule {} diff --git a/apps/server/src/modules/provisioning/service/tsp-provisioning.service.spec.ts b/apps/server/src/modules/provisioning/service/tsp-provisioning.service.spec.ts index 1075f4cd89f..07f6913a06a 100644 --- a/apps/server/src/modules/provisioning/service/tsp-provisioning.service.spec.ts +++ b/apps/server/src/modules/provisioning/service/tsp-provisioning.service.spec.ts @@ -1,6 +1,7 @@ import { faker } from '@faker-js/faker'; import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { AccountService } from '@modules/account'; +import { accountDoFactory } from '@modules/account/testing'; import { ClassService } from '@modules/class'; import { classFactory } from '@modules/class/domain/testing'; import { RoleService } from '@modules/role'; @@ -12,7 +13,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { NotFoundLoggableException } from '@shared/common/loggable-exception'; import { RoleName } from '@shared/domain/interface'; import { SystemProvisioningStrategy } from '@shared/domain/interface/system-provisioning.strategy'; - import { roleFactory } from '@testing/factory/role.factory'; import { userDoFactory } from '@testing/factory/user.do.factory'; import { ExternalClassDto, ExternalSchoolDto, ExternalUserDto, OauthDataDto, ProvisioningSystemDto } from '../dto'; @@ -90,12 +90,111 @@ describe('TspProvisioningService', () => { beforeEach(() => { jest.resetAllMocks(); + jest.restoreAllMocks(); + jest.clearAllMocks(); }); it('should be defined', () => { expect(sut).toBeDefined(); }); + describe('provisionBatch', () => { + describe('when batch is provisioned', () => { + const setup = () => { + const system = setupExternalSystem(); + const externalUser = setupExternalUser(); + const oauthDataDtos = [ + new OauthDataDto({ + system, + externalUser, + }), + ]; + + schoolServiceMock.getSchools.mockResolvedValueOnce(schoolFactory.buildList(1)); + userServiceMock.findByExternalId.mockResolvedValueOnce(userDoFactory.build()); + roleServiceMock.findByNames.mockResolvedValueOnce(roleDtoFactory.buildList(1)); + userServiceMock.saveAll.mockResolvedValueOnce(userDoFactory.buildListWithId(1)); + accountServiceMock.findByUserId.mockResolvedValueOnce(accountDoFactory.build()); + accountServiceMock.saveAll.mockResolvedValueOnce(accountDoFactory.buildList(1)); + + return { oauthDataDtos }; + }; + + it('should return number of provisioned users and call services with available data', async () => { + const { oauthDataDtos } = setup(); + + const result = await sut.provisionBatch(oauthDataDtos); + + expect(result).toBe(1); + expect(schoolServiceMock.getSchools).toHaveBeenCalledTimes(1); + expect(userServiceMock.findByExternalId).toHaveBeenCalledTimes(1); + expect(roleServiceMock.findByNames).toHaveBeenCalledTimes(1); + expect(userServiceMock.saveAll).toHaveBeenCalledTimes(1); + expect(accountServiceMock.findByUserId).toHaveBeenCalledTimes(1); + expect(accountServiceMock.saveAll).toHaveBeenCalledTimes(1); + expect(classServiceMock.findClassWithSchoolIdAndExternalId).toHaveBeenCalledTimes(0); + }); + }); + + describe('when school is not found for an external id', () => { + const setup = () => { + const system = setupExternalSystem(); + const externalUser = setupExternalUser(); + const oauthDataDtos = [ + new OauthDataDto({ + system, + externalUser, + }), + ]; + + schoolServiceMock.getSchools.mockResolvedValueOnce([]); + + return { oauthDataDtos }; + }; + + it('should throw NotFoundLoggableException', async () => { + const { oauthDataDtos } = setup(); + + await expect(sut.provisionBatch(oauthDataDtos)).rejects.toThrow(NotFoundLoggableException); + }); + }); + + describe('when user has classes', () => { + const setup = () => { + const system = setupExternalSystem(); + const externalUser = setupExternalUser(); + const externalClasses = [setupExternalClass()]; + const oauthDataDtos = [ + new OauthDataDto({ + system, + externalUser, + externalClasses, + }), + ]; + + schoolServiceMock.getSchools.mockResolvedValueOnce(schoolFactory.buildList(1)); + userServiceMock.findByExternalId.mockResolvedValueOnce(userDoFactory.build()); + roleServiceMock.findByNames.mockResolvedValueOnce(roleDtoFactory.buildList(1)); + userServiceMock.saveAll.mockResolvedValueOnce([ + userDoFactory.buildWithId({ + externalId: externalUser.externalId, + }), + ]); + accountServiceMock.findByUserId.mockResolvedValueOnce(accountDoFactory.build()); + accountServiceMock.saveAll.mockResolvedValueOnce(accountDoFactory.buildList(1)); + + return { oauthDataDtos }; + }; + + it('should provision the classes', async () => { + const { oauthDataDtos } = setup(); + + await expect(sut.provisionBatch(oauthDataDtos)).resolves.not.toThrow(); + expect(classServiceMock.findClassWithSchoolIdAndExternalId).toHaveBeenCalledTimes(1); + }); + }); + }); + describe('findSchoolOrFail', () => { describe('when school is found', () => { const setup = () => { @@ -232,6 +331,7 @@ describe('TspProvisioningService', () => { userServiceMock.findByExternalId.mockResolvedValue(user); userServiceMock.save.mockResolvedValue(user); schoolServiceMock.getSchools.mockResolvedValue([school]); + roleServiceMock.findByNames.mockResolvedValueOnce(roleDtoFactory.buildList(1)); return { data, school }; }; @@ -274,7 +374,7 @@ describe('TspProvisioningService', () => { await sut.provisionUser(data, school); expect(userServiceMock.save).toHaveBeenCalledTimes(1); - expect(accountServiceMock.saveWithValidation).toHaveBeenCalledTimes(1); + expect(accountServiceMock.save).toHaveBeenCalledTimes(1); }); }); @@ -339,7 +439,7 @@ describe('TspProvisioningService', () => { await sut.provisionUser(data, school); expect(userServiceMock.save).toHaveBeenCalledTimes(1); - expect(accountServiceMock.saveWithValidation).toHaveBeenCalledTimes(1); + expect(accountServiceMock.save).toHaveBeenCalledTimes(1); }); }); diff --git a/apps/server/src/modules/provisioning/service/tsp-provisioning.service.ts b/apps/server/src/modules/provisioning/service/tsp-provisioning.service.ts index d5f8f97fad8..e585e72587b 100644 --- a/apps/server/src/modules/provisioning/service/tsp-provisioning.service.ts +++ b/apps/server/src/modules/provisioning/service/tsp-provisioning.service.ts @@ -1,4 +1,4 @@ -import { AccountSave, AccountService } from '@modules/account'; +import { Account, AccountSave, AccountService } from '@modules/account'; import { ClassFactory, ClassService, ClassSourceOptions } from '@modules/class'; import { RoleService } from '@modules/role'; import { School, SchoolService } from '@modules/school'; @@ -28,6 +28,70 @@ export class TspProvisioningService { private readonly accountService: AccountService ) {} + public async provisionBatch(oauthDataDtos: OauthDataDto[]): Promise { + const schoolArrays = await Promise.all( + oauthDataDtos.map((oauthData, index) => + this.schoolService.getSchools({ + systemId: oauthDataDtos[index].system.systemId, + externalId: oauthData.externalSchool?.externalId, + }) + ) + ); + + const schoolIds = schoolArrays.map((schools, index) => { + if (schools.length !== 1) { + throw new NotFoundLoggableException(School.name, { + systemId: oauthDataDtos[index].system.systemId, + externalId: oauthDataDtos[index].externalSchool?.externalId ?? '', + }); + } + + return schools[0].id; + }); + + const users = await Promise.all( + oauthDataDtos.map((oauth) => + this.userService.findByExternalId(oauth.externalUser.externalId, oauth.system.systemId) + ) + ); + + const roleRefs = await Promise.all( + oauthDataDtos.map((oauthDataDto) => this.getRoleReferencesForUser(oauthDataDto.externalUser)) + ); + + const updatedUsers = users.map((user, index) => { + const oauthDataDto = oauthDataDtos[index]; + return this.createOrUpdateUser(oauthDataDto.externalUser, roleRefs[index], schoolIds[index], user); + }); + + const savedUsers = await this.userService.saveAll(updatedUsers.filter((user) => user !== undefined)); + + await Promise.allSettled( + oauthDataDtos.map((oauth, index) => { + const userForClasses = savedUsers.find((user) => user.externalId === oauth.externalUser.externalId); + if (!userForClasses) { + return Promise.reject(); + } + + const promise = this.provisionClasses(schoolArrays[index][0], oauth.externalClasses ?? [], userForClasses); + return promise; + }) + ); + + const savedUserIds = savedUsers.map((savedUser) => savedUser.id); + const foundAccounts = await Promise.all( + savedUserIds.map((userId) => this.accountService.findByUserId(userId ?? '')) + ); + + const accountsToSave = foundAccounts.map((account, index) => + this.createOrUpdateAccount(oauthDataDtos[index].system.systemId, savedUsers[index], account) + ); + + const savedAccounts = await this.accountService.saveAll(accountsToSave); + + return savedAccounts.length; + } + public async findSchoolOrFail(system: ProvisioningSystemDto, school: ExternalSchoolDto): Promise { const schools = await this.schoolService.getSchools({ systemId: system.systemId, @@ -92,87 +156,85 @@ export class TspProvisioningService { const existingUser = await this.userService.findByExternalId(data.externalUser.externalId, data.system.systemId); const roleRefs = await this.getRoleReferencesForUser(data.externalUser); - let user: UserDO; - if (existingUser) { - user = await this.updateUser(existingUser, data.externalUser, roleRefs, school.id); - } else { - user = await this.createUser(data.externalUser, roleRefs, school.id); + const user = this.createOrUpdateUser(data.externalUser, roleRefs, school.id, existingUser); + if (!user) { + throw new BadDataLoggableException(`Couldn't process user`, { + externalId: data.externalUser.externalId, + }); } + const savedUser = await this.userService.save(user); - await this.createOrUpdateAccount(data.system.systemId, user); + const account = await this.accountService.findByUserId(savedUser.id ?? ''); + const updated = this.createOrUpdateAccount(data.system.systemId, savedUser, account); + await this.accountService.save(updated); return user; } - private async updateUser( - existingUser: UserDO, + private createOrUpdateUser( externalUser: ExternalUserDto, roleRefs: RoleReference[], - schoolId: string - ): Promise { - existingUser.roles = roleRefs; - existingUser.schoolId = schoolId; - existingUser.firstName = externalUser.firstName || existingUser.firstName; - existingUser.lastName = externalUser.lastName || existingUser.lastName; - existingUser.email = externalUser.email || existingUser.email; - existingUser.birthday = externalUser.birthday; - const updatedUser = await this.userService.save(existingUser); - - return updatedUser; - } + schoolId: string, + existingUser?: UserDO | null + ): UserDO | undefined { + if (!existingUser) { + if (!externalUser.firstName || !externalUser.lastName) { + return undefined; + } - private async createUser( - externalUser: ExternalUserDto, - roleRefs: RoleReference[], - schoolId: string - ): Promise { - if (!externalUser.firstName || !externalUser.lastName) { - throw new BadDataLoggableException('User firstname or lastname is missing. TspUid:', { + const newUser = new UserDO({ + roles: roleRefs, + schoolId, + firstName: externalUser.firstName, + lastName: externalUser.lastName, + email: this.createTspEmail(externalUser.externalId), + birthday: externalUser.birthday, externalId: externalUser.externalId, + secondarySchools: [], }); - } - const newUser = new UserDO({ - roles: roleRefs, - schoolId, - firstName: externalUser.firstName, - lastName: externalUser.lastName, - email: this.createTspEmail(externalUser.externalId), - birthday: externalUser.birthday, - externalId: externalUser.externalId, - secondarySchools: [], - }); + this.createTspConsent(newUser); - this.createTspConsent(newUser); + return newUser; + } - const savedUser = await this.userService.save(newUser); + existingUser.roles = roleRefs; + existingUser.schoolId = schoolId; + existingUser.firstName = externalUser.firstName || existingUser.firstName; + existingUser.lastName = externalUser.lastName || existingUser.lastName; + existingUser.email = externalUser.email || existingUser.email; + existingUser.birthday = externalUser.birthday; - return savedUser; + return existingUser; } - private async createOrUpdateAccount(systemId: string, user: UserDO): Promise { - if (!user.id) + private createOrUpdateAccount(systemId: string, user: UserDO, account: Account | null): AccountSave { + if (!user.id) { throw new BadDataLoggableException('user ID is missing', { externalId: user.externalId, }); - - const account = await this.accountService.findByUserId(user.id); + } if (account) { - // Updates account with new systemId and username - await account.update(new AccountSave({ userId: user.id, systemId, username: user.email, activated: true })); - await this.accountService.save(account); - } else { - // Creates new account for user - await this.accountService.saveWithValidation( - new AccountSave({ - userId: user.id, - username: user.email, - systemId, - activated: true, - }) - ); + const updated = new AccountSave({ + userId: user.id, + username: user.email, + activated: true, + systemId: account.systemId, + id: account.id, + }); + + return updated; } + + const newAccount = new AccountSave({ + userId: user.id, + username: user.email, + systemId, + activated: true, + }); + + return newAccount; } private async getRoleReferencesForUser(externalUser: ExternalUserDto): Promise { diff --git a/config/default.schema.json b/config/default.schema.json index f8b10e905e0..d30fd627706 100644 --- a/config/default.schema.json +++ b/config/default.schema.json @@ -215,7 +215,7 @@ }, "TSP_SYNC_MIGRATION_LIMIT": { "type": "number", - "default": "500", + "default": "50", "description": "The amount of users the sync migrates at once." }, "FEATURE_TSP_MIGRATION_ENABLED": {