Merge branch 'main' into BC-7561-batch-delete-fix

Dockerfile

@@ -1,27 +1,46 @@
-FROM docker.io/node:22 AS git
+FROM docker.io/node:22-alpine AS builder
 
-RUN mkdir /app && chown -R node:node /app
 WORKDIR /app
-COPY .git .
-RUN git config --global --add safe.directory /app && echo "{\"sha\": \"$(git rev-parse HEAD)\", \"version\": \"$(git describe --tags --abbrev=0)\", \"commitDate\": \"$(git log -1 --format=%cd --date=format:'%Y-%m-%dT%H:%M:%SZ')\", \"birthdate\": \"$(date +%Y-%m-%dT%H:%M:%SZ)\"}" > /app/serverversion
+
+COPY package.json package-lock.json tsconfig.json tsconfig.build.json nest-cli.json ./
+COPY apps apps
+COPY config config
+COPY esbuild esbuild
+COPY src src
+
+RUN apk add --no-cache git
+COPY .git ./.git
+RUN git config --global --add safe.directory /app  \
+    && echo "{\"sha\": \"$(git rev-parse HEAD)\", \"version\": \"$(git describe --tags --abbrev=0)\", \"commitDate\": \"$(git log -1 --format=%cd --date=format:'%Y-%m-%dT%H:%M:%SZ')\", \"birthdate\": \"$(date +%Y-%m-%dT%H:%M:%SZ)\"}" > apps/server/static-assets/serverversion
+
+RUN npm ci && npm run build
+
 
 FROM docker.io/node:22-alpine
+
 ENV TZ=Europe/Berlin
-RUN apk add --no-cache git make python3
-# to run ldap sync as script curl is needed
-RUN apk add --no-cache curl
+RUN apk add --no-cache python3 curl
+
 WORKDIR /schulcloud-server
-COPY tsconfig.json tsconfig.build.json package.json package-lock.json .eslintrc.js .eslintignore nest-cli.json ./
-COPY esbuild ./esbuild
-RUN npm ci && npm cache clean --force
-COPY config /schulcloud-server/config
-COPY backup /schulcloud-server/backup
-COPY src /schulcloud-server/src
-COPY apps /schulcloud-server/apps
-COPY --from=git /app/serverversion /schulcloud-server/apps/server/static-assets
-COPY scripts/ldapSync.sh /schulcloud-server/scripts/
-RUN npm run build
+
+COPY package.json package-lock.json ./
+COPY backup backup
+COPY config config
+COPY scripts/ldapSync.sh scripts/
+COPY src src
+
+COPY --from=builder /app/dist dist
+
+# The postinstall script must be disabled, because esbuild is a dev dependency and not installed here.
+RUN npm pkg delete scripts.postinstall \
+    && npm ci --omit=dev \
+    && npm cache clean --force
+
+# The modules transpiled by esbuild need to be copied manually from the build stage.
+COPY --from=builder /app/node_modules/@keycloak/keycloak-admin-client-cjs node_modules/@keycloak/keycloak-admin-client-cjs
+COPY --from=builder /app/node_modules/file-type-cjs node_modules/file-type-cjs
 
 ENV NODE_ENV=production
 ENV NO_COLOR="true"
-CMD npm run start
+
+CMD npm run nest:start:prod

ansible/roles/h5p-library-management/templates/api-h5p-library-management-cronjob.yml.j2

@@ -57,7 +57,7 @@ spec:
               subPath: h5p-libraries.yaml
               readOnly: true
             command: ['/bin/sh', '-c']
-            args: ['npm run nest:start:h5p:library-management']
+            args: ['npm run nest:start:h5p:library-management:prod']
             resources:
               limits:
                 cpu: {{ API_H5P_LIBRARY_MANAGEMENT_CPU_LIMITS|default("2000m", true) }}

ansible/roles/media-licenses/templates/vidis-sync-cronjob.yml.j2

@@ -40,7 +40,7 @@ spec:
             - configMapRef:
                 name: vidis-sync-cronjob-configmap
             command: ['/bin/sh','-c']
-            args: ['npm run nest:start:sync:vidis']
+            args: ['npm run nest:start:sync:vidis:prod']
             resources:
               limits:
                 cpu: {{ MEDIA_ACTIVATION_CPU_LIMITS|default("2000m", true) }}

ansible/roles/media-metadata-sync/templates/media-metadata-sync-cronjob.yml.j2

@@ -40,7 +40,7 @@ spec:
             - configMapRef:
                 name: media-metadata-sync-cronjob-configmap
             command: ['/bin/sh','-c']
-            args: ['npm run nest:start:sync:media-metadata']
+            args: ['npm run nest:start:sync:media-metadata:prod']
             resources:
               limits:
                 cpu: {{ MEDIA_METADATA_SYNC_CPU_LIMITS|default("2000m", true) }}

ansible/roles/moin-schule-sync/templates/moin-schule-users-deletion-queueing-cronjob.yml.j2

@@ -30,7 +30,7 @@ spec:
             - secretRef:
                 name: api-files-secret
             command: ['/bin/sh','-c']
-            args: ['npm run nest:start:deletion-console -- queue unsynced --systemId $SYSTEM_ID']
+            args: ['npm run nest:start:deletion-console:prod -- queue unsynced --systemId $SYSTEM_ID']
             resources:
               limits:
                 cpu: {{ API_CPU_LIMITS|default("2000m", true) }}

ansible/roles/moin-schule-sync/templates/moin-schule-users-sync-cronjob.yml.j2

@@ -30,7 +30,7 @@ spec:
             - secretRef:
                 name: api-files-secret
             command: ['/bin/sh','-c']
-            args: ['npm run nest:start:idp-console -- sync users --systemType moin.schule --systemId $SYSTEM_ID']
+            args: ['npm run nest:start:idp-console:prod -- sync users --systemType moin.schule --systemId $SYSTEM_ID']
             resources:
               limits:
                 cpu: {{ API_CPU_LIMITS|default("2000m", true) }}

ansible/roles/schulcloud-server-core/templates/api-delete-s3-files-cronjob.yml.j2

@@ -36,7 +36,7 @@ spec:
             - secretRef:
                 name: api-files-secret
             command: ['/bin/sh', '-c']
-            args: ['npm run nest:start:console -- files cleanup-job 7']
+            args: ['npm run nest:start:console:prod -- files cleanup-job 7']
             resources:
               limits:
                 cpu: {{ API_CPU_LIMITS|default("2000m", true) }}

ansible/roles/schulcloud-server-core/templates/data-deletion-trigger-cronjob.yml.j2

@@ -45,7 +45,7 @@ spec:
             - secretRef:
                 name: api-files-secret
             command: ['/bin/sh', '-c']
-            args: ['npm run nest:start:deletion-console -- execution trigger']
+            args: ['npm run nest:start:deletion-console:prod -- execution trigger']
             resources:
               limits:
                 cpu: {{ API_CPU_LIMITS|default("2000m", true) }}

ansible/roles/schulcloud-server-core/templates/deployment.yml.j2

@@ -65,6 +65,7 @@ spec:
             name: api-secret
         - secretRef:
             name: api-files-secret
+        command: ['npm', 'run', 'nest:start:prod']
         readinessProbe:
           httpGet:
             path: /internal/health

ansible/roles/schulcloud-server-core/templates/migration-job.yml.j2

@@ -28,7 +28,7 @@ spec:
         - secretRef:
             name: api-files-secret
         command: ['/bin/sh','-c']
-        args: ['npm run ensureIndexes && npm run migration:up']
+        args: ['npm run ensureIndexes:prod && npm run migration:up:prod']
         resources:
           limits:
             cpu: {{ API_CPU_LIMITS|default("2000m", true) }}

ansible/roles/schulcloud-server-ldapsync/templates/api-ldap-worker-deployment.yml.j2

@@ -61,6 +61,7 @@ spec:
             name: api-secret
         - secretRef:
             name: api-files-secret
+        command: ['npm', 'run', 'nest:start:prod']
         readinessProbe:
           httpGet:
             path: /serverversion

ansible/roles/schulcloud-server-migration-system/templates/deployment.yml.j2

@@ -65,6 +65,7 @@ spec:
             name: api-secret
         - secretRef:
             name: api-files-secret
+        command: ['npm', 'run', 'nest:start:prod']
         readinessProbe:
           httpGet:
             path: /internal/health

ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob-configmap.yml.j2

@@ -21,7 +21,7 @@ data:
   TSP_API_CLIENT_TOKEN_LIFETIME_MS: "{{ TSP_API_CLIENT_TOKEN_LIFETIME_MS }}"
   TSP_SYNC_SCHOOL_LIMIT: "{{ TSP_SYNC_SCHOOL_LIMIT }}"
   TSP_SYNC_DATA_LIMIT: "{{ TSP_SYNC_DATA_LIMIT }}"
-  TSP_SYNC_SCHOOL_DAYS_TO_FETCH: "{{ TSP_SYNC_SCHOOL_DAYS_TO_FETCH }}"
-  TSP_SYNC_DATA_DAYS_TO_FETCH: "{{ TSP_SYNC_DATA_DAYS_TO_FETCH }}"
+  TSP_SYNC_SCHOOL_DAYS_TO_FETCH: "{{ ((ansible_date_time.date | to_datetime('%Y-%m-%d')) - ("1970-01-01" | to_datetime('%Y-%m-%d'))).days  }}"
+  TSP_SYNC_DATA_DAYS_TO_FETCH: "{{ ((ansible_date_time.date | to_datetime('%Y-%m-%d')) - ("1970-01-01" | to_datetime('%Y-%m-%d'))).days  }}"
   FEATURE_TSP_MIGRATION_ENABLED: "{{ FEATURE_TSP_MIGRATION_ENABLED }}"
   TSP_SYNC_MIGRATION_LIMIT: "{{ TSP_SYNC_MIGRATION_LIMIT }}"

ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-cronjob.yml.j2

@@ -55,7 +55,7 @@ spec:
             - secretRef:
                 name: api-secret
             command: ['/bin/sh', '-c']
-            args: ['npm run nest:start:sync tsp']
+            args: ['npm run nest:start:sync:prod tsp']
             resources:
               limits:
                 cpu: {{ API_CPU_LIMITS|default("2000m", true) }}

ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-deployment.yml.j2

@@ -62,6 +62,7 @@ spec:
             name: api-secret
         - secretRef:
             name: api-files-secret
+        command: ['npm', 'run', 'nest:start:prod']
         readinessProbe:
           httpGet:
             path: /serverversion

ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-init.yml.j2

@@ -42,7 +42,7 @@ spec:
         - secretRef:
             name: api-secret
         command: ['/bin/sh', '-c']
-        args: ['npm run nest:start:sync tsp']
+        args: ['npm run nest:start:sync:prod tsp']
         resources:
           limits:
             cpu: {{ API_CPU_LIMITS|default("2000m", true) }}

apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.spec.ts

@@ -0,0 +1,33 @@
+import { faker } from '@faker-js/faker';
+import { TspDataSyncBatchFinishedLoggable } from './tsp-data-sync-batch-finished.loggable';
+
+describe(TspDataSyncBatchFinishedLoggable.name, () => {
+	describe('getLogMessage is called', () => {
+		const setup = () => {
+			const processedCount = faker.number.int();
+			const batchSize = faker.number.int();
+			const batchCount = faker.number.int();
+			const batchIndex = faker.number.int();
+
+			const expected = {
+				message: `Processed ${processedCount} of ${batchSize} users in batch ${batchIndex} of ${batchCount}.`,
+				data: {
+					processedCount,
+					batchSize,
+					batchCount,
+					batchIndex,
+				},
+			};
+
+			return { processedCount, batchSize, batchCount, batchIndex, expected };
+		};
+
+		it('should return a log message', () => {
+			const { processedCount, batchSize, batchCount, batchIndex, expected } = setup();
+
+			const loggable = new TspDataSyncBatchFinishedLoggable(processedCount, batchSize, batchCount, batchIndex);
+
+			expect(loggable.getLogMessage()).toEqual(expected);
+		});
+	});
+});

apps/server/src/infra/sync/tsp/loggable/tsp-data-sync-batch-finished.loggable.ts

@@ -0,0 +1,24 @@
+import { Loggable, LogMessage } from '@core/logger';
+
+export class TspDataSyncBatchFinishedLoggable implements Loggable {
+	constructor(
+		private readonly processedCount: number,
+		private readonly batchSize: number,
+		private readonly batchCount: number,
+		private readonly batchIndex: number
+	) {}
+
+	public getLogMessage(): LogMessage {
+		const message: LogMessage = {
+			message: `Processed ${this.processedCount} of ${this.batchSize} users in batch ${this.batchIndex} of ${this.batchCount}.`,
+			data: {
+				processedCount: this.processedCount,
+				batchSize: this.batchSize,
+				batchCount: this.batchCount,
+				batchIndex: this.batchIndex,
+			},
+		};
+
+		return message;
+	}
+}

apps/server/src/infra/sync/tsp/tsp-sync.strategy.spec.ts

@@ -10,7 +10,8 @@ import {
 	RobjExportSchule,
 } from '@infra/tsp-client';
 import { Account } from '@modules/account';
-import { ExternalUserDto, OauthDataDto, ProvisioningService, ProvisioningSystemDto } from '@modules/provisioning';
+import { ExternalUserDto, OauthDataDto, ProvisioningSystemDto } from '@modules/provisioning';
+import { TspProvisioningService } from '@modules/provisioning/service/tsp-provisioning.service';
 import { School } from '@modules/school';
 import { schoolFactory } from '@modules/school/testing';
 import { System } from '@modules/system';
@@ -33,7 +34,7 @@ describe(TspSyncStrategy.name, () => {
 	let sut: TspSyncStrategy;
 	let tspSyncService: DeepMocked<TspSyncService>;
 	let tspFetchService: DeepMocked<TspFetchService>;
-	let provisioningService: DeepMocked<ProvisioningService>;
+	let provisioningService: DeepMocked<TspProvisioningService>;
 	let tspOauthDataMapper: DeepMocked<TspOauthDataMapper>;
 	let tspLegacyMigrationService: DeepMocked<TspLegacyMigrationService>;
 	let tspSyncMigrationService: DeepMocked<TspSyncMigrationService>;
@@ -60,8 +61,8 @@ describe(TspSyncStrategy.name, () => {
 					useValue: createMock<ConfigService<TspSyncConfig, true>>(),
 				},
 				{
-					provide: ProvisioningService,
-					useValue: createMock<ProvisioningService>(),
+					provide: TspProvisioningService,
+					useValue: createMock<TspProvisioningService>(),
 				},
 				{
 					provide: TspOauthDataMapper,
@@ -81,7 +82,7 @@ describe(TspSyncStrategy.name, () => {
 		sut = module.get(TspSyncStrategy);
 		tspSyncService = module.get(TspSyncService);
 		tspFetchService = module.get(TspFetchService);
-		provisioningService = module.get(ProvisioningService);
+		provisioningService = module.get(TspProvisioningService);
 		tspOauthDataMapper = module.get(TspOauthDataMapper);
 		tspLegacyMigrationService = module.get(TspLegacyMigrationService);
 		tspSyncMigrationService = module.get(TspSyncMigrationService);
@@ -133,6 +134,7 @@ describe(TspSyncStrategy.name, () => {
 			totalUsers: number;
 			totalAccounts: number;
 		};
+		processBatchSize?: number;
 	}) => {
 		tspFetchService.fetchTspSchools.mockResolvedValueOnce(params.fetchedSchools ?? []);
 		tspFetchService.fetchTspClasses.mockResolvedValueOnce(params.fetchedClasses ?? []);
@@ -156,6 +158,8 @@ describe(TspSyncStrategy.name, () => {
 				totalUsers: faker.number.int(),
 			}
 		);
+
+		provisioningService.provisionBatch.mockResolvedValueOnce(params.processBatchSize ?? 0);
 	};
 
 	describe('sync', () => {
@@ -246,7 +250,7 @@ describe(TspSyncStrategy.name, () => {
 
 				await sut.sync();
 
-				expect(provisioningService.provisionData).toHaveBeenCalledWith(oauthDataDto);
+				expect(provisioningService.provisionBatch).toHaveBeenCalledWith([oauthDataDto]);
 			});
 
 			describe('when feature tsp migration is enabled', () => {

apps/server/src/infra/sync/tsp/tsp-sync.strategy.ts

@@ -1,5 +1,6 @@
 import { Logger } from '@core/logger';
-import { ProvisioningService } from '@modules/provisioning';
+import { OauthDataDto } from '@modules/provisioning';
+import { TspProvisioningService } from '@modules/provisioning/service/tsp-provisioning.service';
 import { School } from '@modules/school';
 import { System } from '@modules/system';
 import { Injectable } from '@nestjs/common';
@@ -8,6 +9,7 @@ import pLimit from 'p-limit';
 import { SyncStrategy } from '../strategy/sync-strategy';
 import { SyncStrategyTarget } from '../sync-strategy.types';
 import { TspDataFetchedLoggable } from './loggable/tsp-data-fetched.loggable';
+import { TspDataSyncBatchFinishedLoggable } from './loggable/tsp-data-sync-batch-finished.loggable';
 import { TspSchoolsFetchedLoggable } from './loggable/tsp-schools-fetched.loggable';
 import { TspSchoolsSyncedLoggable } from './loggable/tsp-schools-synced.loggable';
 import { TspSchulnummerMissingLoggable } from './loggable/tsp-schulnummer-missing.loggable';
@@ -30,7 +32,7 @@ export class TspSyncStrategy extends SyncStrategy {
 		private readonly tspOauthDataMapper: TspOauthDataMapper,
 		private readonly tspLegacyMigrationService: TspLegacyMigrationService,
 		private readonly configService: ConfigService<TspSyncConfig, true>,
-		private readonly provisioningService: ProvisioningService,
+		private readonly provisioningService: TspProvisioningService,
 		private readonly tspSyncMigrationService: TspSyncMigrationService
 	) {
 		super();
@@ -118,16 +120,29 @@ export class TspSyncStrategy extends SyncStrategy {
 
 		this.logger.info(new TspSyncingUsersLoggable(oauthDataDtos.length));
 
-		const dataLimit = this.configService.getOrThrow<number>('TSP_SYNC_DATA_LIMIT');
-		const dataLimitFn = pLimit(dataLimit);
+		const batchSize = this.configService.getOrThrow<number>('TSP_SYNC_DATA_LIMIT');
 
-		const dataPromises = oauthDataDtos.map((oauthDataDto) =>
-			dataLimitFn(() => this.provisioningService.provisionData(oauthDataDto))
+		const batchCount = Math.ceil(oauthDataDtos.length / batchSize);
+		const batches: OauthDataDto[][] = [];
+		for (let i = 0; i < batchCount; i += 1) {
+			const start = i * batchSize;
+			const end = Math.min((i + 1) * batchSize, oauthDataDtos.length);
+			batches.push(oauthDataDtos.slice(start, end));
+		}
+
+		const batchLimit = pLimit(1);
+		const batchPromises = batches.map((batch, index) =>
+			batchLimit(async () => {
+				const processed = await this.provisioningService.provisionBatch(batch);
+				this.logger.info(new TspDataSyncBatchFinishedLoggable(processed, batchSize, batchCount, index));
+				return processed;
+			})
 		);
 
-		const results = await Promise.allSettled(dataPromises);
+		const results = await Promise.all(batchPromises);
+		const total = results.reduce((previousValue, currentValue) => previousValue + currentValue, 0);
 
-		this.logger.info(new TspSyncedUsersLoggable(results.length));
+		this.logger.info(new TspSyncedUsersLoggable(total));
 	}
 
 	private async runMigration(system: System): Promise<void> {