diff --git a/.env.example b/.env.example index 1d8d160..df22182 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,11 @@ APP_LOGO_URL= APP_IMPRINT_URL= APP_PRIVACY_URL= +APPLICATION_LOG_RETENTION_ENABLED=true +APPLICATION_LOG_RETENTION_DAYS=180 +APPLICATION_LOG_CLEANUP_BATCH_SIZE=1000 +APPLICATION_LOG_CLEANUP_MAX_BATCHES=20 + DATABASE_URL=mysql://ldap_portal:change-me@mysql.example.com:3306/ldap_portal DATABASE_SSL=false JWT_SECRET=change-me-long-random-jwt-secret diff --git a/README.md b/README.md index 26ea757..fdc6a93 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,10 @@ Die wichtigsten Variablen aus `.env.example`: | `APP_PRODUCT_NAME`, `APP_COMPANY_NAME` | Zentrale Branding-Namen fuer OIDC-Seiten und Mail-Templates. | | `APP_PRIMARY_COLOR` | Zentrale Primaerfarbe als sechsstelliger Hex-Wert. | | `APP_SUPPORT_EMAIL`, `APP_LOGO_URL`, `APP_IMPRINT_URL`, `APP_PRIVACY_URL` | Zentrale Branding-Werte; Bild- und Link-URLs muessen HTTPS verwenden. | +| `APPLICATION_LOG_RETENTION_ENABLED` | Aktiviert den taeglichen Cleanup von Application Info- und Error-Logs; Standard `true`. | +| `APPLICATION_LOG_RETENTION_DAYS` | Aufbewahrungsdauer beider Logtabellen in Tagen; Standard `180`. | +| `APPLICATION_LOG_CLEANUP_BATCH_SIZE` | Datensaetze pro Loeschbatch; Standard `1000`. | +| `APPLICATION_LOG_CLEANUP_MAX_BATCHES` | Maximale Batches pro Tabelle und Lauf; Standard `20`. | | `DATABASE_URL` | MySQL-Verbindungs-URL. Alternativ `DB_HOST`, `DB_PORT`, `DB_USERNAME`, `DB_PASSWORD`, `DB_DATABASE`. | | `DATABASE_SSL` | `true`, wenn MySQL TLS verlangt. | | `JWT_SECRET` | Signatur-Secret fuer Portal-JWTs. | @@ -251,7 +255,9 @@ Fachliche, sicherheitsrelevante und technische Aktivitaeten werden dauerhaft in Wichtige Felder sind `action`, `category`, `outcome`, `actorType`, `actorId`, `userId`, `clientId`, `correlationId`, HTTP-Metadaten und bereinigter JSON-Kontext. Passwoerter, Tokens, Secrets, Cookies und Authorization-Daten werden entfernt; E-Mail-Adressen werden maskiert. Fehler beim Schreiben des Info-Logs werden auf dem Standardlogger ausgegeben und beeinflussen die protokollierte Aktion nicht. -Erfasst werden unter anderem Portal-Login und -Logout, fehlgeschlagene Logins, OIDC-Benutzerlogin, erfolgreiche Client-Autorisierung, Token-Grants, OIDC-Logout, Consent, Konto-, Passwort-, Registrierungs- und administrative Aenderungen. Die Migration `1721300000000-CreateApplicationInfoLogs.ts` muss in Produktion vor dem Deployment angewendet werden. Als Ausgangspunkt fuer die Aufbewahrung gelten 180 Tage; eine automatische Loeschung ist bewusst nicht aktiviert und muss mit der betrieblichen Audit-Policy abgestimmt werden. +Erfasst werden unter anderem Portal-Login und -Logout, fehlgeschlagene Logins, OIDC-Benutzerlogin, erfolgreiche Client-Autorisierung, Token-Grants, OIDC-Logout, Consent, Konto-, Passwort-, Registrierungs- und administrative Aenderungen. Die Migration `1721300000000-CreateApplicationInfoLogs.ts` muss in Produktion vor dem Deployment angewendet werden. + +Der NestJS-Scheduler entfernt taeglich um 03:15 Serverzeit Info- und Error-Logs, die aelter als `APPLICATION_LOG_RETENTION_DAYS` sind. Die Loeschung erfolgt in begrenzten Batches und wird ueber einen MySQL Advisory Lock zwischen mehreren Instanzen koordiniert. Ein erfolgreicher Lauf erzeugt `maintenance.application_log_cleanup_completed`; Fehler werden nur im Standardlogger ausgegeben und stoppen die Anwendung nicht. ## Produktions-Checkliste diff --git a/apps/api/package.json b/apps/api/package.json index 2b87071..c75c5ee 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -17,6 +17,7 @@ "@nestjs/core": "^11.0.0", "@nestjs/jwt": "^11.0.0", "@nestjs/platform-express": "^11.0.0", + "@nestjs/schedule": "^6.1.3", "@nestjs/throttler": "^6.4.0", "@nestjs/typeorm": "^11.0.0", "bcryptjs": "^2.4.3", diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index e4caf0e..661412d 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; import { ConfigModule, ConfigService } from '@nestjs/config'; +import { ScheduleModule } from '@nestjs/schedule'; import { ThrottlerModule } from '@nestjs/throttler'; import { TypeOrmModule } from '@nestjs/typeorm'; import { AuditModule } from './audit/audit.module'; @@ -19,6 +20,7 @@ import { ApplicationInfoLogModule } from './application-info-log/application-inf isGlobal: true, envFilePath: ['.env', '../../.env'], }), + ScheduleModule.forRoot(), ThrottlerModule.forRoot([ { ttl: 60_000, diff --git a/apps/api/src/application-info-log/application-info-log.module.ts b/apps/api/src/application-info-log/application-info-log.module.ts index 348adb8..0e10ef4 100644 --- a/apps/api/src/application-info-log/application-info-log.module.ts +++ b/apps/api/src/application-info-log/application-info-log.module.ts @@ -5,12 +5,14 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { ApplicationInfoLog } from './application-info-log.entity'; import { ApplicationInfoLoggerService } from './application-info-logger.service'; import { ApplicationInfoInterceptor } from './application-info.interceptor'; +import { ApplicationLogRetentionService } from './application-log-retention.service'; @Global() @Module({ imports: [ConfigModule, TypeOrmModule.forFeature([ApplicationInfoLog])], providers: [ ApplicationInfoLoggerService, + ApplicationLogRetentionService, { provide: APP_INTERCEPTOR, useClass: ApplicationInfoInterceptor }, ], exports: [ApplicationInfoLoggerService], diff --git a/apps/api/src/application-info-log/application-log-retention.service.spec.ts b/apps/api/src/application-info-log/application-log-retention.service.spec.ts new file mode 100644 index 0000000..896e0e8 --- /dev/null +++ b/apps/api/src/application-info-log/application-log-retention.service.spec.ts @@ -0,0 +1,87 @@ +import { Logger } from '@nestjs/common'; +import { ApplicationLogRetentionService } from './application-log-retention.service'; + +describe('ApplicationLogRetentionService', () => { + function createService(options: { lock?: boolean; queryError?: Error; enabled?: string } = {}) { + let infoBatch = 0; + const query = jest.fn(async (sql: string, _parameters?: unknown[]) => { + if (options.queryError && sql.startsWith('DELETE')) { + throw options.queryError; + } + if (sql.includes('GET_LOCK')) { + return [{ acquired: options.lock === false ? 0 : 1 }]; + } + if (sql.startsWith('DELETE FROM application_info_logs')) { + infoBatch += 1; + return { affectedRows: infoBatch === 1 ? 100 : 10 }; + } + if (sql.startsWith('DELETE FROM application_error_logs')) { + return { affectedRows: 5 }; + } + return []; + }); + const queryRunner = { + connect: jest.fn().mockResolvedValue(undefined), + query, + release: jest.fn().mockResolvedValue(undefined), + isReleased: false, + }; + const dataSource = { createQueryRunner: jest.fn().mockReturnValue(queryRunner) }; + const values: Record = { + APPLICATION_LOG_RETENTION_ENABLED: options.enabled ?? 'true', + APPLICATION_LOG_RETENTION_DAYS: '30', + APPLICATION_LOG_CLEANUP_BATCH_SIZE: '100', + APPLICATION_LOG_CLEANUP_MAX_BATCHES: '3', + }; + const config = { get: jest.fn((key: string) => values[key]) }; + const infoLogger = { log: jest.fn().mockResolvedValue(undefined) }; + const service = new ApplicationLogRetentionService(dataSource as any, config as any, infoLogger as any); + return { service, dataSource, queryRunner, query, infoLogger }; + } + + it('deletes expired info and error logs in bounded batches', async () => { + const { service, query, queryRunner, infoLogger } = createService(); + const before = Date.now() - 30 * 24 * 60 * 60_000; + + await service.removeExpiredLogs(); + + const infoDeletes = query.mock.calls.filter(([sql]) => sql.startsWith('DELETE FROM application_info_logs')); + const errorDeletes = query.mock.calls.filter(([sql]) => sql.startsWith('DELETE FROM application_error_logs')); + expect(infoDeletes).toHaveLength(2); + expect(errorDeletes).toHaveLength(1); + const firstParameters = infoDeletes[0]?.[1]; + expect(firstParameters?.[1]).toBe(100); + expect((firstParameters?.[0] as Date).getTime()).toBeGreaterThanOrEqual(before - 1_000); + expect(queryRunner.release).toHaveBeenCalled(); + expect(infoLogger.log).toHaveBeenCalledWith(expect.objectContaining({ + action: 'maintenance.application_log_cleanup_completed', + context: expect.objectContaining({ infoLogsDeleted: 110, errorLogsDeleted: 5, retentionDays: 30 }), + })); + }); + + it('skips cleanup when another instance owns the database lock', async () => { + const { service, query, infoLogger } = createService({ lock: false }); + + await service.removeExpiredLogs(); + + expect(query.mock.calls.some(([sql]) => sql.startsWith('DELETE'))).toBe(false); + expect(infoLogger.log).not.toHaveBeenCalled(); + }); + + it('can be disabled through configuration', async () => { + const { service, dataSource } = createService({ enabled: 'false' }); + + await service.removeExpiredLogs(); + + expect(dataSource.createQueryRunner).not.toHaveBeenCalled(); + }); + + it('does not throw when cleanup persistence fails', async () => { + jest.spyOn(Logger.prototype, 'error').mockImplementation(); + const { service } = createService({ queryError: new Error('database unavailable') }); + + await expect(service.removeExpiredLogs()).resolves.toBeUndefined(); + expect(Logger.prototype.error).toHaveBeenCalled(); + jest.restoreAllMocks(); + }); +}); diff --git a/apps/api/src/application-info-log/application-log-retention.service.ts b/apps/api/src/application-info-log/application-log-retention.service.ts new file mode 100644 index 0000000..08bb6c6 --- /dev/null +++ b/apps/api/src/application-info-log/application-log-retention.service.ts @@ -0,0 +1,128 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Cron } from '@nestjs/schedule'; +import { DataSource, QueryRunner } from 'typeorm'; +import { ApplicationInfoLoggerService } from './application-info-logger.service'; + +interface CleanupResult { + infoLogsDeleted: number; + errorLogsDeleted: number; +} + +@Injectable() +export class ApplicationLogRetentionService { + private readonly logger = new Logger(ApplicationLogRetentionService.name); + + constructor( + private readonly dataSource: DataSource, + private readonly config: ConfigService, + private readonly infoLogger: ApplicationInfoLoggerService, + ) {} + + @Cron('0 15 3 * * *', { name: 'application-log-retention', waitForCompletion: true }) + async removeExpiredLogs(): Promise { + if (this.config.get('APPLICATION_LOG_RETENTION_ENABLED') === 'false') { + return; + } + + const retentionDays = this.numberConfig('APPLICATION_LOG_RETENTION_DAYS', 180, 1, 3_650); + const batchSize = this.numberConfig('APPLICATION_LOG_CLEANUP_BATCH_SIZE', 1_000, 100, 10_000); + const maxBatches = this.numberConfig('APPLICATION_LOG_CLEANUP_MAX_BATCHES', 20, 1, 1_000); + const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60_000); + const queryRunner = this.dataSource.createQueryRunner(); + + try { + await queryRunner.connect(); + if (!(await this.acquireLock(queryRunner))) { + this.logger.log('Application log cleanup skipped because another instance holds the lock'); + return; + } + + const result: CleanupResult = { + infoLogsDeleted: await this.deleteBatches( + queryRunner, + 'application_info_logs', + cutoff, + batchSize, + maxBatches, + ), + errorLogsDeleted: await this.deleteBatches( + queryRunner, + 'application_error_logs', + cutoff, + batchSize, + maxBatches, + ), + }; + + this.logger.log( + `Application log cleanup completed: info=${result.infoLogsDeleted}, errors=${result.errorLogsDeleted}, retentionDays=${retentionDays}`, + ); + await this.infoLogger.log({ + action: 'maintenance.application_log_cleanup_completed', + category: 'MAINTENANCE', + outcome: 'SUCCESS', + actorType: 'SYSTEM', + actorId: 'application-log-retention', + backendModule: 'ApplicationInfoLogModule', + service: ApplicationLogRetentionService.name, + operation: 'removeExpiredLogs', + context: { ...result, retentionDays, cutoff: cutoff.toISOString() }, + }); + } catch (error) { + this.logger.error( + `Application log cleanup failed: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error.stack : undefined, + ); + } finally { + if (queryRunner.isReleased === false) { + await this.releaseLock(queryRunner); + await queryRunner.release(); + } + } + } + + private async deleteBatches( + queryRunner: QueryRunner, + tableName: 'application_info_logs' | 'application_error_logs', + cutoff: Date, + batchSize: number, + maxBatches: number, + ): Promise { + let deleted = 0; + for (let batch = 0; batch < maxBatches; batch += 1) { + const result = (await queryRunner.query( + `DELETE FROM ${tableName} WHERE createdAt < ? ORDER BY createdAt ASC LIMIT ?`, + [cutoff, batchSize], + )) as { affectedRows?: number }; + const affectedRows = Number(result.affectedRows ?? 0); + deleted += affectedRows; + if (affectedRows < batchSize) { + break; + } + } + return deleted; + } + + private async acquireLock(queryRunner: QueryRunner): Promise { + const rows = (await queryRunner.query( + "SELECT GET_LOCK('ldap_portal_application_log_cleanup', 0) AS acquired", + )) as Array<{ acquired?: number | string }>; + return Number(rows[0]?.acquired) === 1; + } + + private async releaseLock(queryRunner: QueryRunner): Promise { + try { + await queryRunner.query("SELECT RELEASE_LOCK('ldap_portal_application_log_cleanup')"); + } catch (error) { + this.logger.warn( + `Application log cleanup lock release failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + private numberConfig(key: string, fallback: number, minimum: number, maximum: number): number { + const value = Number(this.config.get(key) ?? fallback); + return Number.isInteger(value) && value >= minimum && value <= maximum ? value : fallback; + } +} diff --git a/package-lock.json b/package-lock.json index ac8ee21..0c82507 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "@nestjs/core": "^11.0.0", "@nestjs/jwt": "^11.0.0", "@nestjs/platform-express": "^11.0.0", + "@nestjs/schedule": "^6.1.3", "@nestjs/throttler": "^6.4.0", "@nestjs/typeorm": "^11.0.0", "bcryptjs": "^2.4.3", @@ -6801,6 +6802,19 @@ "@nestjs/core": "^11.0.0" } }, + "node_modules/@nestjs/schedule": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-6.1.3.tgz", + "integrity": "sha512-RflMFOpR16Dwd1jAUbeB4mfGTCh65fvEdL4mSjQPJChpkRGRjIXjb+6YQcK2faQrVT60c9DmLmoVR7/ONCtuYQ==", + "license": "MIT", + "dependencies": { + "cron": "4.4.0" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, "node_modules/@nestjs/schematics": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-11.1.0.tgz", @@ -8806,6 +8820,12 @@ "@types/koa": "*" } }, + "node_modules/@types/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-gW+Oib+vUtGJBtNC8V9Reww0oIpusw+4m81uncg9REGZAJfqOQHfo/nkabnc7w0QReXyPqjrbWMJk6NuAkiX3Q==", + "license": "MIT" + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -11494,6 +11514,23 @@ "devOptional": true, "license": "MIT" }, + "node_modules/cron": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/cron/-/cron-4.4.0.tgz", + "integrity": "sha512-fkdfq+b+AHI4cKdhZlppHveI/mgz2qpiYxcm+t5E5TsxX7QrLS1VE0+7GENEk9z0EeGPcpSciGv6ez24duWhwQ==", + "license": "MIT", + "dependencies": { + "@types/luxon": "~3.7.0", + "luxon": "~3.7.0" + }, + "engines": { + "node": ">=18.x" + }, + "funding": { + "type": "ko-fi", + "url": "https://ko-fi.com/intcreator" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -16762,6 +16799,15 @@ "url": "https://github.com/sponsors/wellwelwel" } }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.17", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz",