From ea68e75fe05e68f8be8d1df940b59787b931c4cb Mon Sep 17 00:00:00 2001 From: William Phetsinorath Date: Wed, 25 Mar 2026 14:16:19 +0100 Subject: [PATCH 1/4] refactor(keycloak): migrate Keycloak plugin to NestJS Signed-off-by: William Phetsinorath --- AGENTS.md | 1 - apps/server-nestjs/.env-example | 8 + apps/server-nestjs/.env.docker-example | 4 + apps/server-nestjs/.env.integ-example | 4 + .../02-ARCHITECTURE-MODULES.md | 120 ++++ apps/server-nestjs/package.json | 8 +- .../configuration/configuration.service.ts | 3 + apps/server-nestjs/src/main.module.ts | 12 +- .../src/modules/healthz/healthz.controller.ts | 3 + .../src/modules/healthz/healthz.module.ts | 7 +- .../keycloak/keycloack-testing.utils.ts | 86 +++ .../keycloak/keycloak-client.service.ts | 242 ++++++++ .../keycloak/keycloak-datastore.service.ts | 50 ++ .../keycloak/keycloak-health.service.ts | 29 + .../modules/keycloak/keycloak.constants.ts | 3 + .../src/modules/keycloak/keycloak.module.ts | 30 + .../modules/keycloak/keycloak.service.spec.ts | 381 ++++++++++++ .../src/modules/keycloak/keycloak.service.ts | 580 ++++++++++++++++++ .../src/prisma/schema/project.prisma | 2 + apps/server-nestjs/src/utils/iterable.ts | 19 + apps/server-nestjs/test/keycloak.e2e-spec.ts | 531 ++++++++++++++++ pnpm-lock.yaml | 127 +++- 22 files changed, 2215 insertions(+), 35 deletions(-) create mode 100644 apps/server-nestjs/documentation/Modularisation-de-console-server/02-ARCHITECTURE-MODULES.md create mode 100644 apps/server-nestjs/src/modules/keycloak/keycloack-testing.utils.ts create mode 100644 apps/server-nestjs/src/modules/keycloak/keycloak-client.service.ts create mode 100644 apps/server-nestjs/src/modules/keycloak/keycloak-datastore.service.ts create mode 100644 apps/server-nestjs/src/modules/keycloak/keycloak-health.service.ts create mode 100644 apps/server-nestjs/src/modules/keycloak/keycloak.constants.ts create mode 100644 apps/server-nestjs/src/modules/keycloak/keycloak.module.ts create mode 100644 apps/server-nestjs/src/modules/keycloak/keycloak.service.spec.ts create mode 100644 apps/server-nestjs/src/modules/keycloak/keycloak.service.ts create mode 100644 apps/server-nestjs/src/utils/iterable.ts create mode 100644 apps/server-nestjs/test/keycloak.e2e-spec.ts diff --git a/AGENTS.md b/AGENTS.md index 1a99233e5..5e32b00b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,6 @@ Plugins use TS module augmentation to extend `ProjectStore` and `Config` interfa ## Database (Prisma) Multi-file schema in `apps/server/src/prisma/schema/*.prisma` (project, user, token, admin, topography). -Singleton PrismaClient in `apps/server/src/prisma.ts`. Queries centralized per resource, re-exported via `queries-index.ts`. Migrations: standard Prisma Migrate. Major version data migrations in `migrations/v9/`. ## Environment config diff --git a/apps/server-nestjs/.env-example b/apps/server-nestjs/.env-example index 0525122fc..f4f7960cc 100644 --- a/apps/server-nestjs/.env-example +++ b/apps/server-nestjs/.env-example @@ -15,6 +15,14 @@ KEYCLOAK_PROTOCOL=http KEYCLOAK_CLIENT_ID=dso-console-backend # Secret du client Keycloak backend (confidentiel) KEYCLOAK_CLIENT_SECRET=client-secret-backend +# Identifiant de l'administrateur Keycloak +KEYCLOAK_ADMIN=admin +# Mot de passe de l'administrateur Keycloak +KEYCLOAK_ADMIN_PASSWORD=admin +# Identifiant administrateur Keycloak (utilisé pour l'API admin) +KEYCLOAK_ADMIN=admin +# Mot de passe administrateur Keycloak (confidentiel) +KEYCLOAK_ADMIN_PASSWORD=admin # URL de redirection après authentification Keycloak KEYCLOAK_REDIRECT_URI=http://localhost:8080 # Port d'écoute du serveur backend diff --git a/apps/server-nestjs/.env.docker-example b/apps/server-nestjs/.env.docker-example index 0dc790476..b2034118c 100644 --- a/apps/server-nestjs/.env.docker-example +++ b/apps/server-nestjs/.env.docker-example @@ -16,6 +16,10 @@ KEYCLOAK_PROTOCOL=http KEYCLOAK_CLIENT_ID=dso-console-backend # Secret du client Keycloak backend (confidentiel) KEYCLOAK_CLIENT_SECRET=client-secret-backend +# Identifiant de l'administrateur Keycloak +KEYCLOAK_ADMIN=admin +# Mot de passe de l'administrateur Keycloak +KEYCLOAK_ADMIN_PASSWORD=admin # URL de redirection après authentification Keycloak KEYCLOAK_REDIRECT_URI=http://localhost:8080 # Port d'écoute du serveur dans le réseau Docker diff --git a/apps/server-nestjs/.env.integ-example b/apps/server-nestjs/.env.integ-example index d5850a09c..8c515966b 100644 --- a/apps/server-nestjs/.env.integ-example +++ b/apps/server-nestjs/.env.integ-example @@ -17,6 +17,10 @@ KEYCLOAK_CLIENT_SECRET= KEYCLOAK_DOMAIN= # Royaume Keycloak d'intégration KEYCLOAK_REALM= +# Identifiant de l'administrateur Keycloak +KEYCLOAK_ADMIN= +# Mot de passe de l'administrateur Keycloak +KEYCLOAK_ADMIN_PASSWORD= # --- ArgoCD --- # Namespace Kubernetes dans lequel ArgoCD est déployé diff --git a/apps/server-nestjs/documentation/Modularisation-de-console-server/02-ARCHITECTURE-MODULES.md b/apps/server-nestjs/documentation/Modularisation-de-console-server/02-ARCHITECTURE-MODULES.md new file mode 100644 index 000000000..257b26040 --- /dev/null +++ b/apps/server-nestjs/documentation/Modularisation-de-console-server/02-ARCHITECTURE-MODULES.md @@ -0,0 +1,120 @@ +# Architecture d’un module (pattern `apps/server-nestjs/src/modules/*`) + +Les modules NestJS métier vivent dans `src/modules//` et suivent un découpage “vertical slice” avec des responsabilités explicites : **client**, **service (API publique)**, **controller service (orchestration)**, **datastore**, **utils** et **tests**. + +Exemples concrets : + +- Module GitLab : `src/modules/gitlab/` +- Module Keycloak : `src/modules/keycloak/` + +## Structure type + +```txt +src/modules// +├── .module.ts +├── .constants.ts +├── -client.service.ts +├── .service.ts +├── -datastore.service.ts +├── .utils.ts +├── -testing.utils.ts +└── *.spec.ts +``` + +## Sens des dépendances (flow recommandé) + +Objectif : un flux de dépendances lisible et sans cycles. + +```txt +.service.ts + ↙ ↘ +-client.service.ts -datastore.service.ts +``` + +Règles pratiques : + +- Le `.service.ts` est un entrypoint interne (cron, events, reconcile) et orchestre en appelant directement le `client` (et le `datastore` si nécessaire), sans dépendre du `service`. +- Le `service` contient les règles métier (décisions, transformations, validations) et s’appuie sur le `client` (et le `datastore` quand la lecture/écriture DB fait partie du cas d’usage). +- Le `client` encapsule l’accès à une API externe (initialisation + appels + erreurs bas niveau). +- Le `datastore` encapsule l’accès DB (Prisma) et expose des méthodes de lecture/écriture typées. +- Les `utils` restent “purs” (pas d’IO, pas d’injection Nest). +- Les `testing utils` centralisent les factories/fixtures pour réduire la duplication dans les tests. + +## Composants + +### `.module.ts` + +Rôle : + +- Déclare les providers, imports, exports du module. +- Exporte le `service` du module (`.service.ts`) qui constitue l’API publique. + +### `-client.service.ts` + +Rôle : + +- Adapter vers le système externe (SDK HTTP, client Keycloak, client GitLab, etc.). +- Conserver un contrat stable pour le reste du module. +- Mapper/normaliser les erreurs externes si nécessaire. + +À éviter : + +- Décisions métier (permissions, synchronisation, règles de purge) : elles vont dans `.service.ts` ou le controller service. + +### `.service.ts` + +Rôle : + +- Orchestrateur de workflows : `@Cron`, `@OnEvent`, reconcile périodique, tâches “batch”. +- Coordination entre `client` et `datastore` (sans dépendre du `service`). +- Garde-fous “safety” avant opérations destructrices (ex: suppression de groupes orphelins). + +### `-datastore.service.ts` + +Rôle : + +- Accès DB via Prisma (select/include, transactions, pagination). +- Exposition de types agrégés utiles au domaine (ex: `ProjectWithDetails`). + +À éviter : + +- Appliquer des règles métier (ex: calcul de permissions) : on garde le datastore centré persistence. + +### `.utils.ts` + +Rôle : + +- Fonctions utilitaires pures : mapping, helpers de collections, types partagés. +- Aucune dépendance Nest, aucune lecture/écriture DB, aucun appel réseau. + +### `-testing.utils.ts` + +Rôle : + +- Factories typées pour les structures fréquemment utilisées en tests. +- Support d’`overrides` pour construire rapidement des variantes. +- Centralisation des erreurs/fake responses spécifiques au module (quand utile). + +## Tests (Vitest) + +### `.service.spec.ts` + +Cible : + +- Orchestration : séquences d’appels, side-effects attendus, reconcile. + +Approche : + +- Mock du `service`, du `datastore`, et des appels externes. +- Vérification des appels effectués et des paramètres attendus. + +### `-datastore.service.spec.ts` (si présent) + +Cible : + +- Forme des requêtes Prisma, mapping de résultat, typage de l’agrégat renvoyé. + +Approche : + +- Mock de Prisma/DatabaseService, pas de logique métier. + diff --git a/apps/server-nestjs/package.json b/apps/server-nestjs/package.json index 6fdd2db72..71057c25a 100644 --- a/apps/server-nestjs/package.json +++ b/apps/server-nestjs/package.json @@ -44,11 +44,14 @@ "@fastify/swagger-ui": "^4.2.0", "@gitbeaker/core": "^40.6.0", "@gitbeaker/rest": "^40.6.0", + "@keycloak/keycloak-admin-client": "^24.0.0", "@kubernetes-models/argo-cd": "^2.7.2", "@nestjs/common": "^11.1.16", "@nestjs/config": "^4.0.3", "@nestjs/core": "^11.1.16", + "@nestjs/event-emitter": "^3.0.1", "@nestjs/platform-express": "^11.1.16", + "@nestjs/schedule": "^6.1.1", "@nestjs/terminus": "^11.1.1", "@opentelemetry/api": "^1.9.0", "@opentelemetry/auto-instrumentations-node": "^0.70.1", @@ -76,7 +79,9 @@ "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", "undici": "^7.24.0", - "vitest-mock-extended": "^2.0.2" + "vitest-mock-extended": "^2.0.2", + "yaml": "^2.8.2", + "zod": "^3.25.76" }, "devDependencies": { "@cpn-console/eslint-config": "workspace:^", @@ -95,6 +100,7 @@ "eslint": "^9.39.4", "fastify-plugin": "^5.1.0", "globals": "^16.5.0", + "msw": "^2.12.10", "nodemon": "^3.1.14", "pino-pretty": "^13.1.3", "rimraf": "^6.1.3", diff --git a/apps/server-nestjs/src/cpin-module/infrastructure/configuration/configuration.service.ts b/apps/server-nestjs/src/cpin-module/infrastructure/configuration/configuration.service.ts index b3af13bcb..f049b3764 100644 --- a/apps/server-nestjs/src/cpin-module/infrastructure/configuration/configuration.service.ts +++ b/apps/server-nestjs/src/cpin-module/infrastructure/configuration/configuration.service.ts @@ -24,7 +24,10 @@ export class ConfigurationService { keycloakRealm = process.env.KEYCLOAK_REALM keycloakClientId = process.env.KEYCLOAK_CLIENT_ID keycloakClientSecret = process.env.KEYCLOAK_CLIENT_SECRET + keycloakAdmin = process.env.KEYCLOAK_ADMIN + keycloakAdminPassword = process.env.KEYCLOAK_ADMIN_PASSWORD keycloakRedirectUri = process.env.KEYCLOAK_REDIRECT_URI + adminsUserId = process.env.ADMIN_KC_USER_ID ? process.env.ADMIN_KC_USER_ID.split(',') : [] diff --git a/apps/server-nestjs/src/main.module.ts b/apps/server-nestjs/src/main.module.ts index c92e41621..6fbba7bb4 100644 --- a/apps/server-nestjs/src/main.module.ts +++ b/apps/server-nestjs/src/main.module.ts @@ -1,10 +1,20 @@ import { Module } from '@nestjs/common' +import { EventEmitterModule } from '@nestjs/event-emitter' +import { ScheduleModule } from '@nestjs/schedule' import { CpinModule } from './cpin-module/cpin.module' import { HealthzModule } from './modules/healthz/healthz.module' +import { KeycloakModule } from './modules/keycloak/keycloak.module' import { VersionModule } from './modules/version/version.module' @Module({ - imports: [CpinModule, HealthzModule, VersionModule], + imports: [ + CpinModule, + KeycloakModule, + HealthzModule, + VersionModule, + EventEmitterModule.forRoot(), + ScheduleModule.forRoot(), + ], controllers: [], providers: [], }) diff --git a/apps/server-nestjs/src/modules/healthz/healthz.controller.ts b/apps/server-nestjs/src/modules/healthz/healthz.controller.ts index 25472da06..68724cd78 100644 --- a/apps/server-nestjs/src/modules/healthz/healthz.controller.ts +++ b/apps/server-nestjs/src/modules/healthz/healthz.controller.ts @@ -1,12 +1,14 @@ import { Controller, Get, Inject } from '@nestjs/common' import { HealthCheck, HealthCheckService } from '@nestjs/terminus' import { DatabaseHealthService } from '../../cpin-module/infrastructure/database/database-health.service' +import { KeycloakHealthService } from '../keycloak/keycloak-health.service' @Controller('api/v1/healthz') export class HealthzController { constructor( @Inject(HealthCheckService) private readonly health: HealthCheckService, @Inject(DatabaseHealthService) private readonly database: DatabaseHealthService, + @Inject(KeycloakHealthService) private readonly keycloak: KeycloakHealthService, ) {} @Get() @@ -14,6 +16,7 @@ export class HealthzController { check() { return this.health.check([ () => this.database.check('database'), + () => this.keycloak.check('keycloak'), ]) } } diff --git a/apps/server-nestjs/src/modules/healthz/healthz.module.ts b/apps/server-nestjs/src/modules/healthz/healthz.module.ts index 58b31f46b..7e7307126 100644 --- a/apps/server-nestjs/src/modules/healthz/healthz.module.ts +++ b/apps/server-nestjs/src/modules/healthz/healthz.module.ts @@ -1,10 +1,15 @@ import { Module } from '@nestjs/common' import { TerminusModule } from '@nestjs/terminus' +import { KeycloakModule } from '../keycloak/keycloak.module' import { DatabaseModule } from '../../cpin-module/infrastructure/database/database.module' import { HealthzController } from './healthz.controller' @Module({ - imports: [TerminusModule, DatabaseModule], + imports: [ + TerminusModule, + DatabaseModule, + KeycloakModule, + ], controllers: [HealthzController], }) export class HealthzModule {} diff --git a/apps/server-nestjs/src/modules/keycloak/keycloack-testing.utils.ts b/apps/server-nestjs/src/modules/keycloak/keycloack-testing.utils.ts new file mode 100644 index 000000000..9f8062bf2 --- /dev/null +++ b/apps/server-nestjs/src/modules/keycloak/keycloack-testing.utils.ts @@ -0,0 +1,86 @@ +import type GroupRepresentation from '@keycloak/keycloak-admin-client/lib/defs/groupRepresentation' +import type UserRepresentation from '@keycloak/keycloak-admin-client/lib/defs/userRepresentation' +import type { ProjectWithDetails } from './keycloak-datastore.service' + +import { faker } from '@faker-js/faker' + +export function makeUserRepresentation( + overrides: Partial = {}, +) { + return { + id: faker.string.uuid(), + email: faker.internet.email().toLowerCase(), + username: faker.internet.username(), + enabled: true, + ...overrides, + } satisfies UserRepresentation +} + +export function makeGroupRepresentation( + overrides: Partial = {}, +) { + return { + id: faker.string.uuid(), + name: faker.word.noun(), + path: `/${faker.word.noun()}`, + subGroups: [], + ...overrides, + } satisfies GroupRepresentation +} + +export function makeProjectUser( + overrides: Partial = {}, +) { + return { + id: faker.string.uuid(), + email: faker.internet.email().toLowerCase(), + ...overrides, + } satisfies ProjectWithDetails['members'][number]['user'] +} + +export function makeProjectMember( + overrides: Partial = {}, +) { + return { + roleIds: [], + user: makeProjectUser(), + ...overrides, + } satisfies ProjectWithDetails['members'][number] +} + +export function makeProjectRole( + overrides: Partial = {}, +) { + return { + id: faker.string.uuid(), + permissions: 0n, + oidcGroup: '', + type: 'managed', + ...overrides, + } satisfies ProjectWithDetails['roles'][number] +} + +export function makeProjectEnvironment( + overrides: Partial = {}, +) { + return { + id: faker.string.uuid(), + name: faker.word.noun(), + ...overrides, + } satisfies ProjectWithDetails['environments'][number] +} + +export function makeProjectWithDetails( + overrides: Partial = {}, +) { + return { + id: faker.string.uuid(), + slug: faker.helpers.slugify(faker.word.words({ count: 2 })).toLowerCase(), + ownerId: faker.string.uuid(), + everyonePerms: 0n, + members: [], + roles: [], + environments: [], + ...overrides, + } satisfies ProjectWithDetails +} diff --git a/apps/server-nestjs/src/modules/keycloak/keycloak-client.service.ts b/apps/server-nestjs/src/modules/keycloak/keycloak-client.service.ts new file mode 100644 index 000000000..204c331e8 --- /dev/null +++ b/apps/server-nestjs/src/modules/keycloak/keycloak-client.service.ts @@ -0,0 +1,242 @@ +import type KcAdminClient from '@keycloak/keycloak-admin-client' +import type GroupRepresentation from '@keycloak/keycloak-admin-client/lib/defs/groupRepresentation' +import type { OnModuleInit } from '@nestjs/common' +import type { ProjectWithDetails } from './keycloak-datastore.service' +import { Inject, Injectable, Logger } from '@nestjs/common' +import { trace } from '@opentelemetry/api' +import z from 'zod' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' +import { CONSOLE_GROUP_NAME, SUBGROUPS_PAGINATE_QUERY_MAX } from './keycloak.constants' + +export type GroupRepresentationWith = GroupRepresentation & Required> + +export const KEYCLOAK_ADMIN_CLIENT = Symbol('KEYCLOAK_ADMIN_CLIENT') + +@Injectable() +export class KeycloakClientService implements OnModuleInit { + private readonly logger = new Logger(KeycloakClientService.name) + + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(KEYCLOAK_ADMIN_CLIENT) private readonly client: KcAdminClient, + ) { + } + + async* getAllGroups() { + let first = 0 + while (true) { + const fetched = await this.client.groups.find({ first, max: SUBGROUPS_PAGINATE_QUERY_MAX, briefRepresentation: false }) + if (fetched.length === 0) break + for (const group of fetched) { + yield group + } + if (fetched.length < SUBGROUPS_PAGINATE_QUERY_MAX) break + first += SUBGROUPS_PAGINATE_QUERY_MAX + } + } + + @StartActiveSpan() + async getGroupByName(name: string): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('group.name', name) + const groups = await this.client.groups.find({ search: name, briefRepresentation: false }) ?? [] + return groups.find(g => g.name === name) + } + + async getGroupByPath(path: string): Promise { + const parts = path.split('/').filter(Boolean) + let current: GroupRepresentationWith<'id'> | undefined + + for (const name of parts) { + current = current + ? await this.getSubGroupByName(current.id, name) + : await this.getRootGroupByName(name) + + if (!current) return undefined + } + return current + } + + private async getSubGroupByName(parentId: string, name: string): Promise | undefined> { + for await (const subgroup of this.getSubGroups(parentId)) { + if (subgroup.name === name) { + const parsed = z.object({ id: z.string() }).and(z.record(z.string(), z.unknown())).safeParse(subgroup) + return parsed.success ? parsed.data : undefined + } + } + return undefined + } + + private async getRootGroupByName(name: string): Promise | undefined> { + const candidates = await this.client.groups.find({ search: name, briefRepresentation: false }) ?? [] + const match = candidates.find(g => g.path === `/${name}`) ?? candidates.find(g => g.name === name) + const parsed = z.object({ id: z.string() }).and(z.record(z.string(), z.unknown())).safeParse(match) + return parsed.success ? parsed.data : undefined + } + + async deleteGroup(id: string): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('keycloak.group.id', id) + await this.client.groups.del({ id }) + } + + async getGroupMembers(groupId: string) { + const span = trace.getActiveSpan() + span?.setAttribute('keycloak.group.id', groupId) + const members = await this.client.groups.listMembers({ id: groupId }) + return members || [] + } + + @StartActiveSpan() + async createGroup(name: string) { + const span = trace.getActiveSpan() + span?.setAttribute('group.name', name) + this.logger.debug(`Creating Keycloak group: ${name}`) + const result = await this.client.groups.create({ name }) + return { ...result, name } as GroupRepresentation + } + + async addUserToGroup(userId: string, groupId: string) { + const span = trace.getActiveSpan() + span?.setAttribute('keycloak.group.id', groupId) + return this.client.users.addToGroup({ id: userId, groupId }) + } + + async removeUserFromGroup(userId: string, groupId: string) { + const span = trace.getActiveSpan() + span?.setAttribute('keycloak.group.id', groupId) + return this.client.users.delFromGroup({ id: userId, groupId }) + } + + async* getSubGroups(parentId: string) { + let first = 0 + while (true) { + const page = await this.client.groups.listSubGroups({ + parentId, + briefRepresentation: false, + max: SUBGROUPS_PAGINATE_QUERY_MAX, + first, + }) + if (page.length === 0) break + for (const subgroup of page) { + yield subgroup + } + if (page.length < SUBGROUPS_PAGINATE_QUERY_MAX) break + first += SUBGROUPS_PAGINATE_QUERY_MAX + } + } + + async getOrCreateGroupByPath(path: string) { + const span = trace.getActiveSpan() + span?.setAttribute('group.path.depth', path.split('/').filter(Boolean).length) + const existingGroup = await this.getGroupByPath(path) + if (existingGroup) return existingGroup + + const parts = path.split('/').filter(Boolean) + let parentId: string | undefined + let current: GroupRepresentationWith<'id' | 'name'> | undefined + + for (const name of parts.values()) { + if (current) { + if (!parentId) parentId = current.id + const next = z.object({ id: z.string(), name: z.string() }).safeParse(await this.getOrCreateSubGroupByName(parentId, name)) + if (next.success) current = next.data + } else { + const next = z.object({ id: z.string(), name: z.string() }).safeParse(await this.getGroupByName(name) ?? await this.createGroup(name)) + if (next.success) current = next.data + } + parentId = current?.id + } + + return { ...current, path } as GroupRepresentation + } + + @StartActiveSpan() + async getOrCreateSubGroupByName(parentId: string, name: string) { + const span = trace.getActiveSpan() + span?.setAttribute('group.name', name) + span?.setAttribute('parent.id', parentId) + for await (const subgroup of this.getSubGroups(parentId)) { + if (subgroup.name === name) { + return subgroup + } + } + this.logger.debug(`Creating SubGroup ${name} under parent ${parentId}`) + const createdGroup = await this.client.groups.createChildGroup({ id: parentId }, { name }) + return { id: createdGroup.id, name } satisfies GroupRepresentation + } + + async getOrCreateConsoleGroup(projectGroup: GroupRepresentationWith<'id'>) { + const span = trace.getActiveSpan() + span?.setAttribute('keycloak.group.id', projectGroup.id) + return this.getOrCreateSubGroupByName(projectGroup.id, CONSOLE_GROUP_NAME) + } + + async getOrCreateRoleGroup( + consoleGroup: GroupRepresentationWith<'id' | 'name'>, + oidcGroup: string, + ) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'keycloak.group.id': consoleGroup.id, + 'role.oidc_group.present': !!oidcGroup, + 'role.oidc_group.depth': oidcGroup.split('/').filter(Boolean).length, + }) + const parts = oidcGroup.split('/').filter(Boolean) + if (parts.length === 0) { + throw new Error(`Invalid oidcGroup for project role: "${oidcGroup}"`) + } + + let current = z.object({ + id: z.string(), + name: z.string(), + }).parse(await this.getOrCreateSubGroupByName(consoleGroup.id, parts[0])) + + for (const name of parts.slice(1)) { + current = z.object({ + id: z.string(), + name: z.string(), + }).parse(await this.getOrCreateSubGroupByName(current.id, name)) + } + + return { ...current, path: `/${consoleGroup.name}/${parts.join('/')}` } satisfies GroupRepresentation + } + + async getOrCreateEnvironmentGroups(consoleGroup: GroupRepresentationWith<'id'>, environment: ProjectWithDetails['environments'][number]) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'keycloak.group.id': consoleGroup.id, + 'environment.id': environment.id, + 'environment.name': environment.name, + }) + const envGroup = z.object({ + id: z.string(), + name: z.string(), + }).parse(await this.getOrCreateSubGroupByName(consoleGroup.id, environment.name)) + const [roGroup, rwGroup] = await Promise.all([ + this.getOrCreateSubGroupByName(envGroup.id, 'RO'), + this.getOrCreateSubGroupByName(envGroup.id, 'RW'), + ]) + return { roGroup, rwGroup } + } + + async onModuleInit() { + if (!this.config.keycloakRealm) { + this.logger.fatal('Keycloak realm not configured') + return + } + if (!this.config.keycloakAdmin || !this.config.keycloakAdminPassword) { + this.logger.fatal('Keycloak admin or admin password not configured') + return + } + this.client.setConfig({ realmName: this.config.keycloakRealm }) + await this.client.auth({ + clientId: 'admin-cli', + grantType: 'password', + username: this.config.keycloakAdmin, + password: this.config.keycloakAdminPassword, + }) + this.logger.log('Keycloak Admin Client authenticated') + } +} diff --git a/apps/server-nestjs/src/modules/keycloak/keycloak-datastore.service.ts b/apps/server-nestjs/src/modules/keycloak/keycloak-datastore.service.ts new file mode 100644 index 000000000..81673c526 --- /dev/null +++ b/apps/server-nestjs/src/modules/keycloak/keycloak-datastore.service.ts @@ -0,0 +1,50 @@ +import type { Prisma } from '@prisma/client' +import { Inject, Injectable } from '@nestjs/common' +import { PrismaService } from '../../cpin-module/infrastructure/database/prisma.service' + +export const projectSelect = { + id: true, + slug: true, + ownerId: true, + everyonePerms: true, + members: { + select: { + roleIds: true, + user: { + select: { + id: true, + email: true, + }, + }, + }, + }, + roles: { + select: { + id: true, + permissions: true, + oidcGroup: true, + type: true, + }, + }, + environments: { + select: { + id: true, + name: true, + }, + }, +} satisfies Prisma.ProjectSelect + +export type ProjectWithDetails = Prisma.ProjectGetPayload<{ + select: typeof projectSelect +}> + +@Injectable() +export class KeycloakDatastoreService { + constructor(@Inject(PrismaService) private readonly prisma: PrismaService) {} + + async getAllProjects(): Promise { + return this.prisma.project.findMany({ + select: projectSelect, + }) + } +} diff --git a/apps/server-nestjs/src/modules/keycloak/keycloak-health.service.ts b/apps/server-nestjs/src/modules/keycloak/keycloak-health.service.ts new file mode 100644 index 000000000..ff84eeb07 --- /dev/null +++ b/apps/server-nestjs/src/modules/keycloak/keycloak-health.service.ts @@ -0,0 +1,29 @@ +import { Inject, Injectable } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' + +@Injectable() +export class KeycloakHealthService { + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(HealthIndicatorService) private readonly healthIndicator: HealthIndicatorService, + ) {} + + async check(key: string) { + const indicator = this.healthIndicator.check(key) + const protocol = this.config.keycloakProtocol + const domain = this.config.keycloakDomain + const realm = this.config.keycloakRealm + if (!protocol || !domain || !realm) return indicator.down('Not configured') + + const baseUrl = `${protocol}://${domain}` + const url = new URL(`/realms/${encodeURIComponent(realm)}/.well-known/openid-configuration`, baseUrl).toString() + try { + const response = await fetch(url, { method: 'GET' }) + if (response.status < 500) return indicator.up({ httpStatus: response.status }) + return indicator.down({ httpStatus: response.status }) + } catch (error) { + return indicator.down(error instanceof Error ? error.message : String(error)) + } + } +} diff --git a/apps/server-nestjs/src/modules/keycloak/keycloak.constants.ts b/apps/server-nestjs/src/modules/keycloak/keycloak.constants.ts new file mode 100644 index 000000000..253fc146e --- /dev/null +++ b/apps/server-nestjs/src/modules/keycloak/keycloak.constants.ts @@ -0,0 +1,3 @@ +export const CONSOLE_GROUP_NAME = 'console' +export const GROUPS_PAGINATE_QUERY_MAX = 20 +export const SUBGROUPS_PAGINATE_QUERY_MAX = 20 diff --git a/apps/server-nestjs/src/modules/keycloak/keycloak.module.ts b/apps/server-nestjs/src/modules/keycloak/keycloak.module.ts new file mode 100644 index 000000000..d9d805809 --- /dev/null +++ b/apps/server-nestjs/src/modules/keycloak/keycloak.module.ts @@ -0,0 +1,30 @@ +import KcAdminClient from '@keycloak/keycloak-admin-client' +import { Module } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationService } from 'src/cpin-module/infrastructure/configuration/configuration.service' +import { ConfigurationModule } from '../../cpin-module/infrastructure/configuration/configuration.module' +import { InfrastructureModule } from '../../cpin-module/infrastructure/infrastructure.module' +import { KEYCLOAK_ADMIN_CLIENT, KeycloakClientService } from './keycloak-client.service' +import { KeycloakDatastoreService } from './keycloak-datastore.service' +import { KeycloakHealthService } from './keycloak-health.service' +import { KeycloakService } from './keycloak.service' + +@Module({ + imports: [ConfigurationModule, InfrastructureModule], + providers: [ + { + provide: KEYCLOAK_ADMIN_CLIENT, + inject: [ConfigurationService], + useFactory: (config: ConfigurationService) => new KcAdminClient({ + baseUrl: `${config.keycloakProtocol}://${config.keycloakDomain}`, + }), + }, + HealthIndicatorService, + KeycloakClientService, + KeycloakDatastoreService, + KeycloakHealthService, + KeycloakService, + ], + exports: [KeycloakClientService, KeycloakHealthService], +}) +export class KeycloakModule {} diff --git a/apps/server-nestjs/src/modules/keycloak/keycloak.service.spec.ts b/apps/server-nestjs/src/modules/keycloak/keycloak.service.spec.ts new file mode 100644 index 000000000..0f8e446c1 --- /dev/null +++ b/apps/server-nestjs/src/modules/keycloak/keycloak.service.spec.ts @@ -0,0 +1,381 @@ +import type { Mocked } from 'vitest' +import type { ProjectWithDetails } from './keycloak-datastore.service' +import { Test } from '@nestjs/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { + makeGroupRepresentation, + makeProjectEnvironment, + makeProjectMember, + makeProjectRole, + makeProjectUser, + makeProjectWithDetails, + makeUserRepresentation, +} from './keycloack-testing.utils' +import { KeycloakClientService } from './keycloak-client.service' +import { KeycloakDatastoreService } from './keycloak-datastore.service' +import { KeycloakService } from './keycloak.service' + +function createKeycloakControllerServiceTestingModule() { + return Test.createTestingModule({ + providers: [ + KeycloakService, + { + provide: KeycloakClientService, + useValue: { + getAllGroups: vi.fn().mockImplementation(async function* () {}), + deleteGroup: vi.fn().mockResolvedValue(undefined), + getOrCreateGroupByPath: vi.fn().mockResolvedValue({}), + getGroupMembers: vi.fn().mockResolvedValue([]), + addUserToGroup: vi.fn().mockResolvedValue(undefined), + removeUserFromGroup: vi.fn().mockResolvedValue(undefined), + getOrCreateSubGroupByName: vi.fn().mockResolvedValue({}), + getOrCreateRoleGroup: vi.fn().mockResolvedValue({}), + getSubGroups: vi.fn().mockImplementation(async function* () {}), + getOrCreateConsoleGroup: vi.fn().mockResolvedValue(makeGroupRepresentation({ id: 'console-group-id', name: 'console' })), + getOrCreateEnvironmentGroups: vi.fn().mockResolvedValue({ + roGroup: makeGroupRepresentation({ id: 'ro-id', name: 'RO' }), + rwGroup: makeGroupRepresentation({ id: 'rw-id', name: 'RW' }), + }), + } satisfies Partial, + }, + { + provide: KeycloakDatastoreService, + useValue: { + getAllProjects: vi.fn().mockResolvedValue([]), + } satisfies Partial, + }, + { + provide: ConfigurationService, + useValue: { + } satisfies Partial, + }, + ], + }) +} + +describe('keycloakService', () => { + let service: KeycloakService + let keycloak: Mocked + let keycloakDatastore: Mocked + + beforeEach(async () => { + vi.clearAllMocks() + const module = await createKeycloakControllerServiceTestingModule().compile() + service = module.get(KeycloakService) + keycloak = module.get(KeycloakClientService) + keycloakDatastore = module.get(KeycloakDatastoreService) + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) + + describe('reconcile', () => { + const mockProject: ProjectWithDetails = makeProjectWithDetails({ + id: 'project-id', + slug: 'test-project', + ownerId: 'owner-id', + everyonePerms: 0n, + }) + + it('should purge orphans', async () => { + keycloakDatastore.getAllProjects.mockResolvedValue([mockProject]) + + const projectGroup = makeGroupRepresentation({ id: 'group-id', name: 'test-project', subGroups: [] }) + const orphanGroup = makeGroupRepresentation({ + id: 'orphan-id', + name: 'orphan-project', + subGroups: [makeGroupRepresentation({ name: 'console' })], + }) + + keycloak.getAllGroups.mockImplementation(async function* () { + yield projectGroup + yield orphanGroup + }) + keycloak.getOrCreateGroupByPath.mockResolvedValue(projectGroup) + keycloak.getGroupMembers.mockResolvedValue([]) + keycloak.getOrCreateSubGroupByName.mockResolvedValue(makeGroupRepresentation({ id: 'console-id', name: 'console' })) + keycloak.getSubGroups.mockImplementation(async function* () { /* empty */ }) + await service.handleCron() + + expect(keycloakDatastore.getAllProjects).toHaveBeenCalled() + expect(keycloak.getAllGroups).toHaveBeenCalled() + expect(keycloak.getOrCreateGroupByPath).toHaveBeenCalledWith('/test-project') + expect(keycloak.deleteGroup).toHaveBeenCalledWith('orphan-id') + }) + + it('should sync project members', async () => { + const projectWithMembers = makeProjectWithDetails({ + ...mockProject, + members: [ + makeProjectMember({ + user: makeProjectUser({ id: 'user-1', email: 'user1@example.com' }), + roleIds: [], + }), + ], + }) + keycloakDatastore.getAllProjects.mockResolvedValue([projectWithMembers]) + + const projectGroup = makeGroupRepresentation({ id: 'group-id', name: 'test-project' }) + keycloak.getOrCreateGroupByPath.mockResolvedValue(projectGroup) + + // Current members: user-2 (extra), missing user-1 + keycloak.getGroupMembers.mockResolvedValue([ + makeUserRepresentation({ id: 'user-2', email: 'user2@example.com' }), + ]) + + keycloak.getOrCreateSubGroupByName.mockResolvedValue(makeGroupRepresentation({ id: 'console-id', name: 'console' })) + keycloak.getSubGroups.mockImplementation(async function* () { /* empty */ }) + + await service.handleCron() + + // Should add missing member + expect(keycloak.addUserToGroup).toHaveBeenCalledWith('user-1', 'group-id') + // Should add owner (missing in group members) + expect(keycloak.addUserToGroup).toHaveBeenCalledWith('owner-id', 'group-id') + // Should remove extra member + expect(keycloak.removeUserFromGroup).toHaveBeenCalledWith('user-2', 'group-id') + }) + + it('should sync OIDC role groups', async () => { + const roleWithOidc = makeProjectRole({ + id: 'role-oidc', + permissions: 0n, + oidcGroup: '/oidc-group', + type: 'managed', + }) + const projectWithRole = makeProjectWithDetails({ + ...mockProject, + members: [ + makeProjectMember({ + user: makeProjectUser({ id: 'user-1', email: 'user1@example.com' }), + roleIds: ['role-oidc'], + }), + ], + roles: [roleWithOidc], + }) + keycloakDatastore.getAllProjects.mockResolvedValue([projectWithRole]) + + const projectGroup = makeGroupRepresentation({ id: 'group-id', name: 'test-project' }) + const consoleGroup = { id: 'console-id', name: 'console' } + const roleGroup = makeGroupRepresentation({ id: 'role-group-id', name: 'oidc-group', path: '/console/oidc-group' }) + + keycloak.getOrCreateGroupByPath.mockImplementation((path) => { + if (path === '/test-project') return Promise.resolve(projectGroup) + return Promise.resolve({}) + }) + keycloak.getOrCreateConsoleGroup.mockResolvedValue(consoleGroup) + keycloak.getOrCreateRoleGroup.mockResolvedValue(roleGroup) + + // Project members: owner + keycloak.getGroupMembers.mockImplementation((groupId) => { + if (groupId === 'group-id') return Promise.resolve([makeUserRepresentation({ id: 'owner-id' })]) + // Role group members: user-2 (extra), missing user-1 + if (groupId === 'role-group-id') return Promise.resolve([makeUserRepresentation({ id: 'user-2', email: 'user2@example.com' })]) + return Promise.resolve([]) + }) + + keycloak.getSubGroups.mockImplementation(async function* () { /* empty */ }) + + await service.handleCron() + + // Should create/get role group + expect(keycloak.getOrCreateRoleGroup).toHaveBeenCalledWith(consoleGroup, '/oidc-group') + // Should add user-1 to role group + expect(keycloak.addUserToGroup).toHaveBeenCalledWith('user-1', 'role-group-id') + // Should remove user-2 from role group + expect(keycloak.removeUserFromGroup).toHaveBeenCalledWith('user-2', 'role-group-id') + }) + + it('should sync environment groups', async () => { + const projectWithEnv = makeProjectWithDetails({ + ...mockProject, + environments: [makeProjectEnvironment({ id: 'env-1', name: 'dev' })], + }) + keycloakDatastore.getAllProjects.mockResolvedValue([projectWithEnv]) + + const projectGroup = makeGroupRepresentation({ + id: 'group-id', + name: 'test-project', + subGroups: [makeGroupRepresentation({ name: 'console', id: 'console-id' })], + }) + keycloak.getOrCreateGroupByPath.mockResolvedValue(projectGroup) + keycloak.getGroupMembers.mockResolvedValue([]) + + // Mock console group retrieval + keycloak.getOrCreateConsoleGroup.mockResolvedValue(makeGroupRepresentation({ id: 'console-id', name: 'console' })) + keycloak.getOrCreateEnvironmentGroups.mockResolvedValue({ + roGroup: makeGroupRepresentation({ id: 'dev-ro-id', name: 'RO' }), + rwGroup: makeGroupRepresentation({ id: 'dev-rw-id', name: 'RW' }), + }) + keycloak.getOrCreateSubGroupByName.mockImplementation((_parentId, name) => { + if (name === 'console') return Promise.resolve(makeGroupRepresentation({ id: 'console-id', name: 'console' })) + if (name === 'dev') return Promise.resolve(makeGroupRepresentation({ id: 'dev-id', name: 'dev' })) + if (name === 'RO') return Promise.resolve(makeGroupRepresentation({ id: 'dev-ro-id', name: 'RO' })) + if (name === 'RW') return Promise.resolve(makeGroupRepresentation({ id: 'dev-rw-id', name: 'RW' })) + return Promise.resolve(makeGroupRepresentation({ id: 'new-id', name })) + }) + + // Mock existing environments: 'staging' (extra) + keycloak.getSubGroups.mockImplementation(async function* (parentId) { + if (parentId === 'console-id') { + yield makeGroupRepresentation({ id: 'staging-id', name: 'staging' }) + } + if (parentId === 'staging-id') { + yield makeGroupRepresentation({ name: 'RO' }) + yield makeGroupRepresentation({ name: 'RW' }) + } + }) + + await service.handleCron() + + // Should create dev group + expect(keycloak.getOrCreateConsoleGroup).toHaveBeenCalledWith({ id: 'group-id', name: 'test-project' }) + // Should create RO/RW groups + expect(keycloak.getOrCreateEnvironmentGroups).toHaveBeenCalledWith({ id: 'console-id', name: 'console' }, projectWithEnv.environments[0]) + // Should delete staging group + expect(keycloak.deleteGroup).toHaveBeenCalledWith('staging-id') + }) + + it('should sync environment permissions', async () => { + const userRo = makeUserRepresentation({ id: 'user-ro', email: 'ro@example.com' }) + const userRw = makeUserRepresentation({ id: 'user-rw', email: 'rw@example.com' }) + const userNone = makeUserRepresentation({ id: 'user-none', email: 'none@example.com' }) + + const projectWithEnvAndMembers = makeProjectWithDetails({ + ...mockProject, + members: [ + makeProjectMember({ + user: makeProjectUser({ id: userRo.id, email: userRo.email }), + roleIds: ['role-ro'], + }), + makeProjectMember({ + user: makeProjectUser({ id: userRw.id, email: userRw.email }), + roleIds: ['role-rw'], + }), + makeProjectMember({ + user: makeProjectUser({ id: userNone.id, email: userNone.email }), + roleIds: [], + }), + ], + roles: [ + makeProjectRole({ id: 'role-ro', permissions: 256n, oidcGroup: '', type: 'managed' }), + makeProjectRole({ id: 'role-rw', permissions: 8n, oidcGroup: '', type: 'managed' }), + ], + environments: [makeProjectEnvironment({ id: 'env-1', name: 'dev' })], + }) + keycloakDatastore.getAllProjects.mockResolvedValue([projectWithEnvAndMembers]) + + const projectGroup = makeGroupRepresentation({ + id: 'group-id', + name: 'test-project', + subGroups: [makeGroupRepresentation({ name: 'console', id: 'console-id' })], + }) + keycloak.getOrCreateGroupByPath.mockResolvedValue(projectGroup) + keycloak.getOrCreateConsoleGroup.mockResolvedValue(makeGroupRepresentation({ id: 'console-id', name: 'console' })) + keycloak.getOrCreateEnvironmentGroups.mockResolvedValue({ + roGroup: makeGroupRepresentation({ id: 'dev-ro-id', name: 'RO' }), + rwGroup: makeGroupRepresentation({ id: 'dev-rw-id', name: 'RW' }), + }) + + // Project group members (assume all are in project group for simplicity) + keycloak.getGroupMembers.mockImplementation((groupId) => { + if (groupId === 'group-id') return Promise.resolve([userRo, userRw, userNone]) + // RO group has userNone (extra), missing userRo + if (groupId === 'dev-ro-id') return Promise.resolve([userNone]) + // RW group has userNone (extra), missing userRw + if (groupId === 'dev-rw-id') return Promise.resolve([userNone]) + return Promise.resolve([]) + }) + + keycloak.getOrCreateSubGroupByName.mockImplementation((_parentId, name) => { + if (name === 'console') return Promise.resolve(makeGroupRepresentation({ id: 'console-id', name: 'console' })) + if (name === 'dev') return Promise.resolve(makeGroupRepresentation({ id: 'dev-id', name: 'dev' })) + if (name === 'RO') return Promise.resolve(makeGroupRepresentation({ id: 'dev-ro-id', name: 'RO' })) + if (name === 'RW') return Promise.resolve(makeGroupRepresentation({ id: 'dev-rw-id', name: 'RW' })) + return Promise.resolve(makeGroupRepresentation({ id: 'new-id', name })) + }) + + keycloak.getSubGroups.mockImplementation(async function* () { /* empty */ }) + + await service.handleCron() + + // Sync RO + expect(keycloak.addUserToGroup).toHaveBeenCalledWith('user-ro', 'dev-ro-id') + expect(keycloak.removeUserFromGroup).toHaveBeenCalledWith('user-none', 'dev-ro-id') + // Sync RW + expect(keycloak.addUserToGroup).toHaveBeenCalledWith('user-rw', 'dev-rw-id') + expect(keycloak.removeUserFromGroup).toHaveBeenCalledWith('user-none', 'dev-rw-id') + }) + + it('should handle different role types (managed, external, global)', async () => { + const roleManaged = makeProjectRole({ id: 'role-managed', permissions: 0n, oidcGroup: '/managed-group', type: 'managed' }) + const roleExternal = makeProjectRole({ id: 'role-external', permissions: 0n, oidcGroup: '/external-group', type: 'external' }) + const roleGlobal = makeProjectRole({ id: 'role-global', permissions: 0n, oidcGroup: '/global-group', type: 'global' }) + + const projectWithRoles = makeProjectWithDetails({ + ...mockProject, + members: [ + makeProjectMember({ + user: makeProjectUser({ id: 'user-1', email: 'user1@example.com' }), + roleIds: ['role-managed', 'role-external', 'role-global'], + }), + ], + roles: [roleManaged, roleExternal, roleGlobal], + }) + keycloakDatastore.getAllProjects.mockResolvedValue([projectWithRoles]) + + const projectGroup = makeGroupRepresentation({ id: 'group-id', name: 'test-project' }) + const consoleGroup = { id: 'console-id', name: 'console' } + const managedGroup = makeGroupRepresentation({ id: 'managed-id', name: 'managed-group' }) + const externalGroup = makeGroupRepresentation({ id: 'external-id', name: 'external-group' }) + const globalGroup = makeGroupRepresentation({ id: 'global-id', name: 'global-group' }) + + keycloak.getOrCreateGroupByPath.mockImplementation((path) => { + if (path === '/test-project') return Promise.resolve(projectGroup) + return Promise.resolve({}) + }) + keycloak.getOrCreateConsoleGroup.mockResolvedValue(consoleGroup) + keycloak.getOrCreateRoleGroup.mockImplementation((_consoleGroup, oidcGroup) => { + if (oidcGroup === '/managed-group') return Promise.resolve({ ...managedGroup, path: '/console/managed-group' }) + if (oidcGroup === '/external-group') return Promise.resolve({ ...externalGroup, path: '/console/external-group' }) + if (oidcGroup === '/global-group') return Promise.resolve({ ...globalGroup, path: '/console/global-group' }) + return Promise.resolve(makeGroupRepresentation({ id: 'new-id', name: oidcGroup, path: `/console/${oidcGroup}` })) + }) + + // Group members + keycloak.getGroupMembers.mockImplementation((groupId) => { + if (groupId === 'group-id') return Promise.resolve([makeUserRepresentation({ id: 'owner-id' })]) + + // Managed: has extra user-2, missing user-1 + if (groupId === 'managed-id') return Promise.resolve([makeUserRepresentation({ id: 'user-2' })]) + + // External: has extra user-2, missing user-1 + if (groupId === 'external-id') return Promise.resolve([makeUserRepresentation({ id: 'user-2' })]) + + // Global: create group if it doesn't exist but no members + if (groupId === 'global-id') return Promise.resolve([makeUserRepresentation({ id: 'user-2' })]) + + return Promise.resolve([]) + }) + + keycloak.getSubGroups.mockImplementation(async function* () { /* empty */ }) + + await service.handleCron() + + // Managed: should add user-1, remove user-2 + expect(keycloak.getOrCreateRoleGroup).toHaveBeenCalledWith(consoleGroup, '/managed-group') + expect(keycloak.addUserToGroup).toHaveBeenCalledWith('user-1', 'managed-id') + expect(keycloak.removeUserFromGroup).toHaveBeenCalledWith('user-2', 'managed-id') + + // External: should add user-1, NOT remove user-2 + expect(keycloak.getOrCreateRoleGroup).toHaveBeenCalledWith(consoleGroup, '/external-group') + expect(keycloak.addUserToGroup).toHaveBeenCalledWith('user-1', 'external-id') + expect(keycloak.removeUserFromGroup).not.toHaveBeenCalledWith('user-2', 'external-id') + + // Global: should sync group but no members + expect(keycloak.getOrCreateRoleGroup).toHaveBeenCalledWith(consoleGroup, '/global-group') + }) + }) +}) diff --git a/apps/server-nestjs/src/modules/keycloak/keycloak.service.ts b/apps/server-nestjs/src/modules/keycloak/keycloak.service.ts new file mode 100644 index 000000000..7973a42e6 --- /dev/null +++ b/apps/server-nestjs/src/modules/keycloak/keycloak.service.ts @@ -0,0 +1,580 @@ +import type UserRepresentation from '@keycloak/keycloak-admin-client/lib/defs/userRepresentation' +import type { GroupRepresentationWith } from './keycloak-client.service' +import type { ProjectWithDetails } from './keycloak-datastore.service' +import { getPermsByUserRoles, ProjectAuthorized, resourceListToDict } from '@cpn-console/shared' +import { Inject, Injectable, Logger } from '@nestjs/common' +import { OnEvent } from '@nestjs/event-emitter' +import { Cron, CronExpression } from '@nestjs/schedule' +import { trace } from '@opentelemetry/api' +import z from 'zod' +import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' +import { KeycloakClientService } from './keycloak-client.service' +import { KeycloakDatastoreService } from './keycloak-datastore.service' +import { CONSOLE_GROUP_NAME } from './keycloak.constants' + +@Injectable() +export class KeycloakService { + private readonly logger = new Logger(KeycloakService.name) + + constructor( + @Inject(KeycloakClientService) private readonly keycloak: KeycloakClientService, + @Inject(KeycloakDatastoreService) private readonly keycloakDatastore: KeycloakDatastoreService, + ) { + this.logger.log('KeycloakService initialized') + } + + @OnEvent('project.upsert') + @StartActiveSpan() + async handleUpsert(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project upsert for ${project.slug}`) + await this.ensureProjectGroups([project]) + } + + @OnEvent('project.delete') + @StartActiveSpan() + async handleDelete(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project delete for ${project.slug}`) + await this.purgeOrphanGroups([project]) + } + + @Cron(CronExpression.EVERY_HOUR) + @StartActiveSpan() + async handleCron() { + const span = trace.getActiveSpan() + this.logger.log('Starting periodic Keycloak reconciliation') + const projects = await this.keycloakDatastore.getAllProjects() + span?.setAttribute('keycloak.projects.count', projects.length) + this.logger.debug(`Reconciling ${projects.length} projects`) + await this.ensureProjectGroups(projects) + await this.purgeOrphanGroups(projects) + } + + @StartActiveSpan() + private async ensureProjectGroups(projects: ProjectWithDetails[]) { + const span = trace.getActiveSpan() + span?.setAttribute('keycloak.projects.count', projects.length) + await Promise.all(projects.map(project => this.ensureProjectGroup(project))) + } + + @StartActiveSpan() + private async ensureProjectGroup(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': project.slug, + 'project.members.count': project.members.length, + 'project.roles.count': project.roles.length, + 'project.environments.count': project.environments.length, + }) + + const projectGroup = z.object({ + id: z.string(), + name: z.string(), + }).parse(await this.keycloak.getOrCreateGroupByPath(`/${project.slug}`)) + + span?.setAttribute('keycloak.project_group.id', projectGroup.id) + + await Promise.all([ + this.ensureProjectGroupMembers(project, projectGroup), + this.ensureConsoleGroup(project, projectGroup), + ]) + } + + @StartActiveSpan() + private async ensureConsoleGroup(project: ProjectWithDetails, group: GroupRepresentationWith<'id'>) { + const span = trace.getActiveSpan() + span?.setAttribute('keycloak.console_group.id', group.id) + const consoleGroup = z.object({ + id: z.string(), + name: z.string(), + }).parse(await this.keycloak.getOrCreateConsoleGroup(group)) + await Promise.all([ + this.ensureRoleGroups(project, consoleGroup), + this.ensureEnvironmentGroups(project, consoleGroup), + this.purgeOrphanEnvironmentGroups(project, consoleGroup), + ]) + } + + @StartActiveSpan() + private async purgeOrphanGroups(projects: ProjectWithDetails[]) { + const span = trace.getActiveSpan() + const groups = map(this.keycloak.getAllGroups(), async (group) => { + return z.object({ + id: z.string(), + name: z.string(), + subGroups: z.array(z.object({ name: z.string() })), + }).parse(group) + }) + const projectSlugs = new Set(projects.map(p => p.slug)) + const promises: Promise[] = [] + let purgedCount = 0 + + for await (const group of groups) { + if (!projectSlugs.has(group.name)) { + if (this.isOwnedProjectGroup(group)) { + this.logger.log(`Deleting orphan Keycloak group: ${group.name}`) + purgedCount++ + promises.push(this.keycloak.deleteGroup(group.id)) + } + } + } + span?.setAttribute('purged.count', purgedCount) + await Promise.all(promises) + } + + private isOwnedProjectGroup(group: GroupRepresentationWith<'subGroups'>) { + // Safety check: Only delete if it looks like a project group (has 'console' subgroup) + // or if we can be sure it's not a system group. + // For now, we rely on the 'console' subgroup heuristic as it's created by us. + return !!group.subGroups.some(sg => sg.name === CONSOLE_GROUP_NAME) + } + + private async maybeAddUserToGroup(userId: string, groupId: string, groupName: string) { + try { + await this.keycloak.addUserToGroup(userId, groupId) + this.logger.log(`Added ${userId} to keycloak group ${groupName}`) + } catch (e) { + if (e.response?.status === 404) { + this.logger.warn(`User ${userId} not found in Keycloak, skipping addition to group ${groupName}`) + } else if (e.response?.status === 409) { + this.logger.debug(`User ${userId} is already a member of keycloak group ${groupName}`) + } else { + throw e + } + } + } + + private async maybeRemoveUserFromGroup(userId: string, groupId: string, groupName: string) { + try { + await this.keycloak.removeUserFromGroup(userId, groupId) + this.logger.log(`Removed ${userId} from keycloak group ${groupName}`) + } catch (e) { + if (e.response?.status === 404) { + this.logger.warn(`User ${userId} not found in Keycloak, skipping removal from group ${groupName}`) + } else { + throw e + } + } + } + + @StartActiveSpan() + private async ensureProjectGroupMembers( + project: ProjectWithDetails, + group: GroupRepresentationWith<'id' | 'name'>, + ) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + span?.setAttribute('keycloak.group.id', group.id) + + const groupMembers = await this.keycloak.getGroupMembers(group.id) + const desiredUserIds = new Set([project.ownerId, ...project.members.map(m => m.user.id)]) + + span?.setAttribute('keycloak.group.members.current', groupMembers.length) + span?.setAttribute('keycloak.group.members.desired', desiredUserIds.size) + + let addedCount = 0 + let removedCount = 0 + + await Promise.all([ + ...Array.from(desiredUserIds, async (userId) => { + if (!groupMembers.some(m => m.id === userId)) { + addedCount++ + await this.maybeAddUserToGroup(userId, group.id, group.name) + } + }), + ...groupMembers.map(async (member) => { + if (member.id && !desiredUserIds.has(member.id)) { + removedCount++ + await this.maybeRemoveUserFromGroup(member.id, group.id, group.name) + } + }), + ]) + + span?.setAttribute('keycloak.group.members.added', addedCount) + span?.setAttribute('keycloak.group.members.removed', removedCount) + } + + @StartActiveSpan() + private async ensureRoleGroups( + project: ProjectWithDetails, + group: GroupRepresentationWith<'id' | 'name'>, + ) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': project.slug, + 'keycloak.group.id': group.id, + 'project.roles.count': project.roles.length, + }) + + const rolesWithOidcGroup = project.roles.filter(r => !!r.oidcGroup).length + span?.setAttribute('project.roles.oidc_group.count', rolesWithOidcGroup) + + await Promise.all(project.roles.map(role => this.ensureRoleGroup(project, role, group))) + } + + @StartActiveSpan() + private async ensureRoleGroup( + project: ProjectWithDetails, + role: ProjectWithDetails['roles'][number], + group: GroupRepresentationWith<'id' | 'name'>, + ) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + span?.setAttribute('role.id', role.id) + span?.setAttribute('role.type', role.type) + span?.setAttribute('role.oidc_group.present', !!role.oidcGroup) + if (role.oidcGroup) { + span?.setAttribute('role.oidc_group.depth', role.oidcGroup.split('/').filter(Boolean).length) + } + + const roleGroup = await this.keycloak.getOrCreateRoleGroup(group, role.oidcGroup) + span?.setAttribute('keycloak.group.id', roleGroup.id) + span?.setAttribute('keycloak.group.path', roleGroup.path) + + const groupMembers = await this.keycloak.getGroupMembers(roleGroup.id) + span?.setAttribute('keycloak.group.members.current', groupMembers.length) + + switch (role.type) { + case 'managed': + await Promise.all([ + this.ensureRoleGroupMembers(project, role, roleGroup, groupMembers), + this.purgeOrphanRoleGroupMembers(project, role, roleGroup, groupMembers), + ]) + break + case 'external': + await this.ensureRoleGroupMembers(project, role, roleGroup, groupMembers) + break + case 'global': + await this.ensureRoleGroupMembers(project, role, roleGroup, groupMembers) + break + default: + throw new Error(`Unknown role type ${role.type}`) + } + } + + @StartActiveSpan() + private async ensureRoleGroupMembers( + project: ProjectWithDetails, + role: ProjectWithDetails['roles'][number], + group: GroupRepresentationWith<'id' | 'name' | 'path'>, + members: UserRepresentation[], + ) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + span?.setAttribute('role.id', role.id) + span?.setAttribute('role.type', role.type) + span?.setAttribute('keycloak.group.id', group.id) + span?.setAttribute('keycloak.group.members.current', members.length) + + const desiredMemberIds = project.members + .filter(m => m.roleIds.includes(role.id)) + .map(m => m.user.id) + + span?.setAttribute('keycloak.group.members.desired', desiredMemberIds.length) + + let addedCount = 0 + await Promise.all(project.members.map(async (member) => { + if (!members.some(m => m.id === member.user.id) && member.roleIds.includes(role.id)) { + addedCount++ + await this.maybeAddUserToGroup(member.user.id, group.id, group.name) + } + })) + + span?.setAttribute('keycloak.group.members.added', addedCount) + } + + @StartActiveSpan() + private async purgeOrphanRoleGroupMembers( + project: ProjectWithDetails, + role: ProjectWithDetails['roles'][number], + group: GroupRepresentationWith<'id' | 'name' | 'path'>, + members: UserRepresentation[], + ) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + span?.setAttribute('role.id', role.id) + span?.setAttribute('role.type', role.type) + span?.setAttribute('keycloak.group.id', group.id) + span?.setAttribute('keycloak.group.members.current', members.length) + + let removedCount = 0 + await Promise.all(members.map(async (member) => { + if (!isMember(project, member)) { + if (!member.id) { + throw new Error(`Failed to create or retrieve role group for ${role.oidcGroup}`) + } + removedCount++ + await this.maybeRemoveUserFromGroup(member.id, group.id, group.name) + } + })) + span?.setAttribute('keycloak.group.members.removed', removedCount) + } + + private async ensureEnvironmentGroups( + project: ProjectWithDetails, + group: GroupRepresentationWith<'id'>, + ) { + await Promise.all(project.environments.map(environment => + this.ensureEnvironmentGroup(project, environment, group))) + } + + @StartActiveSpan() + private async ensureEnvironmentGroup( + project: ProjectWithDetails, + environment: ProjectWithDetails['environments'][number], + group: GroupRepresentationWith<'id'>, + ) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': project.slug, + 'environment.id': environment.id, + 'environment.name': environment.name, + 'project.roles.count': project.roles.length, + }) + + const { roGroup, rwGroup } = z.object({ + roGroup: z.object({ + id: z.string(), + name: z.string(), + }), + rwGroup: z.object({ + id: z.string(), + name: z.string(), + }), + }).parse(await this.keycloak.getOrCreateEnvironmentGroups(group, environment)) + + span?.setAttribute('keycloak.env_group.ro.id', roGroup.id) + span?.setAttribute('keycloak.env_group.rw.id', rwGroup.id) + + const rolesById = resourceListToDict(project.roles) + + const [roMembers, rwMembers] = await Promise.all([ + this.keycloak.getGroupMembers(roGroup.id), + this.keycloak.getGroupMembers(rwGroup.id), + ]) + + span?.setAttribute('keycloak.env_group.ro.members.current', roMembers.length) + span?.setAttribute('keycloak.env_group.rw.members.current', rwMembers.length) + + await Promise.all([ + this.ensureEnvironmentGroupMembers( + project, + environment, + rolesById, + roGroup, + rwGroup, + roMembers, + rwMembers, + ), + this.purgeOrphanEnvironmentGroupMembers( + project, + environment, + roGroup, + rwGroup, + roMembers, + rwMembers, + ), + ]) + } + + @StartActiveSpan() + private async ensureEnvironmentGroupMembers( + project: ProjectWithDetails, + environment: ProjectWithDetails['environments'][number], + rolesById: Record, + roGroup: GroupRepresentationWith<'id'>, + rwGroup: GroupRepresentationWith<'id'>, + roMembers: UserRepresentation[], + rwMembers: UserRepresentation[], + ) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + span?.setAttribute('environment.id', environment.id) + span?.setAttribute('environment.name', environment.name) + span?.setAttribute('keycloak.env_group.ro.id', roGroup.id) + span?.setAttribute('keycloak.env_group.rw.id', rwGroup.id) + span?.setAttribute('keycloak.env_group.ro.members.current', roMembers.length) + span?.setAttribute('keycloak.env_group.rw.members.current', rwMembers.length) + + const projectUserIds = new Set([project.ownerId, ...project.members.map(m => m.user.id)]) + span?.setAttribute('project.users.count', projectUserIds.size) + + let roAdded = 0 + let roRemoved = 0 + let rwAdded = 0 + let rwRemoved = 0 + + await Promise.all(Array.from(projectUserIds, async (userId) => { + const perms = this.getUserPermissions(project, rolesById, userId) + + const isInRo = roMembers.some(m => m.id === userId) + if (perms.ro && !isInRo) { + roAdded++ + await this.maybeAddUserToGroup(userId, roGroup.id, `RO group for ${environment.name}`) + } else if (!perms.ro && isInRo) { + roRemoved++ + await this.maybeRemoveUserFromGroup(userId, roGroup.id, `RO group for ${environment.name}`) + } + + const isInRw = rwMembers.some(m => m.id === userId) + if (perms.rw && !isInRw) { + rwAdded++ + await this.maybeAddUserToGroup(userId, rwGroup.id, `RW group for ${environment.name}`) + } else if (!perms.rw && isInRw) { + rwRemoved++ + await this.maybeRemoveUserFromGroup(userId, rwGroup.id, `RW group for ${environment.name}`) + } + })) + + span?.setAttribute('keycloak.env_group.ro.members.added', roAdded) + span?.setAttribute('keycloak.env_group.ro.members.removed', roRemoved) + span?.setAttribute('keycloak.env_group.rw.members.added', rwAdded) + span?.setAttribute('keycloak.env_group.rw.members.removed', rwRemoved) + } + + @StartActiveSpan() + private async purgeOrphanEnvironmentGroupMembers( + project: ProjectWithDetails, + environment: ProjectWithDetails['environments'][number], + roGroup: GroupRepresentationWith<'id'>, + rwGroup: GroupRepresentationWith<'id'>, + roMembers: UserRepresentation[], + rwMembers: UserRepresentation[], + ) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + span?.setAttribute('environment.id', environment.id) + span?.setAttribute('environment.name', environment.name) + span?.setAttribute('keycloak.env_group.ro.id', roGroup.id) + span?.setAttribute('keycloak.env_group.rw.id', rwGroup.id) + span?.setAttribute('keycloak.env_group.ro.members.current', roMembers.length) + span?.setAttribute('keycloak.env_group.rw.members.current', rwMembers.length) + + const projectUserIds = new Set([project.ownerId, ...project.members.map(m => m.user.id)]) + span?.setAttribute('project.users.count', projectUserIds.size) + + let roRemoved = 0 + let rwRemoved = 0 + + await Promise.all([ + ...roMembers.map(async (member) => { + if (!member.id) { + throw new Error(`Failed to create or retrieve RO and RW groups for ${environment.name}`) + } + if (!projectUserIds.has(member.id)) { + roRemoved++ + await this.maybeRemoveUserFromGroup(member.id, roGroup.id, `RO group for ${environment.name}`) + } + }), + ...rwMembers.map(async (member) => { + if (!member.id) { + throw new Error(`Failed to create or retrieve RO and RW groups for ${environment.name}`) + } + if (!projectUserIds.has(member.id)) { + rwRemoved++ + await this.maybeRemoveUserFromGroup(member.id, rwGroup.id, `RW group for ${environment.name}`) + } + }), + ]) + + span?.setAttribute('keycloak.env_group.ro.members.removed', roRemoved) + span?.setAttribute('keycloak.env_group.rw.members.removed', rwRemoved) + } + + @StartActiveSpan() + private async purgeOrphanEnvironmentGroups( + project: ProjectWithDetails, + group: GroupRepresentationWith<'id' | 'name'>, + ) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + span?.setAttribute('keycloak.group.id', group.id) + span?.setAttribute('keycloak.group.name', group.name) + + const envGroups = map(this.keycloak.getSubGroups(group.id), envGroup => z.object({ + id: z.string(), + name: z.string(), + }).parse(envGroup)) + + const promises: Promise[] = [] + let orphanCount = 0 + + for await (const envGroup of envGroups) { + span?.setAttribute('keycloak.env_group.id', envGroup.id) + span?.setAttribute('keycloak.env_group.name', envGroup.name) + + const subGroups = await getAll(map(this.keycloak.getSubGroups(envGroup.id), subgroup => z.object({ + name: z.string(), + }).parse(subgroup))) + + if (this.isEnvironmentGroup(subGroups) && !this.isOwnedEnvironmentGroup(project, envGroup)) { + orphanCount++ + this.logger.log(`Deleting orphan environment group ${envGroup.name} for project ${project.slug}`) + promises.push( + this.keycloak.deleteGroup(envGroup.id) + .catch(e => this.logger.warn(`Failed to delete environment group ${envGroup.name} for project ${project.slug}`, e)), + ) + } + } + + span?.setAttribute('keycloak.env_groups.orphan.count', orphanCount) + await Promise.all(promises) + } + + private isEnvironmentGroup( + subGroups: GroupRepresentationWith<'name'>[], + ) { + return subGroups.some(subgroup => subgroup.name === 'RO' || subgroup.name === 'RW') + } + + private isOwnedEnvironmentGroup( + project: ProjectWithDetails, + group: GroupRepresentationWith<'name'>, + ) { + return project.environments.some(e => e.name === group.name) + } + + private getUserPermissions( + project: ProjectWithDetails, + rolesById: Record, + userId: string, + ) { + if (userId === project.ownerId) return { ro: true, rw: true } + const member = project.members.find(m => m.user.id === userId) + if (!member) return { ro: false, rw: false } + + const projectPermissions = getPermsByUserRoles(member.roleIds, rolesById, project.everyonePerms) + + return { + ro: ProjectAuthorized.ListEnvironments({ adminPermissions: 0n, projectPermissions }), + rw: ProjectAuthorized.ManageEnvironments({ adminPermissions: 0n, projectPermissions }), + } + } +} + +export function isMember(project: ProjectWithDetails, member: UserRepresentation) { + return project.members.some(m => m.user.id === member.id) || project.ownerId === member.id +} + +async function* map( + iterable: AsyncIterable, + mapper: (value: T, index: number) => U | Promise, +): AsyncIterable { + let index = 0 + for await (const value of iterable) { + yield await mapper(value, index++) + } +} + +async function getAll( + iterable: AsyncIterable, +): Promise { + const items: T[] = [] + for await (const item of iterable) { + items.push(item) + } + return items +} diff --git a/apps/server-nestjs/src/prisma/schema/project.prisma b/apps/server-nestjs/src/prisma/schema/project.prisma index e76048675..833845eee 100644 --- a/apps/server-nestjs/src/prisma/schema/project.prisma +++ b/apps/server-nestjs/src/prisma/schema/project.prisma @@ -65,6 +65,8 @@ model ProjectRole { permissions BigInt projectId String @db.Uuid position Int @db.SmallInt + oidcGroup String @default("") + type String @default("managed") project Project @relation(fields: [projectId], references: [id]) } diff --git a/apps/server-nestjs/src/utils/iterable.ts b/apps/server-nestjs/src/utils/iterable.ts new file mode 100644 index 000000000..04b0a70cc --- /dev/null +++ b/apps/server-nestjs/src/utils/iterable.ts @@ -0,0 +1,19 @@ +export async function* map( + iterable: AsyncIterable, + mapper: (value: T, index: number) => U | Promise, +): AsyncIterable { + let index = 0 + for await (const value of iterable) { + yield await mapper(value, index++) + } +} + +export async function getAll( + iterable: AsyncIterable, +): Promise { + const items: T[] = [] + for await (const item of iterable) { + items.push(item) + } + return items +} diff --git a/apps/server-nestjs/test/keycloak.e2e-spec.ts b/apps/server-nestjs/test/keycloak.e2e-spec.ts new file mode 100644 index 000000000..82af95f74 --- /dev/null +++ b/apps/server-nestjs/test/keycloak.e2e-spec.ts @@ -0,0 +1,531 @@ +import type KcAdminClient from '@keycloak/keycloak-admin-client' +import type { TestingModule } from '@nestjs/testing' +import { faker } from '@faker-js/faker' +import { Logger } from '@nestjs/common' +import { Test } from '@nestjs/testing' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import z from 'zod' +import { ConfigurationModule } from '../src/cpin-module/infrastructure/configuration/configuration.module' +import { PrismaService } from '../src/cpin-module/infrastructure/database/prisma.service' +import { InfrastructureModule } from '../src/cpin-module/infrastructure/infrastructure.module' +import { KEYCLOAK_ADMIN_CLIENT, KeycloakClientService } from '../src/modules/keycloak/keycloak-client.service' +import { projectSelect } from '../src/modules/keycloak/keycloak-datastore.service' +import { KeycloakModule } from '../src/modules/keycloak/keycloak.module' +import { KeycloakService } from '../src/modules/keycloak/keycloak.service' + +const canRunKeycloakE2E + = Boolean(process.env.E2E) + && Boolean(process.env.KEYCLOAK_DOMAIN) + && Boolean(process.env.KEYCLOAK_REALM) + && Boolean(process.env.KEYCLOAK_PROTOCOL) + && Boolean(process.env.KEYCLOAK_ADMIN) + && Boolean(process.env.KEYCLOAK_ADMIN_PASSWORD) + +const describeWithKeycloak = describe.runIf(canRunKeycloakE2E) + +describeWithKeycloak('KeycloakController (e2e)', () => { + let moduleRef: TestingModule + let keycloakController: KeycloakService + let keycloak: KeycloakClientService + let keycloakAdminClient: KcAdminClient + let prisma: PrismaService + + let ownerId: string + let testProjectId: string + let testProjectSlug: string + let testRoleName: string + let testRoleId: string + + beforeAll(async () => { + moduleRef = await Test.createTestingModule({ + imports: [KeycloakModule, ConfigurationModule, InfrastructureModule], + }).compile() + + await moduleRef.init() + + keycloakController = moduleRef.get(KeycloakService) + keycloak = moduleRef.get(KeycloakClientService) + keycloakAdminClient = moduleRef.get(KEYCLOAK_ADMIN_CLIENT) + prisma = moduleRef.get(PrismaService) + + ownerId = faker.string.uuid() + testProjectId = faker.string.uuid() + testProjectSlug = faker.helpers.slugify(`test-project-${faker.string.uuid()}`) + testRoleName = faker.helpers.slugify(`test-role-${faker.string.uuid()}`) + testRoleId = faker.string.uuid() + + const ownerEmail = faker.internet.email({ firstName: 'test-owner', provider: 'example.com' }) + + // Create owner in Keycloak + const createdUser = await keycloakAdminClient.users.create({ + id: ownerId, + username: `test-owner-${ownerId}`, + email: ownerEmail, + enabled: true, + firstName: 'Test', + lastName: 'Owner', + }) + if (createdUser.id) { + ownerId = createdUser.id + } + + // Create owner in DB + await prisma.user.create({ + data: { + id: ownerId, + email: ownerEmail, + firstName: 'Test', + lastName: 'Owner', + type: 'human', + }, + }) + }) + + afterAll(async () => { + try { + // Clean Keycloak + const group = await keycloak.getGroupByPath(`/${testProjectSlug}`) + if (group) { + await keycloak.deleteGroup(group.id!) + } + + // Clean owner user + if (ownerId) { + await keycloakAdminClient.users.del({ id: ownerId }).catch(() => {}) + if (prisma) { + await prisma.user.deleteMany({ where: { id: ownerId } }).catch(() => {}) + } + } + + // Clean DB + if (prisma) { + await prisma.projectMembers.deleteMany({ where: { projectId: testProjectId } }) + // Prisma cascade delete should handle roles/envs if configured correctly, but explicit delete is safer + // We catch errors to avoid failing cleanup if tables/relations are different + await prisma.project.deleteMany({ where: { id: testProjectId } }).catch(() => {}) + } + } catch (e: any) { + Logger.warn(`Cleanup failed: ${e.message}`) + } + + await moduleRef.close() + + vi.unstubAllEnvs() + }) + + it('should reconcile and create groups in Keycloak', async () => { + // Create Project in DB + await prisma.project.create({ + data: { + id: testProjectId, + slug: testProjectSlug, + name: testProjectSlug, + ownerId, + description: 'E2E Test Project', + hprodCpu: 0, + hprodGpu: 0, + hprodMemory: 0, + prodCpu: 0, + prodGpu: 0, + prodMemory: 0, + roles: { + create: { + id: testRoleId, + name: testRoleName, + oidcGroup: `/${testRoleName}`, + permissions: BigInt(0), + position: 0, + }, + }, + }, + }) + + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + // Act + await keycloakController.handleUpsert(project) + + // Assert + // Check main project group + const projectGroup = z.object({ + id: z.string(), + name: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}`)) + expect(projectGroup.name).toBe(testProjectSlug) + + const consoleGroup = z.object({ + id: z.string(), + name: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}/console`)) + expect(consoleGroup.name).toBe('console') + + // Check role group + const roleGroup = z.object({ + name: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}/console/${testRoleName}`)) + expect(roleGroup.name).toBe(testRoleName) + + // Check membership (owner should be added) + const members = await keycloak.getGroupMembers(projectGroup.id) + const isMember = members.some(m => m.id === ownerId) + expect(isMember).toBe(true) + }, 60000) + + it('should add member to project group when added in DB', async () => { + // Create another user in Keycloak and DB + const newUserId = faker.string.uuid() + const newUserEmail = `test-user-${newUserId}@example.com` + + // Create in Keycloak + const kcUser = await keycloakAdminClient.users.create({ + username: `test-user-${newUserId}`, + email: newUserEmail, + enabled: true, + firstName: 'Test', + lastName: 'User', + }) + + // Create in DB + await prisma.user.create({ + data: { + id: kcUser.id, + email: newUserEmail, + firstName: 'Test', + lastName: 'User', + type: 'human', + }, + }) + + // Add member to project in DB + await prisma.projectMembers.create({ + data: { + projectId: testProjectId, + userId: kcUser.id, + roleIds: [testRoleId], + }, + }) + + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + // Act + await keycloakController.handleUpsert(project) + + // Assert + const projectGroup = z.object({ + id: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}`)) + const members = await keycloak.getGroupMembers(projectGroup.id) + const isMember = members.some(m => m.id === kcUser.id) + expect(isMember).toBe(true) + + // Check role group membership + const roleGroup = z.object({ + id: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}/console/${testRoleName}`)) + const roleMembers = await keycloak.getGroupMembers(roleGroup.id) + const isRoleMember = roleMembers.some(m => m.id === kcUser.id) + expect(isRoleMember).toBe(true) + + // Cleanup user + await keycloakAdminClient.users.del({ id: kcUser.id }) + await prisma.projectMembers.deleteMany({ where: { userId: kcUser.id } }) + await prisma.user.delete({ where: { id: kcUser.id } }) + }, 60000) + + it('should remove member from project group when removed in DB', async () => { + const newUserId = faker.string.uuid() + const newUserEmail = `test-user-remove-${newUserId}@example.com` + + // Create in Keycloak + const kcUser = await keycloakAdminClient.users.create({ + username: `test-user-remove-${newUserId}`, + email: newUserEmail, + enabled: true, + firstName: 'Test', + lastName: 'UserRemove', + }) + + // Create in DB + await prisma.user.create({ + data: { + id: kcUser.id, + email: newUserEmail, + firstName: 'Test', + lastName: 'UserRemove', + type: 'human', + }, + }) + + // Add member to project in DB + await prisma.projectMembers.create({ + data: { + projectId: testProjectId, + userId: kcUser.id, + roleIds: [], // No roles + }, + }) + + let project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + // Sync add + await keycloakController.handleUpsert(project) + + // Verify added + const projectGroup = z.object({ + id: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}`)) + let members = await keycloak.getGroupMembers(projectGroup.id) + expect(members.some(m => m.id === kcUser.id)).toBe(true) + + // Remove from DB + await prisma.projectMembers.delete({ + where: { + projectId_userId: { + projectId: testProjectId, + userId: kcUser.id, + }, + }, + }) + + project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + // Sync remove + await keycloakController.handleUpsert(project) + + // Verify removed + members = await keycloak.getGroupMembers(projectGroup.id) + expect(members.some(m => m.id === kcUser.id)).toBe(false) + + // Cleanup + await keycloakAdminClient.users.del({ id: kcUser.id }) + await prisma.projectMembers.deleteMany({ where: { userId: kcUser.id } }) + await prisma.user.delete({ where: { id: kcUser.id } }) + }, 60000) + + it('should handle non-existent users gracefully', async () => { + // Add a member in DB that does not exist in Keycloak + const fakeUserId = faker.string.uuid() + + await prisma.user.create({ + data: { + id: fakeUserId, + email: `fake-${fakeUserId}@example.com`, + firstName: 'Fake', + lastName: 'User', + type: 'human', + }, + }) + + await prisma.projectMembers.create({ + data: { + projectId: testProjectId, + userId: fakeUserId, + roleIds: [], + }, + }) + + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + // Act - should not throw + await expect(keycloakController.handleUpsert(project)).resolves.not.toThrow() + + // Cleanup + await prisma.projectMembers.deleteMany({ where: { userId: fakeUserId } }) + await prisma.user.delete({ where: { id: fakeUserId } }) + }, 60000) + + it('should add user back to Keycloak group if missing but present in DB', async () => { + // Create user and add to project in DB + const newUserId = faker.string.uuid() + const newUserEmail = `test-user-sync-${newUserId}@example.com` + + const kcUser = await keycloakAdminClient.users.create({ + username: `test-user-sync-${newUserId}`, + email: newUserEmail, + enabled: true, + firstName: 'Test', + lastName: 'UserSync', + }) + + await prisma.user.create({ + data: { + id: kcUser.id, + email: newUserEmail, + firstName: 'Test', + lastName: 'UserSync', + type: 'human', + }, + }) + + await prisma.projectMembers.create({ + data: { + projectId: testProjectId, + userId: kcUser.id, + roleIds: [], + }, + }) + + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + // Sync to ensure they are added initially + await keycloakController.handleUpsert(project) + + const projectGroup = z.object({ + id: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}`)) + + // Manually remove user from Keycloak group + await keycloak.removeUserFromGroup(kcUser.id, projectGroup.id) + + // Verify removal + let members = await keycloak.getGroupMembers(projectGroup.id) + expect(members.some(m => m.id === kcUser.id)).toBe(false) + + // Sync again + await keycloakController.handleUpsert(project) + + // Verify added back + members = await keycloak.getGroupMembers(projectGroup.id) + expect(members.some(m => m.id === kcUser.id)).toBe(true) + + // Cleanup + await keycloakAdminClient.users.del({ id: kcUser.id }) + await prisma.projectMembers.deleteMany({ where: { userId: kcUser.id } }) + await prisma.user.delete({ where: { id: kcUser.id } }) + }, 60000) + + it('should remove user from Keycloak group if present but missing in DB', async () => { + // Create user + const newUserId = faker.string.uuid() + const newUserEmail = `test-user-orphan-${newUserId}@example.com` + + const kcUser = await keycloakAdminClient.users.create({ + username: `test-user-orphan-${newUserId}`, + email: newUserEmail, + enabled: true, + firstName: 'Test', + lastName: 'UserOrphan', + }) + + // We only need them in Keycloak for this test, but the controller checks if user is in DB to define "missing". + // Actually, `deleteExtraProjectMembers` iterates over Keycloak group members. + // So we don't strictly need the user in DB, but to be "clean" we should probably have them in DB but NOT in the project. + + await prisma.user.create({ + data: { + id: kcUser.id, + email: newUserEmail, + firstName: 'Test', + lastName: 'UserOrphan', + type: 'human', + }, + }) + + // Get project from DB (user is NOT a member) + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + // Sync to create group + await keycloakController.handleUpsert(project) + + // Manually add user to Keycloak group + const projectGroup = z.object({ + id: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}`)) + await keycloak.addUserToGroup(kcUser.id, projectGroup.id) + + // Verify added + let members = await keycloak.getGroupMembers(projectGroup.id) + expect(members.some(m => m.id === kcUser.id)).toBe(true) + + // Sync again to remove user + await keycloakController.handleUpsert(project) + + // Verify removed + members = await keycloak.getGroupMembers(projectGroup.id) + expect(members.some(m => m.id === kcUser.id)).toBe(false) + + // Cleanup + await keycloakAdminClient.users.del({ id: kcUser.id }) + await prisma.projectMembers.deleteMany({ where: { userId: kcUser.id } }) + await prisma.user.delete({ where: { id: kcUser.id } }) + }, 60000) + + it('should recreate project group if deleted in Keycloak', async () => { + // Ensure project exists and is synced + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + await keycloakController.handleUpsert(project) + + const projectGroup = z.object({ + id: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}`)) + + // Delete group in Keycloak + await keycloak.deleteGroup(projectGroup.id) + + // Verify deleted + const deletedProjectGroup = await keycloak.getGroupByPath(`/${testProjectSlug}`) + expect(deletedProjectGroup).toBeUndefined() + + // Sync + await keycloakController.handleUpsert(project) + + // Verify recreated + const recreatedProjectGroup = z.object({ + name: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}`)) + expect(recreatedProjectGroup?.name).toBe(testProjectSlug) + }, 60000) + + it('should recreate role group if deleted in Keycloak', async () => { + // Ensure project exists and is synced + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + await keycloakController.handleUpsert(project) + + const roleGroup = z.object({ + id: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}/console/${testRoleName}`)) + + // Delete role group in Keycloak + await keycloak.deleteGroup(roleGroup.id) + + // Verify deleted + const deletedRoleGroup = await keycloak.getGroupByPath(`/${testProjectSlug}/console/${testRoleName}`) + expect(deletedRoleGroup).toBeUndefined() + + // Sync + await keycloakController.handleUpsert(project) + + // Verify recreated + const recreatedRoleGroup = z.object({ + name: z.string(), + }).parse(await keycloak.getGroupByPath(`/${testProjectSlug}/console/${testRoleName}`)) + expect(recreatedRoleGroup?.name).toBe(testRoleName) + }, 60000) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ebe3ee2e0..6b3f3bc50 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -388,6 +388,9 @@ importers: '@gitbeaker/rest': specifier: ^40.6.0 version: 40.6.0 + '@keycloak/keycloak-admin-client': + specifier: ^24.0.0 + version: 24.0.5 '@kubernetes-models/argo-cd': specifier: ^2.7.2 version: 2.7.2 @@ -400,9 +403,15 @@ importers: '@nestjs/core': specifier: ^11.1.16 version: 11.1.16(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@11.1.16)(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/event-emitter': + specifier: ^3.0.1 + version: 3.0.1(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16) '@nestjs/platform-express': specifier: ^11.1.16 version: 11.1.16(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16) + '@nestjs/schedule': + specifier: ^6.1.1 + version: 6.1.1(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16) '@nestjs/terminus': specifier: ^11.1.1 version: 11.1.1(@grpc/grpc-js@1.14.3)(@grpc/proto-loader@0.8.0)(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16)(@prisma/client@6.19.2(prisma@6.19.2(magicast@0.3.5)(typescript@5.9.3))(typescript@5.9.3))(reflect-metadata@0.2.2)(rxjs@7.8.2) @@ -487,6 +496,12 @@ importers: vitest-mock-extended: specifier: ^2.0.2 version: 2.0.2(typescript@5.9.3)(vitest@2.1.9(@types/node@22.19.15)(jsdom@25.0.1)(msw@2.12.10(@types/node@22.19.15)(typescript@5.9.3))(terser@5.46.0)) + yaml: + specifier: ^2.8.2 + version: 2.8.2 + zod: + specifier: ^3.25.76 + version: 3.25.76 devDependencies: '@cpn-console/eslint-config': specifier: workspace:^ @@ -536,6 +551,9 @@ importers: globals: specifier: ^16.5.0 version: 16.5.0 + msw: + specifier: ^2.12.10 + version: 2.12.10(@types/node@22.19.15)(typescript@5.9.3) nodemon: specifier: ^3.1.14 version: 3.1.14 @@ -2628,6 +2646,10 @@ packages: '@js-sdsl/ordered-map@4.4.2': resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + '@keycloak/keycloak-admin-client@24.0.5': + resolution: {integrity: sha512-SXDVtQ3ov7GQbxXq51Uq8lzhwzQwNg6XiY50ZA9whuUe2t/0zPT4Zd/LcULcjweIjSNWWgfbDyN1E3yRSL8Qqw==} + engines: {node: '>=18'} + '@keycloak/keycloak-admin-client@26.5.5': resolution: {integrity: sha512-ZYP1Z+4qZ+vChNKWI+g1X08F2gCpZEWRlEMjwF03xet7bB5j5898nSJNFT1g6XIqVslHq15R8pt55fcULpRuvw==} engines: {node: '>=18'} @@ -2722,12 +2744,24 @@ packages: '@nestjs/websockets': optional: true + '@nestjs/event-emitter@3.0.1': + resolution: {integrity: sha512-0Ln/x+7xkU6AJFOcQI9tIhUMXVF7D5itiaQGOyJbXtlAfAIt8gzDdJm+Im7cFzKoWkiW5nCXCPh6GSvdQd/3Dw==} + peerDependencies: + '@nestjs/common': ^10.0.0 || ^11.0.0 + '@nestjs/core': ^10.0.0 || ^11.0.0 + '@nestjs/platform-express@11.1.16': resolution: {integrity: sha512-IOegr5+ZfUiMKgk+garsSU4MOkPRhm46e6w8Bp1GcO4vCdl9Piz6FlWAzKVfa/U3Hn/DdzSVJOW3TWcQQFdBDw==} peerDependencies: '@nestjs/common': ^11.0.0 '@nestjs/core': ^11.0.0 + '@nestjs/schedule@6.1.1': + resolution: {integrity: sha512-kQl1RRgi02GJ0uaUGCrXHCcwISsCsJDciCKe38ykJZgnAeeoeVWs8luWtBo4AqAAXm4nS5K8RlV0smHUJ4+2FA==} + peerDependencies: + '@nestjs/common': ^10.0.0 || ^11.0.0 + '@nestjs/core': ^10.0.0 || ^11.0.0 + '@nestjs/schematics@11.0.9': resolution: {integrity: sha512-0NfPbPlEaGwIT8/TCThxLzrlz3yzDNkfRNpbL7FiplKq3w4qXpJg0JYwqgMEJnLQZm3L/L/5XjoyfJHUO3qX9g==} peerDependencies: @@ -3900,6 +3934,9 @@ packages: '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/luxon@3.7.1': + resolution: {integrity: sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==} + '@types/mdast@4.0.4': resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} @@ -4917,6 +4954,10 @@ packages: cron-validator@1.4.0: resolution: {integrity: sha512-wGcJ9FCy65iaU6egSH8b5dZYJF7GU/3Jh06wzaT9lsa5dbqExjljmu+0cJ8cpKn+vUyZa/EM4WAxeLR6SypJXw==} + cron@4.4.0: + resolution: {integrity: sha512-fkdfq+b+AHI4cKdhZlppHveI/mgz2qpiYxcm+t5E5TsxX7QrLS1VE0+7GENEk9z0EeGPcpSciGv6ez24duWhwQ==} + engines: {node: '>=18.x'} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -5487,6 +5528,9 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + eventemitter2@6.4.9: + resolution: {integrity: sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==} + eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} @@ -6526,6 +6570,10 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + luxon@3.7.2: + resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==} + engines: {node: '>=12'} + magic-regexp@0.10.0: resolution: {integrity: sha512-Uly1Bu4lO1hwHUW0CQeSWuRtzCMNO00CmXtS8N6fyvB3B979GOEEeAkiTUDsmbYLAbvpUS/Kt5c4ibosAzVyVg==} @@ -8401,6 +8449,10 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + url-join@5.0.0: + resolution: {integrity: sha512-n2huDr9h9yzd6exQVnH/jU5mr+Pfx08LRXXZhkLLetAMESRj+anQsTAh940iMrIetKAmry9coFuZQ2jY8/p3WA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + url-template@3.1.1: resolution: {integrity: sha512-4oszoaEKE/mQOtAmdMWqIRHmkxWkUZMnXFnjQ5i01CuRSK3uluxcH1MRVVVWmhlnzT1SCDfKxxficm2G37qzCA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -10637,6 +10689,12 @@ snapshots: '@js-sdsl/ordered-map@4.4.2': {} + '@keycloak/keycloak-admin-client@24.0.5': + dependencies: + camelize-ts: 3.0.0 + url-join: 5.0.0 + url-template: 3.1.1 + '@keycloak/keycloak-admin-client@26.5.5': dependencies: camelize-ts: 3.0.0 @@ -10692,7 +10750,6 @@ snapshots: is-node-process: 1.2.0 outvariant: 1.4.3 strict-event-emitter: 0.5.1 - optional: true '@napi-rs/wasm-runtime@1.1.1': dependencies: @@ -10761,6 +10818,12 @@ snapshots: optionalDependencies: '@nestjs/platform-express': 11.1.16(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16) + '@nestjs/event-emitter@3.0.1(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16)': + dependencies: + '@nestjs/common': 11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/core': 11.1.16(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@11.1.16)(reflect-metadata@0.2.2)(rxjs@7.8.2) + eventemitter2: 6.4.9 + '@nestjs/platform-express@11.1.16(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16)': dependencies: '@nestjs/common': 11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2) @@ -10773,6 +10836,12 @@ snapshots: transitivePeerDependencies: - supports-color + '@nestjs/schedule@6.1.1(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16)': + dependencies: + '@nestjs/common': 11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/core': 11.1.16(@nestjs/common@11.1.16(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@11.1.16)(reflect-metadata@0.2.2)(rxjs@7.8.2) + cron: 4.4.0 + '@nestjs/schematics@11.0.9(chokidar@4.0.3)(typescript@5.9.3)': dependencies: '@angular-devkit/core': 19.2.17(chokidar@4.0.3) @@ -10823,17 +10892,14 @@ snapshots: dependencies: consola: 3.4.2 - '@open-draft/deferred-promise@2.2.0': - optional: true + '@open-draft/deferred-promise@2.2.0': {} '@open-draft/logger@0.3.0': dependencies: is-node-process: 1.2.0 outvariant: 1.4.3 - optional: true - '@open-draft/until@2.1.0': - optional: true + '@open-draft/until@2.1.0': {} '@opentelemetry/api-logs@0.212.0': dependencies: @@ -12076,6 +12142,8 @@ snapshots: '@types/json-schema@7.0.15': {} + '@types/luxon@3.7.1': {} + '@types/mdast@4.0.4': dependencies: '@types/unist': 3.0.3 @@ -12129,8 +12197,7 @@ snapshots: '@types/http-errors': 2.0.5 '@types/node': 22.19.15 - '@types/statuses@2.0.6': - optional: true + '@types/statuses@2.0.6': {} '@types/superagent@8.1.9': dependencies: @@ -13337,8 +13404,7 @@ snapshots: cookie@0.7.2: {} - cookie@1.1.1: - optional: true + cookie@1.1.1: {} cookiejar@2.1.4: {} @@ -13382,6 +13448,11 @@ snapshots: cron-validator@1.4.0: {} + cron@4.4.0: + dependencies: + '@types/luxon': 3.7.1 + luxon: 3.7.2 + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -14115,6 +14186,8 @@ snapshots: etag@1.8.1: {} + eventemitter2@6.4.9: {} + eventemitter3@5.0.4: {} events@3.3.0: {} @@ -14620,8 +14693,7 @@ snapshots: jwk-to-pem: 2.0.7 jws: 4.0.1 - graphql@16.13.1: - optional: true + graphql@16.13.1: {} gzip-size@6.0.0: dependencies: @@ -14665,8 +14737,7 @@ snapshots: he@1.2.0: {} - headers-polyfill@4.0.3: - optional: true + headers-polyfill@4.0.3: {} helmet@7.2.0: {} @@ -14885,8 +14956,7 @@ snapshots: is-negative-zero@2.0.3: {} - is-node-process@1.2.0: - optional: true + is-node-process@1.2.0: {} is-number-object@1.1.1: dependencies: @@ -15256,6 +15326,8 @@ snapshots: dependencies: yallist: 3.1.1 + luxon@3.7.2: {} + magic-regexp@0.10.0: dependencies: estree-walker: 3.0.3 @@ -15726,7 +15798,6 @@ snapshots: typescript: 5.9.3 transitivePeerDependencies: - '@types/node' - optional: true msw@2.12.10(@types/node@24.12.0)(typescript@5.7.2): dependencies: @@ -15983,8 +16054,7 @@ snapshots: strip-ansi: 6.0.1 wcwidth: 1.0.1 - outvariant@1.4.3: - optional: true + outvariant@1.4.3: {} own-keys@1.0.1: dependencies: @@ -16081,8 +16151,7 @@ snapshots: lru-cache: 11.2.6 minipass: 7.1.3 - path-to-regexp@6.3.0: - optional: true + path-to-regexp@6.3.0: {} path-to-regexp@8.3.0: {} @@ -16503,8 +16572,7 @@ snapshots: ret@0.4.3: {} - rettime@0.10.1: - optional: true + rettime@0.10.1: {} reusify@1.1.0: {} @@ -16867,8 +16935,7 @@ snapshots: streamsearch@1.1.0: {} - strict-event-emitter@0.5.1: - optional: true + strict-event-emitter@0.5.1: {} string-argv@0.3.2: {} @@ -17212,8 +17279,7 @@ snapshots: tldts-core@6.1.86: {} - tldts-core@7.0.25: - optional: true + tldts-core@7.0.25: {} tldts@6.1.86: dependencies: @@ -17222,7 +17288,6 @@ snapshots: tldts@7.0.25: dependencies: tldts-core: 7.0.25 - optional: true to-regex-range@5.0.1: dependencies: @@ -17258,7 +17323,6 @@ snapshots: tough-cookie@6.0.0: dependencies: tldts: 7.0.25 - optional: true tr46@0.0.3: {} @@ -17598,8 +17662,7 @@ snapshots: picomatch: 4.0.3 webpack-virtual-modules: 0.6.2 - until-async@3.0.2: - optional: true + until-async@3.0.2: {} upath@1.2.0: {} @@ -17613,6 +17676,8 @@ snapshots: dependencies: punycode: 2.3.1 + url-join@5.0.0: {} + url-template@3.1.1: {} util-deprecate@1.0.2: {} From 3f514d143e0836aacf56a80471616a85ee261e29 Mon Sep 17 00:00:00 2001 From: William Phetsinorath Date: Wed, 25 Mar 2026 14:16:19 +0100 Subject: [PATCH 2/4] refactor(argocd): migrate ArgoCD to NestJS Signed-off-by: William Phetsinorath --- apps/server-nestjs/package.json | 1 + .../configuration/configuration.service.ts | 50 +- .../argocd/argocd-datastore.service.ts | 63 ++ .../modules/argocd/argocd-health.service.ts | 25 + .../src/modules/argocd/argocd.module.ts | 16 + .../src/modules/argocd/argocd.service.spec.ts | 294 ++++++++++ .../src/modules/argocd/argocd.service.ts | 446 +++++++++++++++ .../src/modules/gitlab/files/.gitlab-ci.yml | 22 + .../src/modules/gitlab/files/mirror.sh | 83 +++ .../gitlab/gitlab-client.service.spec.ts | 541 ++++++++++++++++++ .../modules/gitlab/gitlab-client.service.ts | 345 +++++++++++ .../gitlab/gitlab-datastore.service.ts | 103 ++++ .../modules/gitlab/gitlab-health.service.ts | 25 + .../modules/gitlab/gitlab-testing.utils.ts | 335 +++++++++++ .../src/modules/gitlab/gitlab.constants.ts | 12 + .../src/modules/gitlab/gitlab.module.ts | 32 ++ .../src/modules/gitlab/gitlab.service.spec.ts | 346 +++++++++++ .../src/modules/gitlab/gitlab.service.ts | 478 ++++++++++++++++ .../src/modules/healthz/healthz.controller.ts | 9 + .../src/modules/healthz/healthz.module.ts | 6 + .../vault/vault-client.service.spec.ts | 119 ++++ .../src/modules/vault/vault-client.service.ts | 478 ++++++++++++++++ .../modules/vault/vault-datastore.service.ts | 58 ++ .../src/modules/vault/vault-health.service.ts | 25 + .../src/modules/vault/vault.module.ts | 21 + .../src/modules/vault/vault.service.spec.ts | 94 +++ .../src/modules/vault/vault.service.ts | 355 ++++++++++++ .../src/modules/vault/vault.utils.ts | 17 + .../src/prisma/schema/project.prisma | 1 + apps/server-nestjs/src/utils/iterable.ts | 7 + apps/server-nestjs/test/argocd.e2e-spec.ts | 308 ++++++++++ apps/server-nestjs/test/gitlab.e2e-spec.ts | 223 ++++++++ apps/server-nestjs/test/vault.e2e-spec.ts | 102 ++++ pnpm-lock.yaml | 3 + 34 files changed, 5042 insertions(+), 1 deletion(-) create mode 100644 apps/server-nestjs/src/modules/argocd/argocd-datastore.service.ts create mode 100644 apps/server-nestjs/src/modules/argocd/argocd-health.service.ts create mode 100644 apps/server-nestjs/src/modules/argocd/argocd.module.ts create mode 100644 apps/server-nestjs/src/modules/argocd/argocd.service.spec.ts create mode 100644 apps/server-nestjs/src/modules/argocd/argocd.service.ts create mode 100644 apps/server-nestjs/src/modules/gitlab/files/.gitlab-ci.yml create mode 100644 apps/server-nestjs/src/modules/gitlab/files/mirror.sh create mode 100644 apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts create mode 100644 apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts create mode 100644 apps/server-nestjs/src/modules/gitlab/gitlab-datastore.service.ts create mode 100644 apps/server-nestjs/src/modules/gitlab/gitlab-health.service.ts create mode 100644 apps/server-nestjs/src/modules/gitlab/gitlab-testing.utils.ts create mode 100644 apps/server-nestjs/src/modules/gitlab/gitlab.constants.ts create mode 100644 apps/server-nestjs/src/modules/gitlab/gitlab.module.ts create mode 100644 apps/server-nestjs/src/modules/gitlab/gitlab.service.spec.ts create mode 100644 apps/server-nestjs/src/modules/gitlab/gitlab.service.ts create mode 100644 apps/server-nestjs/src/modules/vault/vault-client.service.spec.ts create mode 100644 apps/server-nestjs/src/modules/vault/vault-client.service.ts create mode 100644 apps/server-nestjs/src/modules/vault/vault-datastore.service.ts create mode 100644 apps/server-nestjs/src/modules/vault/vault-health.service.ts create mode 100644 apps/server-nestjs/src/modules/vault/vault.module.ts create mode 100644 apps/server-nestjs/src/modules/vault/vault.service.spec.ts create mode 100644 apps/server-nestjs/src/modules/vault/vault.service.ts create mode 100644 apps/server-nestjs/src/modules/vault/vault.utils.ts create mode 100644 apps/server-nestjs/test/argocd.e2e-spec.ts create mode 100644 apps/server-nestjs/test/gitlab.e2e-spec.ts create mode 100644 apps/server-nestjs/test/vault.e2e-spec.ts diff --git a/apps/server-nestjs/package.json b/apps/server-nestjs/package.json index 71057c25a..331fef094 100644 --- a/apps/server-nestjs/package.json +++ b/apps/server-nestjs/package.json @@ -43,6 +43,7 @@ "@fastify/swagger": "^8.15.0", "@fastify/swagger-ui": "^4.2.0", "@gitbeaker/core": "^40.6.0", + "@gitbeaker/requester-utils": "^40.6.0", "@gitbeaker/rest": "^40.6.0", "@keycloak/keycloak-admin-client": "^24.0.0", "@kubernetes-models/argo-cd": "^2.7.2", diff --git a/apps/server-nestjs/src/cpin-module/infrastructure/configuration/configuration.service.ts b/apps/server-nestjs/src/cpin-module/infrastructure/configuration/configuration.service.ts index f049b3764..750f450d7 100644 --- a/apps/server-nestjs/src/cpin-module/infrastructure/configuration/configuration.service.ts +++ b/apps/server-nestjs/src/cpin-module/infrastructure/configuration/configuration.service.ts @@ -36,10 +36,58 @@ export class ConfigurationService { = process.env.CONTACT_EMAIL ?? 'cloudpinative-relations@interieur.gouv.fr' + // argocd + argoNamespace = process.env.ARGO_NAMESPACE ?? 'argocd' + argocdUrl = process.env.ARGOCD_URL + argocdExtraRepositories = process.env.ARGOCD_EXTRA_REPOSITORIES + + // dso + dsoEnvChartVersion = process.env.DSO_ENV_CHART_VERSION ?? 'dso-env-1.6.0' + dsoNsChartVersion = process.env.DSO_NS_CHART_VERSION ?? 'dso-ns-1.1.5' + // plugins mockPlugins = process.env.MOCK_PLUGINS === 'true' - projectRootDir = process.env.PROJECTS_ROOT_DIR + projectRootPath = process.env.PROJECTS_ROOT_DIR pluginsDir = process.env.PLUGINS_DIR ?? '/plugins' + + // gitlab + gitlabToken = process.env.GITLAB_TOKEN + gitlabUrl = process.env.GITLAB_URL + gitlabInternalUrl = process.env.GITLAB_INTERNAL_URL + ? process.env.GITLAB_INTERNAL_URL + : process.env.GITLAB_URL + + gitlabMirrorTokenExpirationDays = Number(process.env.GITLAB_MIRROR_TOKEN_EXPIRATION_DAYS ?? 180) + gitlabMirrorTokenRotationThresholdDays = Number(process.env.GITLAB_MIRROR_TOKEN_ROTATION_THRESHOLD_DAYS ?? 90) + + // vault + vaultToken = process.env.VAULT_TOKEN + vaultUrl = process.env.VAULT_URL + vaultInternalUrl = process.env.VAULT_INTERNAL_URL + ? process.env.VAULT_INTERNAL_URL + : process.env.VAULT_URL + + vaultKvName = process.env.VAULT_KV_NAME ?? 'forge-dso' + + // registry (harbor) + harborUrl = process.env.HARBOR_URL + harborInternalUrl = process.env.HARBOR_INTERNAL_URL ?? process.env.HARBOR_URL + harborAdmin = process.env.HARBOR_ADMIN + harborAdminPassword = process.env.HARBOR_ADMIN_PASSWORD + harborRuleTemplate = process.env.HARBOR_RULE_TEMPLATE + harborRuleCount = process.env.HARBOR_RULE_COUNT + harborRetentionCron = process.env.HARBOR_RETENTION_CRON + + // nexus + nexusUrl = process.env.NEXUS_URL + nexusInternalUrl = process.env.NEXUS_INTERNAL_URL ?? process.env.NEXUS_URL + nexusAdmin = process.env.NEXUS_ADMIN + nexusAdminPassword = process.env.NEXUS_ADMIN_PASSWORD + nexusSecretExposedUrl + = process.env.NEXUS__SECRET_EXPOSE_INTERNAL_URL === 'true' + ? (process.env.NEXUS_INTERNAL_URL ?? process.env.NEXUS_URL) + : process.env.NEXUS_URL + NODE_ENV = process.env.NODE_ENV === 'test' ? 'test' diff --git a/apps/server-nestjs/src/modules/argocd/argocd-datastore.service.ts b/apps/server-nestjs/src/modules/argocd/argocd-datastore.service.ts new file mode 100644 index 000000000..ffe9d6a93 --- /dev/null +++ b/apps/server-nestjs/src/modules/argocd/argocd-datastore.service.ts @@ -0,0 +1,63 @@ +import type { Prisma } from '@prisma/client' +import { Inject, Injectable } from '@nestjs/common' +import { PrismaService } from '../../cpin-module/infrastructure/database/prisma.service' + +export const projectSelect = { + id: true, + name: true, + slug: true, + plugins: { + select: { + pluginName: true, + key: true, + value: true, + }, + }, + repositories: { + select: { + id: true, + internalRepoName: true, + isInfra: true, + helmValuesFiles: true, + deployRevision: true, + deployPath: true, + }, + }, + environments: { + select: { + id: true, + name: true, + clusterId: true, + cpu: true, + gpu: true, + memory: true, + autosync: true, + }, + }, + clusters: { + select: { + id: true, + label: true, + zone: { + select: { + slug: true, + }, + }, + }, + }, +} satisfies Prisma.ProjectSelect + +export type ProjectWithDetails = Prisma.ProjectGetPayload<{ + select: typeof projectSelect +}> + +@Injectable() +export class ArgoCDDatastoreService { + constructor(@Inject(PrismaService) private readonly prisma: PrismaService) {} + + async getAllProjects(): Promise { + return this.prisma.project.findMany({ + select: projectSelect, + }) + } +} diff --git a/apps/server-nestjs/src/modules/argocd/argocd-health.service.ts b/apps/server-nestjs/src/modules/argocd/argocd-health.service.ts new file mode 100644 index 000000000..cccc0d28e --- /dev/null +++ b/apps/server-nestjs/src/modules/argocd/argocd-health.service.ts @@ -0,0 +1,25 @@ +import { Inject, Injectable } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' + +@Injectable() +export class ArgoCDHealthService { + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(HealthIndicatorService) private readonly healthIndicator: HealthIndicatorService, + ) {} + + async check(key: string) { + const indicator = this.healthIndicator.check(key) + if (!this.config.argocdUrl) return indicator.down('Not configured') + + const url = new URL('/api/version', this.config.argocdUrl).toString() + try { + const response = await fetch(url, { method: 'GET' }) + if (response.status < 500) return indicator.up({ httpStatus: response.status }) + return indicator.down({ httpStatus: response.status }) + } catch (error) { + return indicator.down(error instanceof Error ? error.message : String(error)) + } + } +} diff --git a/apps/server-nestjs/src/modules/argocd/argocd.module.ts b/apps/server-nestjs/src/modules/argocd/argocd.module.ts new file mode 100644 index 000000000..b554561f9 --- /dev/null +++ b/apps/server-nestjs/src/modules/argocd/argocd.module.ts @@ -0,0 +1,16 @@ +import { Module } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationModule } from '../../cpin-module/infrastructure/configuration/configuration.module' +import { InfrastructureModule } from '../../cpin-module/infrastructure/infrastructure.module' +import { GitlabModule } from '../gitlab/gitlab.module' +import { VaultModule } from '../vault/vault.module' +import { ArgoCDDatastoreService } from './argocd-datastore.service' +import { ArgoCDHealthService } from './argocd-health.service' +import { ArgoCDService } from './argocd.service' + +@Module({ + imports: [ConfigurationModule, InfrastructureModule, GitlabModule, VaultModule], + providers: [HealthIndicatorService, ArgoCDHealthService, ArgoCDService, ArgoCDDatastoreService], + exports: [ArgoCDHealthService], +}) +export class ArgoCDModule {} diff --git a/apps/server-nestjs/src/modules/argocd/argocd.service.spec.ts b/apps/server-nestjs/src/modules/argocd/argocd.service.spec.ts new file mode 100644 index 000000000..413ccd8b8 --- /dev/null +++ b/apps/server-nestjs/src/modules/argocd/argocd.service.spec.ts @@ -0,0 +1,294 @@ +import type { TestingModule } from '@nestjs/testing' +import type { Mocked } from 'vitest' +import type { ProjectWithDetails } from './argocd-datastore.service' +import { generateNamespaceName } from '@cpn-console/shared' +import { Test } from '@nestjs/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { stringify } from 'yaml' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { GitlabClientService } from '../gitlab/gitlab-client.service' +import { makeProjectSchema } from '../gitlab/gitlab-testing.utils' +import { VaultClientService } from '../vault/vault-client.service' +import { ArgoCDDatastoreService } from './argocd-datastore.service' +import { ArgoCDService } from './argocd.service' + +function createArgoCDControllerServiceTestingModule() { + return Test.createTestingModule({ + providers: [ + ArgoCDService, + { + provide: ArgoCDDatastoreService, + useValue: { + getAllProjects: vi.fn(), + } satisfies Partial, + }, + { + provide: ConfigurationService, + useValue: { + argoNamespace: 'argocd', + argocdUrl: 'https://argocd.internal', + argocdExtraRepositories: 'repo3', + dsoEnvChartVersion: 'dso-env-1.6.0', + dsoNsChartVersion: 'dso-ns-1.1.5', + } satisfies Partial, + }, + { + provide: GitlabClientService, + useValue: { + getOrCreateInfraGroupRepo: vi.fn(), + getOrCreateProjectGroupPublicUrl: vi.fn(), + getOrCreateInfraGroupRepoPublicUrl: vi.fn(), + generateCreateOrUpdateAction: vi.fn(), + maybeCreateCommit: vi.fn(), + listFiles: vi.fn(), + } satisfies Partial, + }, + { + provide: VaultClientService, + useValue: { + readProjectValues: vi.fn(), + } satisfies Partial, + }, + ], + }) +} + +describe('argoCDService', () => { + let service: ArgoCDService + let datastore: Mocked + let gitlab: Mocked + let vault: Mocked + + beforeEach(async () => { + vi.clearAllMocks() + const module: TestingModule = await createArgoCDControllerServiceTestingModule().compile() + service = module.get(ArgoCDService) + datastore = module.get(ArgoCDDatastoreService) + gitlab = module.get(GitlabClientService) + vault = module.get(VaultClientService) + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) + + it('should sync project environments', async () => { + const mockProject = { + id: '123e4567-e89b-12d3-a456-426614174000', + slug: 'project-1', + name: 'Project 1', + environments: [ + { id: '123e4567-e89b-12d3-a456-426614174001', name: 'dev', clusterId: 'c1', cpu: 1, gpu: 0, memory: 1, autosync: true }, + { id: '123e4567-e89b-12d3-a456-426614174002', name: 'prod', clusterId: 'c1', cpu: 1, gpu: 0, memory: 1, autosync: true }, + ], + clusters: [ + { id: 'c1', label: 'cluster-1', zone: { slug: 'zone-1' } }, + ], + repositories: [ + { + id: 'repo-1', + internalRepoName: 'infra-repo', + isInfra: true, + deployRevision: 'HEAD', + deployPath: '.', + helmValuesFiles: '', + }, + ], + plugins: [{ pluginName: 'argocd', key: 'extraRepositories', value: 'repo2' }], + } satisfies ProjectWithDetails + + const infraProject = makeProjectSchema({ id: 100, http_url_to_repo: 'https://gitlab.internal/infra' }) + datastore.getAllProjects.mockResolvedValue([mockProject]) + gitlab.getOrCreateInfraGroupRepo.mockResolvedValue(infraProject) + gitlab.getOrCreateProjectGroupPublicUrl.mockResolvedValue('https://gitlab.internal/group') + gitlab.getOrCreateInfraGroupRepoPublicUrl.mockResolvedValue('https://gitlab.internal/infra-repo') + gitlab.listFiles.mockResolvedValue([]) + vault.readProjectValues.mockResolvedValue({ secret: 'value' }) + gitlab.generateCreateOrUpdateAction.mockImplementation(async (_repoId, _ref, filePath: string, content: string) => { + return { action: 'create', filePath, content } as any + }) + + await expect(service.handleCron()).resolves.not.toThrow() + + // Verify Gitlab calls + expect(gitlab.maybeCreateCommit).toHaveBeenCalledTimes(1) + expect(gitlab.maybeCreateCommit).toHaveBeenCalledWith( + infraProject, + 'ci: :robot_face: Sync project-1', + expect.arrayContaining([ + { + action: 'create', + content: stringify({ + common: { + 'dso/project': 'Project 1', + 'dso/project.id': '123e4567-e89b-12d3-a456-426614174000', + 'dso/project.slug': 'project-1', + 'dso/environment': 'dev', + 'dso/environment.id': '123e4567-e89b-12d3-a456-426614174001', + }, + argocd: { + cluster: 'in-cluster', + namespace: 'argocd', + project: 'project-1-dev-6293', + envChartVersion: 'dso-env-1.6.0', + nsChartVersion: 'dso-ns-1.1.5', + }, + environment: { + valueFileRepository: 'https://gitlab.internal/infra', + valueFileRevision: 'HEAD', + valueFilePath: 'Project 1/cluster-1/dev/values.yaml', + roGroup: '/project-project-1/console/dev/RO', + rwGroup: '/project-project-1/console/dev/RW', + }, + application: { + quota: { + cpu: 1, + gpu: 0, + memory: '1Gi', + }, + sourceRepositories: [ + 'https://gitlab.internal/group/**', + 'repo3', + 'repo2', + ], + destination: { + namespace: generateNamespaceName(mockProject.id, mockProject.environments[0].id), + name: 'cluster-1', + }, + autosync: true, + vault: { secret: 'value' }, + repositories: [ + { + repoURL: 'https://gitlab.internal/infra-repo', + targetRevision: 'HEAD', + path: '.', + valueFiles: [], + }, + ], + }, + }), + filePath: 'Project 1/cluster-1/dev/values.yaml', + }, + { + action: 'create', + content: stringify({ + common: { + 'dso/project': 'Project 1', + 'dso/project.id': '123e4567-e89b-12d3-a456-426614174000', + 'dso/project.slug': 'project-1', + 'dso/environment': 'prod', + 'dso/environment.id': '123e4567-e89b-12d3-a456-426614174002', + }, + argocd: { + cluster: 'in-cluster', + namespace: 'argocd', + project: 'project-1-prod-c626', + envChartVersion: 'dso-env-1.6.0', + nsChartVersion: 'dso-ns-1.1.5', + }, + environment: { + valueFileRepository: 'https://gitlab.internal/infra', + valueFileRevision: 'HEAD', + valueFilePath: 'Project 1/cluster-1/prod/values.yaml', + roGroup: '/project-project-1/console/prod/RO', + rwGroup: '/project-project-1/console/prod/RW', + }, + application: { + quota: { + cpu: 1, + gpu: 0, + memory: '1Gi', + }, + sourceRepositories: [ + 'https://gitlab.internal/group/**', + 'repo3', + 'repo2', + ], + destination: { + namespace: generateNamespaceName(mockProject.id, mockProject.environments[1].id), + name: 'cluster-1', + }, + autosync: true, + vault: { secret: 'value' }, + repositories: [ + { + repoURL: 'https://gitlab.internal/infra-repo', + targetRevision: 'HEAD', + path: '.', + valueFiles: [], + }, + ], + }, + }), + filePath: 'Project 1/cluster-1/prod/values.yaml', + }, + ]), + ) + + expect(gitlab.listFiles).toHaveBeenCalledWith(infraProject, { + path: 'Project 1/', + recursive: true, + }) + + expect(gitlab.generateCreateOrUpdateAction).toHaveBeenCalledTimes(2) + }) + + it('should delete values file when an environment is removed', async () => { + const mockProject = { + id: '123e4567-e89b-12d3-a456-426614174000', + slug: 'project-1', + name: 'Project 1', + environments: [ + { id: '123e4567-e89b-12d3-a456-426614174001', name: 'dev', clusterId: 'c1', cpu: 1, gpu: 0, memory: 1, autosync: true }, + ], + clusters: [ + { id: 'c1', label: 'cluster-1', zone: { slug: 'zone-1' } }, + ], + repositories: [ + { + id: 'repo-1', + internalRepoName: 'infra-repo', + isInfra: true, + deployRevision: 'HEAD', + deployPath: '.', + helmValuesFiles: '', + }, + ], + plugins: [], + } satisfies ProjectWithDetails + + const infraProject = makeProjectSchema({ id: 100, http_url_to_repo: 'https://gitlab.internal/infra' }) + datastore.getAllProjects.mockResolvedValue([mockProject]) + gitlab.getOrCreateInfraGroupRepo.mockResolvedValue(infraProject) + gitlab.getOrCreateProjectGroupPublicUrl.mockResolvedValue('https://gitlab.internal/group') + gitlab.getOrCreateInfraGroupRepoPublicUrl.mockResolvedValue('https://gitlab.internal/infra-repo') + gitlab.listFiles.mockResolvedValue([ + { name: 'values.yaml', path: 'Project 1/cluster-1/dev/values.yaml' }, + { name: 'values.yaml', path: 'Project 1/cluster-1/prod/values.yaml' }, + ] as any) + vault.readProjectValues.mockResolvedValue({ secret: 'value' }) + gitlab.generateCreateOrUpdateAction.mockImplementation(async (_repoId, _ref, filePath: string, content: string) => { + return { action: 'create', filePath, content } as any + }) + + await expect(service.handleCron()).resolves.not.toThrow() + + expect(gitlab.maybeCreateCommit).toHaveBeenCalledTimes(1) + expect(gitlab.maybeCreateCommit).toHaveBeenCalledWith( + infraProject, + 'ci: :robot_face: Sync project-1', + expect.arrayContaining([ + expect.objectContaining({ + action: 'create', + filePath: 'Project 1/cluster-1/dev/values.yaml', + }), + { + action: 'delete', + filePath: 'Project 1/cluster-1/prod/values.yaml', + }, + ]), + ) + + expect(gitlab.generateCreateOrUpdateAction).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/server-nestjs/src/modules/argocd/argocd.service.ts b/apps/server-nestjs/src/modules/argocd/argocd.service.ts new file mode 100644 index 000000000..f2211d0ee --- /dev/null +++ b/apps/server-nestjs/src/modules/argocd/argocd.service.ts @@ -0,0 +1,446 @@ +import type { CommitAction, CondensedProjectSchema, SimpleProjectSchema } from '@gitbeaker/core' +import type { ProjectWithDetails } from './argocd-datastore.service' +import { createHmac } from 'node:crypto' +import { generateNamespaceName, inClusterLabel } from '@cpn-console/shared' +import { Inject, Injectable, Logger } from '@nestjs/common' +import { OnEvent } from '@nestjs/event-emitter' +import { Cron, CronExpression } from '@nestjs/schedule' +import { trace } from '@opentelemetry/api' +import { stringify } from 'yaml' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' +import { GitlabClientService } from '../gitlab/gitlab-client.service' +import { VaultClientService } from '../vault/vault-client.service' +import { ArgoCDDatastoreService } from './argocd-datastore.service' + +@Injectable() +export class ArgoCDService { + private readonly logger = new Logger(ArgoCDService.name) + + constructor( + @Inject(ArgoCDDatastoreService) private readonly argoCDDatastore: ArgoCDDatastoreService, + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(GitlabClientService) private readonly gitlab: GitlabClientService, + @Inject(VaultClientService) private readonly vault: VaultClientService, + ) { + this.logger.log('ArgoCDService initialized') + } + + @OnEvent('project.upsert') + @StartActiveSpan() + async handleUpsert(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project upsert for ${project.slug}`) + await this.ensureProject(project) + } + + @OnEvent('project.delete') + @StartActiveSpan() + async handleDelete(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project delete for ${project.slug}`) + await this.ensureProject(project) + } + + @Cron(CronExpression.EVERY_HOUR) + @StartActiveSpan() + async handleCron() { + this.logger.log('Starting ArgoCD reconciliation') + const projects = await this.argoCDDatastore.getAllProjects() + const span = trace.getActiveSpan() + span?.setAttribute('argocd.projects.count', projects.length) + await this.ensureProjects(projects) + } + + @StartActiveSpan() + private async ensureProjects(projects: ProjectWithDetails[]) { + const span = trace.getActiveSpan() + span?.setAttribute('argocd.projects.count', projects.length) + await Promise.all(projects.map(project => this.ensureProject(project))) + } + + @StartActiveSpan() + private async ensureProject(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + await this.ensureZones(project) + } + + @StartActiveSpan() + private async ensureZones( + project: ProjectWithDetails, + ): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + const zones = getDistinctZones(project) + span?.setAttribute('argocd.zones.count', zones.length) + await Promise.all(zones.map(zoneSlug => this.ensureZone(project, zoneSlug))) + } + + @StartActiveSpan() + private async ensureZone( + project: ProjectWithDetails, + zoneSlug: string, + ): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + const infraProject = await this.gitlab.getOrCreateInfraGroupRepo(zoneSlug) + span?.setAttributes({ + 'argocd.repo.id': infraProject.id, + 'argocd.repo.path': infraProject.path_with_namespace, + 'zone.slug': zoneSlug, + }) + + const environmentActions = await this.generateEnvironmentsUpdateActions( + project, + project.environments, + infraProject, + zoneSlug, + ) + const purgeEnvironmentActions = await this.generatePurgeEnvironmentActions( + project, + infraProject, + zoneSlug, + ) + const actions: CommitAction[] = [ + ...environmentActions, + ...purgeEnvironmentActions, + ] + + span?.setAttribute('argocd.repo.actions.count', actions.length) + await this.gitlab.maybeCreateCommit(infraProject, `ci: :robot_face: Sync ${project.slug}`, actions) + } + + private async generatePurgeEnvironmentActions( + project: ProjectWithDetails, + infraProject: CondensedProjectSchema, + zoneSlug: string, + ): Promise { + const neededFiles = new Set() + const clusterLabelsInZone = new Set( + project.clusters + .filter(c => c.zone.slug === zoneSlug) + .map(c => c.label), + ) + + project.environments.forEach((env) => { + const cluster = project.clusters.find(c => c.id === env.clusterId) + if (!cluster || cluster.zone.slug !== zoneSlug) return + neededFiles.add(formatEnvironmentValuesFilePath(project, cluster, env)) + }) + + const existingFiles = await this.gitlab.listFiles(infraProject, { + path: `${project.name}/`, + recursive: true, + }) + + const projectPrefix = `${project.name}/` + return existingFiles + .filter((existingFile) => { + if (existingFile.name !== 'values.yaml') return false + if (!existingFile.path.startsWith(projectPrefix)) return false + + const remaining = existingFile.path.slice(projectPrefix.length) + const clusterLabel = remaining.split('/')[0] + if (!clusterLabel || !clusterLabelsInZone.has(clusterLabel)) return false + + return !neededFiles.has(existingFile.path) + }) + .map(existingFile => ({ action: 'delete', filePath: existingFile.path } satisfies CommitAction)) + } + + private async generateEnvironmentsUpdateActions( + project: ProjectWithDetails, + environments: ProjectWithDetails['environments'], + infraProject: SimpleProjectSchema, + zoneSlug: string, + ): Promise { + const actions = await Promise.all( + environments + .filter((env) => { + const cluster = project.clusters.find(c => c.id === env.clusterId) + return cluster?.zone.slug === zoneSlug + }) + .map(env => this.generateEnvironmentUpdateAction(project, env, infraProject)), + ) + return actions.filter(a => !!a) as CommitAction[] + } + + private async generateEnvironmentUpdateAction( + project: ProjectWithDetails, + environment: ProjectWithDetails['environments'][number], + infraProject: SimpleProjectSchema, + ): Promise { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': project.slug, + 'environment.id': environment.id, + 'environment.name': environment.name, + }) + const vaultValues = await this.vault.readProjectValues(project.id) ?? {} + const cluster = project.clusters.find(c => c.id === environment.clusterId) + if (!cluster) throw new Error(`Cluster not found for environment ${environment.id}`) + span?.setAttribute('zone.slug', cluster.zone.slug) + + const valueFilePath = formatEnvironmentValuesFilePath(project, cluster, environment) + + const repo = project.repositories.find(r => r.isInfra) + if (!repo) throw new Error(`Infra repository not found for project ${project.id}`) + const repoUrl = await this.gitlab.getOrCreateInfraGroupRepoPublicUrl(repo.internalRepoName) + + const values = formatValues({ + project, + environment, + cluster, + gitlabPublicGroupUrl: await this.gitlab.getOrCreateProjectGroupPublicUrl(), + argocdExtraRepositories: this.config.argocdExtraRepositories, + infraProject, + valueFilePath, + repoUrl, + vaultValues, + argoNamespace: this.config.argoNamespace, + envChartVersion: this.config.dsoEnvChartVersion, + nsChartVersion: this.config.dsoNsChartVersion, + }) + + return this.gitlab.generateCreateOrUpdateAction( + infraProject, + 'main', + valueFilePath, + stringify(values), + ) + } +} +interface ValuesSchema { + common: { + 'dso/project': string + 'dso/project.id': string + 'dso/project.slug': string + 'dso/environment': string + 'dso/environment.id': string + } + argocd: { + cluster: string + namespace: string + project: string + envChartVersion: string + nsChartVersion: string + } + environment: { + valueFileRepository: string + valueFileRevision: string + valueFilePath: string + roGroup: string + rwGroup: string + } + application: { + quota: { + cpu: number + gpu: number + memory: string + } + sourceRepositories: string[] + destination: { + namespace: string + name: string + } + autosync: boolean + vault: Record + repositories: { + repoURL: string + targetRevision: string + path: string + valueFiles: string[] + }[] + } +} + +function formatReadOnlyGroupName(projectSlug: string, environmentName: string) { + return `/project-${projectSlug}/console/${environmentName}/RO` +} + +function formatReadWriteGroupName(projectSlug: string, environmentName: string) { + return `/project-${projectSlug}/console/${environmentName}/RW` +} + +function formatAppProjectName(projectSlug: string, env: string) { + const envHash = createHmac('sha256', '') + .update(env) + .digest('hex') + .slice(0, 4) + return `${projectSlug}-${env}-${envHash}` +} + +function formatEnvironmentValuesFilePath(project: { name: string }, cluster: { label: string }, env: { name: string }): string { + return `${project.name}/${cluster.label}/${env.name}/values.yaml` +} + +function getDistinctZones(project: ProjectWithDetails) { + const zones = new Set() + project.clusters.forEach(c => zones.add(c.zone.slug)) + return [...zones] +} + +function splitExtraRepositories(extraRepositories: string | undefined): string[] { + if (!extraRepositories) return [] + return extraRepositories.split(',').map(r => r.trim()).filter(r => r.length > 0) +} + +function formatRepositoriesValues( + repositories: ProjectWithDetails['repositories'], + repoUrl: string, + envName: string, +) { + return repositories + .filter(repo => repo.isInfra) + .map((repository) => { + const valueFiles = splitExtraRepositories(repository.helmValuesFiles?.replaceAll('', envName)) + return { + repoURL: repoUrl, + targetRevision: repository.deployRevision || 'HEAD', + path: repository.deployPath || '.', + valueFiles, + } satisfies ValuesSchema['application']['repositories'][number] + }) +} + +function formatEnvironmentValues( + infraProject: SimpleProjectSchema, + valueFilePath: string, + roGroup: string, + rwGroup: string, +) { + return { + valueFileRepository: infraProject.http_url_to_repo, + valueFileRevision: 'HEAD', + valueFilePath, + roGroup, + rwGroup, + } satisfies ValuesSchema['environment'] +} + +interface FormatSourceRepositoriesValuesOptions { + gitlabPublicGroupUrl: string + argocdExtraRepositories?: string + projectPlugins?: ProjectWithDetails['plugins'] +} + +function formatSourceRepositoriesValues( + { gitlabPublicGroupUrl, argocdExtraRepositories, projectPlugins }: FormatSourceRepositoriesValuesOptions, +): string[] { + let projectExtraRepositories = '' + if (projectPlugins) { + const argocdPlugin = projectPlugins.find(p => p.pluginName === 'argocd' && p.key === 'extraRepositories') + if (argocdPlugin) projectExtraRepositories = argocdPlugin.value + } + + return [ + `${gitlabPublicGroupUrl}/**`, + ...splitExtraRepositories(argocdExtraRepositories), + ...splitExtraRepositories(projectExtraRepositories), + ] +} + +interface FormatCommonOptions { + project: ProjectWithDetails + environment: ProjectWithDetails['environments'][number] +} + +function formatCommon({ project, environment }: FormatCommonOptions) { + return { + 'dso/project': project.name, + 'dso/project.id': project.id, + 'dso/project.slug': project.slug, + 'dso/environment': environment.name, + 'dso/environment.id': environment.id, + } satisfies ValuesSchema['common'] +} + +interface FormatArgoCDValuesOptions { + namespace: string + project: string + envChartVersion: string + nsChartVersion: string +} + +function formatArgoCDValues(options: FormatArgoCDValuesOptions) { + const { namespace, project, envChartVersion, nsChartVersion } = options + return { + cluster: inClusterLabel, + namespace, + project, + envChartVersion, + nsChartVersion, + } satisfies ValuesSchema['argocd'] +} + +interface FormatValuesOptions { + project: ProjectWithDetails + environment: ProjectWithDetails['environments'][number] + cluster: ProjectWithDetails['clusters'][number] + gitlabPublicGroupUrl: string + argocdExtraRepositories?: string + vaultValues: Record + infraProject: SimpleProjectSchema + valueFilePath: string + repoUrl: string + argoNamespace: string + envChartVersion: string + nsChartVersion: string +} + +function formatValues({ + project, + environment, + cluster, + gitlabPublicGroupUrl, + argocdExtraRepositories, + vaultValues, + infraProject, + valueFilePath, + repoUrl, + argoNamespace, + envChartVersion, + nsChartVersion, +}: FormatValuesOptions) { + return { + common: formatCommon({ project, environment }), + argocd: formatArgoCDValues({ + namespace: argoNamespace, + project: formatAppProjectName(project.slug, environment.name), + envChartVersion, + nsChartVersion, + }), + environment: formatEnvironmentValues( + infraProject, + valueFilePath, + formatReadOnlyGroupName(project.slug, environment.name), + formatReadWriteGroupName(project.slug, environment.name), + ), + application: { + quota: { + cpu: environment.cpu, + gpu: environment.gpu, + memory: `${environment.memory}Gi`, + }, + sourceRepositories: formatSourceRepositoriesValues({ + gitlabPublicGroupUrl, + argocdExtraRepositories, + projectPlugins: project.plugins, + }), + destination: { + namespace: generateNamespaceName(project.id, environment.id), + name: cluster.label, + }, + autosync: environment.autosync, + vault: vaultValues, + repositories: formatRepositoriesValues( + project.repositories, + repoUrl, + environment.name, + ), + }, + } satisfies ValuesSchema +} diff --git a/apps/server-nestjs/src/modules/gitlab/files/.gitlab-ci.yml b/apps/server-nestjs/src/modules/gitlab/files/.gitlab-ci.yml new file mode 100644 index 000000000..ca9be2984 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/files/.gitlab-ci.yml @@ -0,0 +1,22 @@ +variables: + PROJECT_NAME: + description: Nom du dépôt (dans ce Gitlab) à synchroniser. + GIT_BRANCH_DEPLOY: + description: Nom de la branche à synchroniser. + value: main + SYNC_ALL: + description: Synchroniser toutes les branches. + value: "false" + +include: + - project: $CATALOG_PATH + file: mirror.yml + ref: main + +repo_pull_sync: + extends: .repo_pull_sync + only: + - api + - triggers + - web + - schedules diff --git a/apps/server-nestjs/src/modules/gitlab/files/mirror.sh b/apps/server-nestjs/src/modules/gitlab/files/mirror.sh new file mode 100644 index 000000000..c50c923f8 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/files/mirror.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +set -e + +# Colorize terminal +red='\\e[0;31m' +no_color='\\033[0m' + +# Console step increment +i=1 + +# Default values +BRANCH_TO_SYNC=main + +print_help() { + TEXT_HELPER="\\nThis script aims to send a synchronization request to DSO.\\nFollowing flags are available: + -a Api url to send the synchronization request + -b Branch which is wanted to be synchronize for the given repository (default '$BRANCH_TO_SYNC') + -g GitLab token to trigger the pipeline on the gitlab mirror project + -i Gitlab mirror project id + -r Gitlab repository name to mirror + -h Print script help\\n" + printf "$TEXT_HELPER" +} + +print_args() { + printf "\\nArguments received: + -a API_URL: $API_URL + -b BRANCH_TO_SYNC: $BRANCH_TO_SYNC + -g GITLAB_TRIGGER_TOKEN length: \${#GITLAB_TRIGGER_TOKEN} + -i GITLAB_MIRROR_PROJECT_ID: $GITLAB_MIRROR_PROJECT_ID + -r REPOSITORY_NAME: $REPOSITORY_NAME\\n" +} + +# Parse options +while getopts :ha:b:g:i:r: flag +do + case "\${flag}" in + a) + API_URL=\${OPTARG};; + b) + BRANCH_TO_SYNC=\${OPTARG};; + g) + GITLAB_TRIGGER_TOKEN=\${OPTARG};; + i) + GITLAB_MIRROR_PROJECT_ID=\${OPTARG};; + r) + REPOSITORY_NAME=\${OPTARG};; + h) + printf "\\nHelp requested.\\n" + print_help + printf "\\nExiting.\\n" + exit 0;; + *) + printf "\\nInvalid argument \${OPTARG} (\${flag}).\\n" + print_help + print_args + exit 1;; + esac +done + +# Test if arguments are missing +if [ -z \${API_URL} ] || [ -z \${BRANCH_TO_SYNC} ] || [ -z \${GITLAB_TRIGGER_TOKEN} ] || [ -z \${GITLAB_MIRROR_PROJECT_ID} ] || [ -z \${REPOSITORY_NAME} ]; then + printf "\\nArgument(s) missing!\\n" + print_help + print_args + exit 2 +fi + +# Print arguments +print_args + +# Send synchronization request +printf "\\n\${red}\${i}.\${no_color} Send request to DSO api.\\n\\n" + +curl \\ + -X POST \\ + --fail \\ + -F token=\${GITLAB_TRIGGER_TOKEN} \\ + -F ref=main \\ + -F variables[GIT_BRANCH_DEPLOY]=\${BRANCH_TO_SYNC} \\ + -F variables[PROJECT_NAME]=\${REPOSITORY_NAME} \\ + "\${API_URL}/api/v4/projects/\${GITLAB_MIRROR_PROJECT_ID}/trigger/pipeline" diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts b/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts new file mode 100644 index 000000000..544287e63 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts @@ -0,0 +1,541 @@ +import type { ExpandedGroupSchema, Gitlab as GitlabApi, MemberSchema, ProjectSchema, RepositoryFileExpandedSchema } from '@gitbeaker/core' +import type { TestingModule } from '@nestjs/testing' +import type { MockedFunction } from 'vitest' +import { GitbeakerRequestError } from '@gitbeaker/requester-utils' +import { Test } from '@nestjs/testing' +import { beforeEach, describe, expect, it } from 'vitest' +import { mockDeep, mockReset } from 'vitest-mock-extended' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { GITLAB_REST_CLIENT, GitlabClientService } from './gitlab-client.service' +import { makeExpandedUserSchema } from './gitlab-testing.utils' + +const gitlabMock = mockDeep() + +function createGitlabClientServiceTestingModule() { + return Test.createTestingModule({ + providers: [ + GitlabClientService, + { + provide: GITLAB_REST_CLIENT, + useValue: gitlabMock, + }, + { + provide: ConfigurationService, + useValue: { + gitlabUrl: 'https://gitlab.internal', + gitlabToken: 'token', + gitlabInternalUrl: 'https://gitlab.internal', + projectRootPath: 'forge', + gitlabMirrorTokenExpirationDays: 30, + } satisfies Partial, + }, + ], + }) +} + +describe('gitlab-client', () => { + let service: GitlabClientService + + beforeEach(async () => { + mockReset(gitlabMock) + const module: TestingModule = await createGitlabClientServiceTestingModule().compile() + service = module.get(GitlabClientService) + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) + + describe('getOrCreateInfraProject', () => { + it('should create infra project if not exists', async () => { + const zoneSlug = 'zone-1' + const rootId = 123 + const infraGroupId = 456 + const projectId = 789 + + const gitlabGroupsAllMock = gitlabMock.Groups.all as MockedFunction + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: rootId, full_path: 'forge' }], + paginationInfo: { next: null }, + }) + + gitlabMock.Groups.show.mockResolvedValueOnce({ id: rootId, full_path: 'forge' } as ExpandedGroupSchema) + + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [], + paginationInfo: { next: null }, + }) + + gitlabMock.Groups.create.mockResolvedValue({ id: infraGroupId, full_path: 'forge/infra' } as ExpandedGroupSchema) + + const gitlabProjectsAllMock = gitlabMock.Projects.all as MockedFunction + gitlabProjectsAllMock.mockResolvedValueOnce({ + data: [], + paginationInfo: { next: null }, + }) + + gitlabMock.Projects.create.mockResolvedValue({ + id: projectId, + path_with_namespace: 'forge/infra/zone-1', + http_url_to_repo: 'https://gitlab.internal/infra/zone-1.git', + } as ProjectSchema) + + const result = await service.getOrCreateInfraGroupRepo(zoneSlug) + + expect(result).toEqual({ + id: projectId, + http_url_to_repo: 'https://gitlab.internal/infra/zone-1.git', + path_with_namespace: 'forge/infra/zone-1', + }) + expect(gitlabMock.Groups.create).toHaveBeenCalledWith('infra', 'infra', expect.any(Object)) + expect(gitlabMock.Projects.create).toHaveBeenCalledWith(expect.objectContaining({ + name: zoneSlug, + path: zoneSlug, + namespaceId: infraGroupId, + })) + }) + }) + + describe('commitCreateOrUpdate', () => { + it('should create commit if file not exists', async () => { + const repoId = 1 + const repo = { id: repoId } as any + const content = 'content' + const filePath = 'file.txt' + const message = 'ci: :robot_face: Update file content' + + const gitlabRepositoryFilesShowMock = gitlabMock.RepositoryFiles.show as MockedFunction + const notFoundError = new GitbeakerRequestError('Not Found', { cause: { description: '404 File Not Found' } } as any) + gitlabRepositoryFilesShowMock.mockRejectedValue(notFoundError) + + const action = await service.generateCreateOrUpdateAction(repo, 'main', filePath, content) + await service.maybeCreateCommit(repo, message, action ? [action] : []) + + expect(gitlabMock.Commits.create).toHaveBeenCalledWith( + repoId, + 'main', + message, + [{ action: 'create', filePath, content }], + ) + }) + + it('should update commit if content differs', async () => { + const repoId = 1 + const repo = { id: repoId } as any + const content = 'new content' + const filePath = 'file.txt' + const oldHash = 'oldhash' + const message = 'ci: :robot_face: Update file content' + + const gitlabRepositoryFilesShowMock = gitlabMock.RepositoryFiles.show as MockedFunction + gitlabRepositoryFilesShowMock.mockResolvedValue({ + content_sha256: oldHash, + } as RepositoryFileExpandedSchema) + + const action = await service.generateCreateOrUpdateAction(repo, 'main', filePath, content) + await service.maybeCreateCommit(repo, message, action ? [action] : []) + + expect(gitlabMock.Commits.create).toHaveBeenCalledWith( + repoId, + 'main', + message, + [{ action: 'update', filePath, content }], + ) + }) + + it('should do nothing if content matches', async () => { + const repoId = 1 + const repo = { id: repoId } as any + const content = 'content' + const filePath = 'file.txt' + const hash = 'ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73' + const message = 'ci: :robot_face: Update file content' + + const gitlabRepositoryFilesShowMock = gitlabMock.RepositoryFiles.show as MockedFunction + gitlabRepositoryFilesShowMock.mockResolvedValue({ + content_sha256: hash, + } as RepositoryFileExpandedSchema) + + const action = await service.generateCreateOrUpdateAction(repo, 'main', filePath, content) + await service.maybeCreateCommit(repo, message, action ? [action] : []) + + expect(gitlabMock.Commits.create).not.toHaveBeenCalled() + }) + }) + + describe('getOrCreateProjectGroup', () => { + it('should create project group if not exists', async () => { + const projectSlug = 'project-1' + const rootId = 123 + const groupId = 456 + + const gitlabGroupsAllMock = gitlabMock.Groups.all as MockedFunction + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: rootId, full_path: 'forge' }], + paginationInfo: { next: null }, + }) + gitlabMock.Groups.show.mockResolvedValueOnce({ id: rootId, full_path: 'forge' } as ExpandedGroupSchema) + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [], + paginationInfo: { next: null }, + }) + gitlabMock.Groups.create.mockResolvedValue({ id: groupId, name: projectSlug } as ExpandedGroupSchema) + + const result = await service.getOrCreateProjectSubGroup(projectSlug) + + expect(result).toEqual({ id: groupId, name: projectSlug }) + expect(gitlabMock.Groups.create).toHaveBeenCalledWith(projectSlug, projectSlug, expect.objectContaining({ + parentId: rootId, + })) + }) + + it('should return existing group', async () => { + const projectSlug = 'project-1' + const rootId = 123 + const groupId = 456 + + const gitlabGroupsAllMock = gitlabMock.Groups.all as MockedFunction + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: rootId, full_path: 'forge' }], + paginationInfo: { next: null }, + }) + gitlabMock.Groups.show.mockResolvedValueOnce({ id: rootId, full_path: 'forge' } as ExpandedGroupSchema) + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: groupId, name: projectSlug, parent_id: rootId, full_path: 'forge/project-1' }], + paginationInfo: { next: null }, + }) + + const result = await service.getOrCreateProjectSubGroup(projectSlug) + + expect(result).toEqual({ id: groupId, name: projectSlug, parent_id: rootId, full_path: 'forge/project-1' }) + expect(gitlabMock.Groups.create).not.toHaveBeenCalled() + }) + }) + + describe('repositories', () => { + it('should return internal repo url', async () => { + const projectSlug = 'project-1' + const repoName = 'repo-1' + const rootId = 123 + const groupId = 1 + + const gitlabGroupsAllMock = gitlabMock.Groups.all as MockedFunction + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: rootId, full_path: 'forge' }], + paginationInfo: { next: null }, + }) + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: groupId, full_path: 'forge/project-1' }], + paginationInfo: { next: null }, + }) + + const result = await service.getOrCreateProjectGroupInternalRepoUrl(projectSlug, repoName) + expect(result).toBe('https://gitlab.internal/forge/project-1/repo-1.git') + }) + + it('should upsert mirror repo', async () => { + const projectSlug = 'project-1' + const repoId = 1 + + const gitlabProjectsAllMock = gitlabMock.Projects.all as MockedFunction + gitlabProjectsAllMock.mockResolvedValue({ + data: [{ id: repoId, path_with_namespace: 'forge/project-1/mirror' }], + paginationInfo: { next: null }, + }) + + gitlabMock.Projects.edit.mockResolvedValue({ id: repoId, name: 'mirror' } as ProjectSchema) + + const result = await service.upsertProjectMirrorRepo(projectSlug) + + expect(result).toEqual({ id: repoId, name: 'mirror' }) + expect(gitlabMock.Projects.edit).toHaveBeenCalledWith(repoId, expect.objectContaining({ + name: 'mirror', + path: 'mirror', + })) + }) + + it('should create pipeline trigger token if not exists', async () => { + const projectSlug = 'project-1' + const repoId = 1 + const tokenDescription = 'mirroring-from-external-repo' + + const gitlabProjectsAllMock = gitlabMock.Projects.all as MockedFunction + gitlabProjectsAllMock.mockResolvedValue({ + data: [{ id: repoId, path_with_namespace: 'forge/project-1/mirror' }], + paginationInfo: { next: null }, + }) + gitlabMock.Projects.edit.mockResolvedValue({ id: repoId, name: 'mirror' } as ProjectSchema) + + const gitlabPipelineTriggerTokensAllMock = gitlabMock.PipelineTriggerTokens.all as MockedFunction + gitlabPipelineTriggerTokensAllMock.mockResolvedValue({ + data: [], + paginationInfo: { next: null } as any, + }) + + gitlabMock.PipelineTriggerTokens.create.mockResolvedValue({ id: 2, description: tokenDescription } as any) + + const result = await service.getOrCreateMirrorPipelineTriggerToken(projectSlug) + + expect(result).toEqual({ id: 2, description: tokenDescription }) + expect(gitlabMock.PipelineTriggerTokens.create).toHaveBeenCalledWith(repoId, tokenDescription) + }) + }) + + describe('group Members', () => { + it('should get group members', async () => { + const groupId = 1 + const group = { id: groupId } as any + const members = [{ id: 1, name: 'user' } as MemberSchema] + const gitlabGroupMembersAllMock = gitlabMock.GroupMembers.all as MockedFunction + gitlabGroupMembersAllMock.mockResolvedValue(members) + + const result = await service.getGroupMembers(group) + expect(result).toEqual(members) + expect(gitlabMock.GroupMembers.all).toHaveBeenCalledWith(groupId) + }) + + it('should add group member', async () => { + const groupId = 1 + const group = { id: groupId } as any + const userId = 2 + const accessLevel = 30 + gitlabMock.GroupMembers.add.mockResolvedValue({ id: userId } as MemberSchema) + + await service.addGroupMember(group, userId, accessLevel) + expect(gitlabMock.GroupMembers.add).toHaveBeenCalledWith(groupId, userId, accessLevel) + }) + + it('should remove group member', async () => { + const groupId = 1 + const group = { id: groupId } as any + const userId = 2 + gitlabMock.GroupMembers.remove.mockResolvedValue(undefined) + + await service.removeGroupMember(group, userId) + expect(gitlabMock.GroupMembers.remove).toHaveBeenCalledWith(groupId, userId) + }) + }) + + describe('createProjectMirrorAccessToken', () => { + it('should create project access token with correct scopes', async () => { + const projectSlug = 'project-1' + const groupId = 456 + const tokenName = `${projectSlug}-bot` + const token = { id: 1, name: tokenName, token: 'secret-token' } + + const gitlabGroupsAllMock = gitlabMock.Groups.all as MockedFunction + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: 123, full_path: 'forge' }], + paginationInfo: { next: null }, + }) + gitlabMock.Groups.show.mockResolvedValueOnce({ id: 123, full_path: 'forge' } as ExpandedGroupSchema) + + const gitlabGroupsAllSubgroupsMock = gitlabMock.Groups.allSubgroups as MockedFunction + gitlabGroupsAllSubgroupsMock.mockResolvedValueOnce({ + data: [{ id: groupId, name: projectSlug, parent_id: 123, full_path: 'forge/project-1' }], + paginationInfo: { next: null }, + }) + + gitlabMock.GroupAccessTokens.create.mockResolvedValue(token as any) + + const result = await service.createMirrorAccessToken(projectSlug) + + expect(result).toEqual(token) + expect(gitlabMock.GroupAccessTokens.create).toHaveBeenCalledWith( + groupId, + tokenName, + ['write_repository', 'read_repository', 'read_api'], + expect.any(String), + ) + }) + }) + + describe('getOrCreateProjectGroupRepo', () => { + it('should return existing repo', async () => { + const subGroupPath = 'project-1' + const repoName = 'repo-1' + const fullPath = `${subGroupPath}/${repoName}` + const projectId = 789 + + const gitlabProjectsAllMock = gitlabMock.Projects.all as MockedFunction + gitlabProjectsAllMock.mockResolvedValueOnce({ + data: [{ id: projectId, path_with_namespace: `forge/${fullPath}` }], + paginationInfo: { next: null }, + }) + + const result = await service.getOrCreateProjectGroupRepo(fullPath) + + expect(result).toEqual(expect.objectContaining({ id: projectId })) + }) + + it('should create repo if not exists', async () => { + const subGroupPath = 'project-1' + const repoName = 'repo-1' + const fullPath = `${subGroupPath}/${repoName}` + const projectId = 789 + const groupId = 456 + const rootId = 123 + + const gitlabProjectsAllMock = gitlabMock.Projects.all as MockedFunction + gitlabProjectsAllMock.mockResolvedValueOnce({ + data: [], + paginationInfo: { next: null }, + }) + + const gitlabGroupsAllMock = gitlabMock.Groups.all as MockedFunction + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: rootId, full_path: 'forge' }], + paginationInfo: { next: null }, + }) + + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: groupId, name: subGroupPath, parent_id: rootId, full_path: `forge/${subGroupPath}` }], + paginationInfo: { next: null }, + }) + + gitlabMock.Projects.create.mockResolvedValue({ id: projectId, name: repoName } as ProjectSchema) + + const result = await service.getOrCreateProjectGroupRepo(fullPath) + + expect(result).toEqual(expect.objectContaining({ id: projectId })) + expect(gitlabMock.Projects.create).toHaveBeenCalledWith(expect.objectContaining({ + name: repoName, + path: repoName, + namespaceId: groupId, + })) + }) + }) + + describe('getFile', () => { + it('should return file content', async () => { + const repoId = 1 + const repo = { id: repoId } as any + const filePath = 'file.txt' + const ref = 'main' + const file = { content: 'content' } + + gitlabMock.RepositoryFiles.show.mockResolvedValue(file as any) + + const result = await service.getFile(repo, filePath, ref) + expect(result).toEqual(file) + }) + + it('should return undefined on 404', async () => { + const repoId = 1 + const repo = { id: repoId } as any + const filePath = 'file.txt' + const ref = 'main' + const error = new GitbeakerRequestError('Not Found', { cause: { description: '404 File Not Found' } } as any) + + gitlabMock.RepositoryFiles.show.mockRejectedValue(error) + + const result = await service.getFile(repo, filePath, ref) + expect(result).toBeUndefined() + }) + + it('should throw on other errors', async () => { + const repoId = 1 + const repo = { id: repoId } as any + const filePath = 'file.txt' + const ref = 'main' + const error = new Error('Some other error') + + gitlabMock.RepositoryFiles.show.mockRejectedValue(error) + + await expect(service.getFile(repo, filePath, ref)).rejects.toThrow(error) + }) + }) + + describe('listFiles', () => { + it('should return files', async () => { + const repoId = 1 + const repo = { id: repoId } as any + const files = [{ path: 'file.txt' }] + + gitlabMock.Repositories.allRepositoryTrees.mockResolvedValue(files as any) + + const result = await service.listFiles(repo) + expect(result).toEqual(files) + }) + + it('should return empty array on 404', async () => { + const repoId = 1 + const repo = { id: repoId } as any + const error = new GitbeakerRequestError('Not Found', { cause: { description: '404 Tree Not Found' } } as any) + + gitlabMock.Repositories.allRepositoryTrees.mockRejectedValue(error) + + const result = await service.listFiles(repo) + expect(result).toEqual([]) + }) + }) + + describe('getProjectToken', () => { + it('should return specific token', async () => { + const projectSlug = 'project-1' + const groupId = 456 + const tokenName = `${projectSlug}-bot` + const token = { id: 1, name: tokenName } + + const gitlabGroupsAllMock = gitlabMock.Groups.all as MockedFunction + gitlabGroupsAllMock.mockResolvedValueOnce({ + data: [{ id: 123, full_path: 'forge' }], + paginationInfo: { next: null }, + }) + gitlabMock.Groups.show.mockResolvedValueOnce({ id: 123, full_path: 'forge' } as ExpandedGroupSchema) + const gitlabGroupsAllSubgroupsMock = gitlabMock.Groups.allSubgroups as MockedFunction + gitlabGroupsAllSubgroupsMock.mockResolvedValueOnce({ + data: [{ id: groupId, name: projectSlug, parent_id: 123, full_path: `forge/${projectSlug}` }], + paginationInfo: { next: null }, + }) + + const gitlabGroupAccessTokensAllMock = gitlabMock.GroupAccessTokens.all as MockedFunction + gitlabGroupAccessTokensAllMock.mockResolvedValue({ + data: [token] as any, + paginationInfo: { next: null } as any, + }) + + const result = await service.getProjectToken(projectSlug) + expect(result).toEqual(token) + }) + }) + + describe('createUser', () => { + it('should create user', async () => { + const email = 'user@example.com' + const username = 'user' + const name = 'User Name' + const user = makeExpandedUserSchema({ id: 1, username }) + + gitlabMock.Users.create.mockResolvedValue(user) + + const result = await service.createUser(email, username, name) + + expect(result).toEqual(user) + expect(gitlabMock.Users.create).toHaveBeenCalledWith(expect.objectContaining({ + email, + username, + name, + skipConfirmation: true, + })) + }) + }) + + describe('commitMirror', () => { + it('should create mirror commit', async () => { + const repoId = 1 + + await service.commitMirror(repoId) + + expect(gitlabMock.Commits.create).toHaveBeenCalledWith( + repoId, + 'main', + expect.any(String), + expect.arrayContaining([ + expect.objectContaining({ filePath: '.gitlab-ci.yml', action: 'create' }), + expect.objectContaining({ filePath: 'mirror.sh', action: 'create' }), + ]), + ) + }) + }) +}) diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts b/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts new file mode 100644 index 000000000..53be516b1 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts @@ -0,0 +1,345 @@ +import type { + AccessTokenScopes, + BaseRequestOptions, + CommitAction, + CondensedGroupSchema, + CondensedProjectSchema, + Gitlab, + GroupSchema, + OffsetPagination, + PaginationRequestOptions, + PipelineTriggerTokenSchema, +} from '@gitbeaker/core' +import { createHash } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { GitbeakerRequestError } from '@gitbeaker/requester-utils' +import { Inject, Injectable, Logger } from '@nestjs/common' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { find } from '../../utils/iterable' +import { INFRA_GROUP_PATH, MIRROR_REPO_NAME, TOKEN_DESCRIPTION, TOPIC_PLUGIN_MANAGED } from './gitlab.constants' + +export const GITLAB_REST_CLIENT = Symbol('GITLAB_REST_CLIENT') + +@Injectable() +export class GitlabClientService { + private readonly logger = new Logger(GitlabClientService.name) + + constructor( + @Inject(ConfigurationService) readonly config: ConfigurationService, + @Inject(GITLAB_REST_CLIENT) private readonly client: Gitlab, + ) { + } + + async getGroupByPath(path: string) { + return find( + offsetPaginate(opts => this.client.Groups.all({ search: path, orderBy: 'path', ...opts })), + g => g.full_path === path, + ) + } + + async createGroup(path: string) { + return this.client.Groups.create(path, path) + } + + async createSubGroup(parentGroup: CondensedGroupSchema, name: string) { + return this.client.Groups.create(name, name, { parentId: parentGroup.id }) + } + + async getOrCreateGroupByPath(path: string) { + const parts = path.split('/') + const rootGroupPath = parts.shift() + if (!rootGroupPath) throw new Error('Invalid projects root dir') + + let parentGroup = await this.getGroupByPath(rootGroupPath) ?? await this.createGroup(rootGroupPath) + + for (const part of parts) { + const fullPath = `${parentGroup.full_path}/${part}` + parentGroup = await this.getGroupByPath(fullPath) ?? await this.createSubGroup(parentGroup, part) + } + + return parentGroup + } + + async getOrCreateProjectGroup() { + if (!this.config.projectRootPath) throw new Error('projectRootPath not configured') + return this.getOrCreateGroupByPath(this.config.projectRootPath) + } + + async getOrCreateProjectSubGroup(subGroupPath: string) { + const fullPath = this.config.projectRootPath + ? `${this.config.projectRootPath}/${subGroupPath}` + : subGroupPath + return this.getOrCreateGroupByPath(fullPath) + } + + async getOrCreateProjectGroupPublicUrl(): Promise { + const projectGroup = await this.getOrCreateProjectGroup() + return `${this.config.gitlabUrl}/${projectGroup.full_path}` + } + + async getOrCreateInfraGroupRepoPublicUrl(repoName: string): Promise { + const projectGroup = await this.getOrCreateProjectGroup() + return `${this.config.gitlabUrl}/${projectGroup.full_path}/${INFRA_GROUP_PATH}/${repoName}.git` + } + + async getOrCreateProjectGroupInternalRepoUrl(subGroupPath: string, repoName: string): Promise { + const projectGroup = await this.getOrCreateProjectSubGroup(subGroupPath) + return `${this.config.gitlabInternalUrl}/${projectGroup.full_path}/${repoName}.git` + } + + async getOrCreateProjectGroupRepo(subGroupPath: string) { + const fullPath = this.config.projectRootPath + ? `${this.config.projectRootPath}/${subGroupPath}` + : subGroupPath + try { + const existingRepo = await this.client.Projects.show(fullPath) + if (existingRepo) return existingRepo + } catch (error) { + if (!(error instanceof GitbeakerRequestError) || !error.cause?.description?.includes('404')) { + throw error + } + } + const repo = await find( + offsetPaginate(opts => this.client.Projects.all({ + search: fullPath, + orderBy: 'path', + ...opts, + })), + p => p.path_with_namespace === fullPath, + ) + if (repo) return repo + const parts = subGroupPath.split('/') + const repoName = parts.pop() + if (!repoName) throw new Error('Invalid repo path') + const parentGroup = await this.getOrCreateProjectSubGroup(parts.join('/')) + try { + return await this.client.Projects.create({ + name: repoName, + path: repoName, + namespaceId: parentGroup.id, + defaultBranch: 'main', + } as any) + } catch (error) { + if (error instanceof GitbeakerRequestError && error.cause?.description?.includes('has already been taken')) { + return this.client.Projects.show(fullPath) + } + throw error + } + } + + async getOrCreateInfraGroupRepo(path: string) { + return this.getOrCreateProjectGroupRepo(join(INFRA_GROUP_PATH, path)) + } + + async getFile(repo: CondensedProjectSchema, filePath: string, ref: string = 'main') { + try { + return await this.client.RepositoryFiles.show(repo.id, filePath, ref) + } catch (error) { + if (error instanceof GitbeakerRequestError && error.cause?.description?.includes('Not Found')) { + this.logger.debug(`File not found: ${filePath}`) + return + } + throw error + } + } + + async maybeCreateCommit( + repo: CondensedProjectSchema, + message: string, + actions: CommitAction[], + ref: string = 'main', + ): Promise { + if (actions.length === 0) { + this.logger.debug('No actions to commit') + return + } + await this.client.Commits.create(repo.id, ref, message, actions) + } + + async generateCreateOrUpdateAction(repo: CondensedProjectSchema, ref: string, filePath: string, content: string) { + const file = await this.getFile(repo, filePath, ref) + if (file && !hasFileContentChanged(file, content)) { + this.logger.debug(`File content is up to date, no need to commit: ${filePath}`) + return null + } + return { + action: file ? 'update' : 'create', + filePath, + content, + } satisfies CommitAction + } + + async listFiles(repo: CondensedProjectSchema, options: { path?: string, recursive?: boolean, ref?: string } = {}) { + try { + return await this.client.Repositories.allRepositoryTrees(repo.id, { + path: options.path ?? '/', + recursive: options.recursive ?? false, + ref: options.ref ?? 'main', + }) + } catch (error) { + if (error instanceof GitbeakerRequestError && error.cause?.description?.includes('Not Found')) { + return [] + } + if (error instanceof GitbeakerRequestError && error.cause?.description?.includes('404 Tree Not Found')) { + return [] + } + throw error + } + } + + async getProjectGroup(projectSlug: string): Promise { + const parentGroup = await this.getOrCreateProjectGroup() + return find( + offsetPaginate(opts => this.client.Groups.allSubgroups(parentGroup.id, opts)), + g => g.name === projectSlug, + ) + } + + async deleteGroup(group: CondensedGroupSchema): Promise { + await this.client.Groups.remove(group.id) + } + + async getGroupMembers(group: CondensedGroupSchema) { + return this.client.GroupMembers.all(group.id) + } + + async addGroupMember(group: CondensedGroupSchema, userId: number, accessLevel: number) { + return this.client.GroupMembers.add(group.id, userId, accessLevel) + } + + async editGroupMember(group: CondensedGroupSchema, userId: number, accessLevel: number) { + return this.client.GroupMembers.edit(group.id, userId, accessLevel) + } + + async removeGroupMember(group: CondensedGroupSchema, userId: number) { + return this.client.GroupMembers.remove(group.id, userId) + } + + async getUserByEmail(email: string) { + const users = await this.client.Users.all({ search: email, orderBy: 'username' }) + if (users.length === 0) return null + return users[0] + } + + async createUser(email: string, username: string, name: string) { + return this.client.Users.create({ + email, + username, + name, + skipConfirmation: true, + }) + } + + async* getRepos(projectSlug: string) { + const group = await this.getOrCreateProjectSubGroup(projectSlug) + const repos = offsetPaginate(opts => this.client.Groups.allProjects(group.id, { simple: false, ...opts })) + for await (const repo of repos) { + yield repo + } + } + + async upsertProjectGroupRepo(projectSlug: string, repoName: string, description?: string) { + const repo = await this.getOrCreateProjectGroupRepo(`${projectSlug}/${repoName}`) + return this.client.Projects.edit(repo.id, { + name: repoName, + path: repoName, + topics: [TOPIC_PLUGIN_MANAGED], + description, + }) + } + + async deleteProjectGroupRepo(projectSlug: string, repoName: string) { + const repo = await this.getOrCreateProjectGroupRepo(`${projectSlug}/${repoName}`) + return this.client.Projects.remove(repo.id) + } + + async commitMirror(repoId: number) { + const actions: CommitAction[] = [ + { + action: 'create', + filePath: '.gitlab-ci.yml', + content: await readGitlabCIConfigContent(), + execute_filemode: false, + }, + { + action: 'create', + filePath: 'mirror.sh', + content: await readMirrorScriptContent(), + execute_filemode: true, + }, + ] + + await this.client.Commits.create( + repoId, + 'main', + 'ci: :construction_worker: first mirror', + actions, + ) + } + + async upsertProjectMirrorRepo(projectSlug: string) { + return this.upsertProjectGroupRepo(projectSlug, MIRROR_REPO_NAME) + } + + async getProjectToken(projectSlug: string) { + const group = await this.getProjectGroup(projectSlug) + if (!group) throw new Error('Unable to retrieve gitlab project group') + return find( + offsetPaginate<{ name: string }>(opts => this.client.GroupAccessTokens.all(group.id, opts) as any), + token => token.name === `${projectSlug}-bot`, + ) + } + + async createProjectToken(projectSlug: string, tokenName: string, scopes: AccessTokenScopes[]) { + const group = await this.getProjectGroup(projectSlug) + if (!group) throw new Error('Unable to retrieve gitlab project group') + const expirationDays = Number(this.config.gitlabMirrorTokenExpirationDays) + const effectiveExpirationDays = Number.isFinite(expirationDays) && expirationDays > 0 ? expirationDays : 30 + const expiryDate = new Date(Date.now() + effectiveExpirationDays * 24 * 60 * 60 * 1000) + return this.client.GroupAccessTokens.create(group.id, tokenName, scopes, expiryDate.toISOString().slice(0, 10)) + } + + async createMirrorAccessToken(projectSlug: string) { + const tokenName = `${projectSlug}-bot` + return this.createProjectToken(projectSlug, tokenName, ['write_repository', 'read_repository', 'read_api']) + } + + async getOrCreateMirrorPipelineTriggerToken(projectSlug: string): Promise { + const mirrorRepo = await this.upsertProjectMirrorRepo(projectSlug) + const currentTriggerToken = await find( + offsetPaginate(opts => this.client.PipelineTriggerTokens.all(mirrorRepo.id, opts) as any), + token => token.description === TOKEN_DESCRIPTION, + ) + return (currentTriggerToken ?? await this.client.PipelineTriggerTokens.create(mirrorRepo.id, TOKEN_DESCRIPTION)) as any + } +} + +function hasFileContentChanged(file: any, content: string) { + return file?.content_sha256 !== digestContent(content) +} + +function digestContent(content: string) { + return createHash('sha256').update(content).digest('hex') +} + +function readGitlabCIConfigContent() { + return readFile(join(__dirname, './files/.gitlab-ci.yml'), 'utf-8') +} + +function readMirrorScriptContent() { + return readFile(join(__dirname, './files/mirror.sh'), 'utf-8') +} + +export async function* offsetPaginate( + request: (options: PaginationRequestOptions<'offset'> & BaseRequestOptions) => Promise<{ data: T[], paginationInfo: OffsetPagination }>, +): AsyncGenerator { + let page: number | null = 1 + while (page !== null) { + const { data, paginationInfo } = await request({ page, showExpanded: true, pagination: 'offset' }) + for (const item of data) { + yield item + } + page = paginationInfo.next ? paginationInfo.next : null + } +} diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab-datastore.service.ts b/apps/server-nestjs/src/modules/gitlab/gitlab-datastore.service.ts new file mode 100644 index 000000000..893414df6 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/gitlab-datastore.service.ts @@ -0,0 +1,103 @@ +import type { Prisma } from '@prisma/client' +import { Inject, Injectable } from '@nestjs/common' +import { PrismaService } from '../../cpin-module/infrastructure/database/prisma.service' + +export const projectSelect = { + id: true, + name: true, + slug: true, + description: true, + owner: { + select: { + id: true, + email: true, + firstName: true, + lastName: true, + adminRoleIds: true, + }, + }, + plugins: { + select: { + key: true, + value: true, + }, + }, + roles: { + select: { + id: true, + oidcGroup: true, + }, + }, + members: { + select: { + user: { + select: { + id: true, + email: true, + firstName: true, + lastName: true, + adminRoleIds: true, + }, + }, + roleIds: true, + }, + }, + repositories: { + select: { + id: true, + internalRepoName: true, + isInfra: true, + isPrivate: true, + externalRepoUrl: true, + externalUserName: true, + }, + }, + clusters: { + select: { + id: true, + label: true, + zone: { + select: { + slug: true, + }, + }, + }, + }, +} satisfies Prisma.ProjectSelect + +export type ProjectWithDetails = Prisma.ProjectGetPayload<{ + select: typeof projectSelect +}> + +@Injectable() +export class GitlabDatastoreService { + constructor(@Inject(PrismaService) private readonly prisma: PrismaService) {} + + async getAllProjects(): Promise { + return this.prisma.project.findMany({ + select: projectSelect, + where: { + plugins: { + some: { + pluginName: 'gitlab', + }, + }, + }, + }) + } + + async getProject(id: string): Promise { + return this.prisma.project.findUnique({ + where: { id }, + select: projectSelect, + }) + } + + async getUser(id: string) { + return this.prisma.user.findUnique({ + where: { + id, + }, + }) + } +} diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab-health.service.ts b/apps/server-nestjs/src/modules/gitlab/gitlab-health.service.ts new file mode 100644 index 000000000..7ab18f3d8 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/gitlab-health.service.ts @@ -0,0 +1,25 @@ +import { Inject, Injectable } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' + +@Injectable() +export class GitlabHealthService { + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(HealthIndicatorService) private readonly healthIndicator: HealthIndicatorService, + ) {} + + async check(key: string) { + const indicator = this.healthIndicator.check(key) + if (!this.config.gitlabInternalUrl) return indicator.down('Not configured') + + const url = new URL('/-/health', this.config.gitlabInternalUrl).toString() + try { + const response = await fetch(url, { method: 'GET' }) + if (response.status < 500) return indicator.up({ httpStatus: response.status }) + return indicator.down({ httpStatus: response.status }) + } catch (error) { + return indicator.down(error instanceof Error ? error.message : String(error)) + } + } +} diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab-testing.utils.ts b/apps/server-nestjs/src/modules/gitlab/gitlab-testing.utils.ts new file mode 100644 index 000000000..ed789aa39 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/gitlab-testing.utils.ts @@ -0,0 +1,335 @@ +import type { AccessTokenExposedSchema, AccessTokenSchema, ExpandedUserSchema, GroupSchema, MemberSchema, OffsetPagination, PipelineTriggerTokenSchema, ProjectSchema, RepositoryFileExpandedSchema, RepositoryTreeSchema, SimpleUserSchema } from '@gitbeaker/core' +import type { ProjectWithDetails } from './gitlab-datastore.service' +import { faker } from '@faker-js/faker' +import { AccessLevel } from '@gitbeaker/core' +import { GitbeakerRequestError } from '@gitbeaker/requester-utils' +import { VaultError } from '../vault/vault-client.service' + +export function notFoundError(method: string, path: string) { + return new VaultError('NotFound', 'Not Found', { status: 404, method, path }) +} + +export function makeSimpleUserSchema(overrides: Partial = {}) { + return { + id: 1, + name: 'User', + username: 'user', + state: 'active', + avatar_url: '', + web_url: 'https://gitlab.example/users/user', + created_at: faker.date.past().toISOString(), + ...overrides, + } satisfies SimpleUserSchema +} + +export function makeExpandedUserSchema(overrides: Partial = {}): ExpandedUserSchema { + const isoDate = faker.date.past().toISOString() + return { + id: 1, + name: 'User', + username: 'user', + state: 'active', + avatar_url: '', + web_url: 'https://gitlab.example/users/user', + created_at: isoDate, + locked: null, + bio: null, + bot: false, + location: null, + public_email: null, + skype: null, + linkedin: null, + twitter: null, + discord: null, + website_url: null, + pronouns: null, + organization: null, + job_title: null, + work_information: null, + followers: null, + following: null, + local_time: null, + is_followed: null, + is_admin: null, + last_sign_in_at: isoDate, + confirmed_at: isoDate, + last_activity_on: isoDate, + email: 'user@example.com', + theme_id: 1, + color_scheme_id: 1, + projects_limit: 0, + current_sign_in_at: null, + note: null, + identities: null, + can_create_group: false, + can_create_project: false, + two_factor_enabled: false, + external: false, + private_profile: null, + namespace_id: null, + created_by: null, + ...overrides, + } satisfies ExpandedUserSchema +} + +export function makeMemberSchema(overrides: Partial = {}) { + return { + id: 1, + username: 'user', + name: 'User', + state: 'active', + avatar_url: '', + web_url: 'https://gitlab.example/users/user', + expires_at: faker.date.future().toISOString(), + access_level: 30, + email: 'user@example.com', + group_saml_identity: { + extern_uid: '', + provider: '', + saml_provider_id: 1, + }, + ...overrides, + } satisfies MemberSchema +} + +export function makeGroupSchema(overrides: Partial = {}) { + return { + id: 123, + web_url: 'https://gitlab.example/groups/forge', + name: 'forge', + avatar_url: '', + full_name: 'forge', + full_path: 'forge', + path: 'forge', + description: '', + visibility: 'private', + share_with_group_lock: false, + require_two_factor_authentication: false, + two_factor_grace_period: 0, + project_creation_level: 'maintainer', + subgroup_creation_level: 'maintainer', + lfs_enabled: true, + default_branch_protection: 0, + request_access_enabled: false, + created_at: faker.date.past().toISOString(), + parent_id: 0, + ...overrides, + } satisfies GroupSchema +} + +export function makeProjectSchema(overrides: Partial = {}) { + return { + id: 1, + web_url: 'https://gitlab.example/projects/1', + name: 'repo', + path: 'repo', + description: '', + name_with_namespace: 'forge / repo', + path_with_namespace: 'forge/repo', + created_at: faker.date.past().toISOString(), + default_branch: 'main', + topics: [], + ssh_url_to_repo: 'ssh://gitlab.example/forge/repo.git', + http_url_to_repo: 'https://gitlab.example/forge/repo.git', + readme_url: '', + forks_count: 0, + avatar_url: null, + star_count: 0, + last_activity_at: faker.date.future().toISOString(), + namespace: { id: 1, name: 'forge', path: 'forge', kind: 'group', full_path: 'forge', avatar_url: '', web_url: 'https://gitlab.example/groups/forge' }, + description_html: '', + visibility: 'private', + empty_repo: false, + owner: { id: 1, name: 'Owner', created_at: faker.date.past().toISOString() }, + issues_enabled: true, + open_issues_count: 0, + merge_requests_enabled: true, + jobs_enabled: true, + wiki_enabled: true, + snippets_enabled: true, + can_create_merge_request_in: true, + resolve_outdated_diff_discussions: false, + container_registry_access_level: 'enabled', + security_and_compliance_access_level: 'enabled', + container_expiration_policy: { + cadence: '1d', + enabled: false, + keep_n: null, + older_than: null, + name_regex_delete: null, + name_regex_keep: null, + next_run_at: faker.date.future().toISOString(), + }, + updated_at: faker.date.past().toISOString(), + creator_id: 1, + import_url: null, + import_type: null, + import_status: 'none', + import_error: null, + permissions: { + project_access: { access_level: 0, notification_level: 0 }, + group_access: { access_level: 0, notification_level: 0 }, + }, + archived: false, + license_url: '', + license: { key: 'mit', name: 'MIT', nickname: 'MIT', html_url: '', source_url: '' }, + shared_runners_enabled: true, + group_runners_enabled: true, + runners_token: '', + ci_default_git_depth: 0, + ci_forward_deployment_enabled: false, + ci_forward_deployment_rollback_allowed: false, + ci_allow_fork_pipelines_to_run_in_parent_project: false, + ci_separated_caches: false, + ci_restrict_pipeline_cancellation_role: '', + public_jobs: false, + shared_with_groups: null, + repository_storage: '', + only_allow_merge_if_pipeline_succeeds: false, + allow_merge_on_skipped_pipeline: false, + restrict_user_defined_variables: false, + only_allow_merge_if_all_discussions_are_resolved: false, + remove_source_branch_after_merge: false, + printing_merge_requests_link_enabled: false, + request_access_enabled: false, + merge_method: '', + squash_option: '', + auto_devops_enabled: false, + auto_devops_deploy_strategy: '', + mirror: false, + mirror_user_id: 1, + mirror_trigger_builds: false, + only_mirror_protected_branches: false, + mirror_overwrites_diverged_branches: false, + external_authorization_classification_label: '', + packages_enabled: false, + service_desk_enabled: false, + service_desk_address: 'service-desk@example.com', + service_desk_reply_to: 'service-desk@example.com', + autoclose_referenced_issues: false, + suggestion_commit_message: 'Add suggestion commit message', + enforce_auth_checks_on_uploads: false, + merge_commit_template: 'Add suggestion commit message', + squash_commit_template: 'Add suggestion commit message', + issue_branch_template: 'Add suggestion commit message', + marked_for_deletion_on: faker.date.future().toISOString(), + compliance_frameworks: [], + warn_about_potentially_unwanted_characters: false, + container_registry_image_prefix: 'registry.gitlab.example/forge/repo', + _links: { + self: 'https://gitlab.example/projects/1', + issues: 'https://gitlab.example/projects/1/issues', + merge_requests: 'https://gitlab.example/projects/1/merge_requests', + repo_branches: 'https://gitlab.example/projects/1/repository/branches', + labels: 'https://gitlab.example/projects/1/labels', + events: 'https://gitlab.example/projects/1/events', + members: 'https://gitlab.example/projects/1/members', + cluster_agents: 'https://gitlab.example/projects/1/cluster_agents', + }, + ...overrides, + } satisfies ProjectSchema +} + +export function makeProjectWithDetails(overrides: Partial = {}) { + return { + id: 'p1', + slug: 'project-1', + name: 'Project 1', + description: 'Test project', + owner: { id: 'o1', email: 'owner@example.com', firstName: 'Owner', lastName: 'User', adminRoleIds: [] }, + plugins: [], + roles: [], + members: [], + repositories: [], + clusters: [], + ...overrides, + } satisfies ProjectWithDetails +} + +export function makePipelineTriggerToken(overrides: Partial = {}) { + return { + id: 1, + description: 'mirroring-from-external-repo', + created_at: faker.date.past().toISOString(), + last_used: null, + token: 'trigger-token', + updated_at: faker.date.past().toISOString(), + owner: null, + repoId: 1, + ...overrides, + } satisfies PipelineTriggerTokenSchema +} + +export function makeOffsetPagination(overrides: Partial = {}) { + return { + total: 1, + next: null, + current: 1, + previous: null, + perPage: 20, + totalPages: 1, + ...overrides, + } satisfies OffsetPagination +} + +export function makeAccessTokenSchema(overrides: Partial = {}) { + const isoDate = faker.date.past().toISOString() + return { + id: 1, + user_id: 1, + name: 'token', + expires_at: isoDate, + active: true, + created_at: isoDate, + revoked: false, + access_level: AccessLevel.DEVELOPER, + ...overrides, + } satisfies AccessTokenSchema +} + +export function makeAccessTokenExposedSchema(overrides: Partial = {}) { + return { + ...makeAccessTokenSchema(), + token: 'secret-token', + ...overrides, + } satisfies AccessTokenExposedSchema +} + +export function makeRepositoryFileExpandedSchema(overrides: Partial = {}) { + return { + file_name: 'file.txt', + file_path: 'file.txt', + size: 7, + encoding: 'base64', + content: 'content', + content_sha256: 'sha256', + ref: 'main', + blob_id: 'blob', + commit_id: 'commit', + last_commit_id: 'last-commit', + ...overrides, + } satisfies RepositoryFileExpandedSchema +} + +export function makeRepositoryTreeSchema(overrides: Partial = {}) { + return { + id: 'id', + name: 'file.txt', + type: 'blob', + path: 'file.txt', + mode: '100644', + ...overrides, + } satisfies RepositoryTreeSchema +} + +export function makeGitbeakerRequestError(params: { message?: string, status?: number, statusText?: string, description: string }) { + const request = new Request('https://gitlab.internal.example/api') + const response = new Response(null, { status: params.status ?? 404, statusText: params.statusText ?? 'Not Found' }) + return new GitbeakerRequestError(params.message ?? params.statusText ?? 'Error', { + cause: { + description: params.description, + request, + response, + }, + }) +} diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab.constants.ts b/apps/server-nestjs/src/modules/gitlab/gitlab.constants.ts new file mode 100644 index 000000000..b4c0736ac --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/gitlab.constants.ts @@ -0,0 +1,12 @@ +export const INFRA_GROUP_NAME = 'Infra' +export const INFRA_GROUP_PATH = 'infra' +export const INFRA_APPS_REPO_NAME = 'infra-apps' +export const MIRROR_REPO_NAME = 'mirror' +export const TOPIC_PLUGIN_MANAGED = 'plugin-managed' +export const TOKEN_DESCRIPTION = 'mirroring-from-external-repo' + +export const DEFAULT_ADMIN_GROUP_PATH = '/console/admin' +export const DEFAULT_AUDITOR_GROUP_PATH = '/console/readonly' +export const DEFAULT_PROJECT_MAINTAINER_GROUP_PATH_SUFFIX = '/console/admin' +export const DEFAULT_PROJECT_DEVELOPER_GROUP_PATH_SUFFIX = '/console/developer,/console/devops' +export const DEFAULT_PROJECT_REPORTER_GROUP_PATH_SUFFIX = '/console/readonly' diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab.module.ts b/apps/server-nestjs/src/modules/gitlab/gitlab.module.ts new file mode 100644 index 000000000..4425d3997 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/gitlab.module.ts @@ -0,0 +1,32 @@ +import { Gitlab } from '@gitbeaker/rest' +import { Module } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationModule } from '../../cpin-module/infrastructure/configuration/configuration.module' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { InfrastructureModule } from '../../cpin-module/infrastructure/infrastructure.module' +import { VaultModule } from '../vault/vault.module' +import { GITLAB_REST_CLIENT, GitlabClientService } from './gitlab-client.service' +import { GitlabDatastoreService } from './gitlab-datastore.service' +import { GitlabHealthService } from './gitlab-health.service' +import { GitlabService } from './gitlab.service' + +@Module({ + imports: [ConfigurationModule, InfrastructureModule, VaultModule], + providers: [ + { + provide: GITLAB_REST_CLIENT, + inject: [ConfigurationService], + useFactory: (config: ConfigurationService) => new Gitlab({ + token: config.gitlabToken, + host: config.gitlabInternalUrl, + }), + }, + HealthIndicatorService, + GitlabClientService, + GitlabDatastoreService, + GitlabHealthService, + GitlabService, + ], + exports: [GitlabClientService, GitlabHealthService], +}) +export class GitlabModule {} diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab.service.spec.ts b/apps/server-nestjs/src/modules/gitlab/gitlab.service.spec.ts new file mode 100644 index 000000000..06de2cba7 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/gitlab.service.spec.ts @@ -0,0 +1,346 @@ +import type { AccessTokenExposedSchema } from '@gitbeaker/core' +import type { Mocked } from 'vitest' +import { ENABLED } from '@cpn-console/shared' +import { faker } from '@faker-js/faker' +import { AccessLevel } from '@gitbeaker/core' +import { Test } from '@nestjs/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { VaultClientService } from '../vault/vault-client.service' +import { GitlabClientService } from './gitlab-client.service' +import { GitlabDatastoreService } from './gitlab-datastore.service' +import { makeExpandedUserSchema, makeGroupSchema, makeMemberSchema, makePipelineTriggerToken, makeProjectSchema, makeProjectWithDetails, makeSimpleUserSchema } from './gitlab-testing.utils' +import { TOPIC_PLUGIN_MANAGED } from './gitlab.constants' +import { GitlabService } from './gitlab.service' + +function createGitlabControllerServiceTestingModule() { + return Test.createTestingModule({ + providers: [ + GitlabService, + { + provide: GitlabClientService, + useValue: { + getOrCreateProjectSubGroup: vi.fn(), + getGroupMembers: vi.fn(), + addGroupMember: vi.fn(), + editGroupMember: vi.fn(), + removeGroupMember: vi.fn(), + getUserByEmail: vi.fn(), + createUser: vi.fn(), + getRepos: vi.fn(), + getProjectToken: vi.fn(), + deleteGroup: vi.fn(), + commitMirror: vi.fn(), + getOrCreateMirrorPipelineTriggerToken: vi.fn(), + createProjectToken: vi.fn(), + createMirrorAccessToken: vi.fn(), + upsertProjectGroupRepo: vi.fn(), + upsertProjectMirrorRepo: vi.fn(), + getOrCreateProjectGroupInternalRepoUrl: vi.fn(), + deleteProjectGroupRepo: vi.fn(), + getOrCreateInfraGroupRepoPublicUrl: vi.fn(), + } satisfies Partial, + }, + { + provide: GitlabDatastoreService, + useValue: { + getAllProjects: vi.fn(), + } satisfies Partial, + }, + { + provide: VaultClientService, + useValue: { + read: vi.fn(), + write: vi.fn(), + delete: vi.fn(), + readGitlabMirrorCreds: vi.fn(), + writeGitlabMirrorCreds: vi.fn(), + deleteGitlabMirrorCreds: vi.fn(), + readTechnReadOnlyCreds: vi.fn(), + writeTechReadOnlyCreds: vi.fn(), + writeMirrorTriggerToken: vi.fn(), + } satisfies Partial, + }, + { + provide: ConfigurationService, + useValue: { + projectRootDir: 'forge/console', + projectRootPath: 'forge', + }, + }, + ], + }) +} + +describe('gitlabService', () => { + let service: GitlabService + let gitlab: Mocked + let vault: Mocked + let gitlabDatastore: Mocked + + beforeEach(async () => { + const moduleRef = await createGitlabControllerServiceTestingModule().compile() + service = moduleRef.get(GitlabService) + gitlab = moduleRef.get(GitlabClientService) + vault = moduleRef.get(VaultClientService) + gitlabDatastore = moduleRef.get(GitlabDatastoreService) + + vault.writeGitlabMirrorCreds.mockResolvedValue(undefined) + vault.deleteGitlabMirrorCreds.mockResolvedValue(undefined) + vault.writeTechReadOnlyCreds.mockResolvedValue(undefined) + vault.writeMirrorTriggerToken.mockResolvedValue(undefined) + vault.readTechnReadOnlyCreds.mockResolvedValue(null) + vault.readGitlabMirrorCreds.mockResolvedValue(null) + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) + + describe('handleUpsert', () => { + it('should reconcile project members and repositories', async () => { + const project = makeProjectWithDetails() + const group = makeGroupSchema({ + id: 123, + full_path: 'forge/console/project-1', + full_name: 'forge/console/project-1', + name: 'project-1', + path: 'project-1', + parent_id: 1, + }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([]) + gitlab.getRepos.mockReturnValue((async function* () { })()) + gitlab.upsertProjectGroupRepo.mockResolvedValue(makeProjectSchema({ id: 1 })) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateInfraGroupRepoPublicUrl.mockResolvedValue('https://gitlab.internal/repo') + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + gitlab.getUserByEmail.mockResolvedValue(makeSimpleUserSchema({ id: 123, username: 'user' })) + + await service.handleUpsert(project) + + expect(gitlab.getOrCreateProjectSubGroup).toHaveBeenCalledWith(project.slug) + expect(gitlab.getGroupMembers).toHaveBeenCalledWith(group) + expect(gitlab.getRepos).toHaveBeenCalledWith(project.slug) + }) + + it('should remove orphan member if purge enabled', async () => { + const project = makeProjectWithDetails({ + plugins: [{ key: 'purge', value: ENABLED }], + }) + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([makeMemberSchema({ id: 999, username: 'orphan' })]) + gitlab.getRepos.mockReturnValue((async function* () { })()) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.removeGroupMember).toHaveBeenCalledWith(group, 999) + }) + + it('should not remove managed user (bot) even if purge enabled', async () => { + const project = makeProjectWithDetails({ + plugins: [{ key: 'purge', value: ENABLED }], + }) + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([makeMemberSchema({ id: 888, username: 'group_123_bot' })]) + gitlab.getRepos.mockReturnValue((async function* () { })()) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.removeGroupMember).not.toHaveBeenCalled() + }) + + it('should not remove orphan member if purge disabled', async () => { + const project = makeProjectWithDetails() + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([makeMemberSchema({ id: 999, username: 'orphan' })]) + gitlab.getRepos.mockReturnValue((async function* () { })()) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.removeGroupMember).not.toHaveBeenCalled() + }) + + it('should delete orphan repositories if purge enabled', async () => { + const project = makeProjectWithDetails({ + plugins: [{ key: 'purge', value: ENABLED }], + repositories: [], + }) + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + const orphanRepo = makeProjectSchema({ name: 'orphan-repo', topics: [TOPIC_PLUGIN_MANAGED] }) + const unmanagedRepo = makeProjectSchema({ name: 'unmanaged-repo', topics: [] }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([]) + gitlab.getRepos.mockImplementation(() => (async function* () { + yield orphanRepo + yield unmanagedRepo + })()) + gitlab.deleteProjectGroupRepo.mockResolvedValue(undefined) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledWith(project.slug, 'orphan-repo') + expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledTimes(1) + }) + + it('should not delete orphan repositories if purge disabled', async () => { + const project = makeProjectWithDetails({ + repositories: [], + }) + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + const orphanRepo = makeProjectSchema({ name: 'orphan-repo', topics: [TOPIC_PLUGIN_MANAGED] }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([]) + gitlab.getRepos.mockImplementation(() => (async function* () { + yield orphanRepo + })()) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.deleteProjectGroupRepo).not.toHaveBeenCalled() + }) + + it('should not delete orphan repositories without the correct topic even if purge enabled', async () => { + const project = makeProjectWithDetails({ + plugins: [{ key: 'purge', value: ENABLED }], + repositories: [], + }) + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + const orphanRepoWithoutTopic = makeProjectSchema({ name: 'orphan-repo', topics: [] }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([]) + gitlab.getRepos.mockImplementation(() => (async function* () { + yield orphanRepoWithoutTopic + })()) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.deleteProjectGroupRepo).not.toHaveBeenCalled() + }) + + it('should create gitlab user if not exists', async () => { + const project = makeProjectWithDetails({ + members: [{ user: { id: 'u1', email: 'new@example.com', firstName: 'New', lastName: 'User', adminRoleIds: [] }, roleIds: [] }], + }) + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([]) + gitlab.getUserByEmail.mockResolvedValue(null) + gitlab.createUser.mockImplementation(async (email, username, name) => { + return makeExpandedUserSchema({ + id: email === 'new@example.com' ? 999 : 998, + email, + username, + name, + }) + }) + gitlab.getRepos.mockReturnValue((async function* () { })()) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.createUser).toHaveBeenCalledWith('new@example.com', 'new.example.com', 'New User') + expect(gitlab.createUser).toHaveBeenCalledWith('owner@example.com', 'owner.example.com', 'Owner User') + expect(gitlab.addGroupMember).toHaveBeenCalledWith(group, 999, AccessLevel.GUEST) + expect(gitlab.addGroupMember).toHaveBeenCalledWith(group, 998, AccessLevel.OWNER) + }) + + it('should configure repository mirroring if external url is present', async () => { + const project = makeProjectWithDetails({ + repositories: [{ + id: 'r1', + internalRepoName: 'repo-1', + externalRepoUrl: 'https://github.com/org/repo.git', + isPrivate: true, + externalUserName: 'user', + isInfra: false, + }], + }) + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + const gitlabRepo = makeProjectSchema({ id: 101, name: 'repo-1', path: 'repo-1', path_with_namespace: 'forge/console/project-1/repo-1' }) + const accessToken = { + id: 1, + user_id: 1, + scopes: ['read_api'], + name: 'bot', + expires_at: faker.date.future().toISOString(), + active: true, + created_at: faker.date.past().toISOString(), + revoked: false, + access_level: 40, + token: faker.internet.password(), + } satisfies AccessTokenExposedSchema + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([]) + gitlab.getRepos.mockReturnValue((async function* () { yield gitlabRepo })()) + gitlab.getOrCreateProjectGroupInternalRepoUrl.mockResolvedValue('https://gitlab.internal/group/repo-1.git') + gitlab.createMirrorAccessToken.mockResolvedValue(accessToken) + vault.readTechnReadOnlyCreds.mockResolvedValue(null) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.createMirrorAccessToken).toHaveBeenCalledWith('project-1') + expect(gitlab.upsertProjectMirrorRepo).toHaveBeenCalledWith('project-1') + + expect(vault.writeGitlabMirrorCreds).toHaveBeenCalledWith( + 'project-1', + 'repo-1', + expect.objectContaining({ + GIT_INPUT_URL: 'github.com/org/repo.git', + GIT_OUTPUT_USER: 'bot', + GIT_OUTPUT_PASSWORD: accessToken.token, + }), + ) + expect(vault.writeTechReadOnlyCreds).toHaveBeenCalledWith('project-1', { + MIRROR_USER: 'bot', + MIRROR_TOKEN: accessToken.token, + }) + }) + }) + + describe('handleCron', () => { + it('should reconcile all projects', async () => { + const projects = [makeProjectWithDetails({ id: 'p1', slug: 'project-1' })] + gitlabDatastore.getAllProjects.mockResolvedValue(projects) + + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([]) + gitlab.getRepos.mockReturnValue((async function* () { })()) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleCron() + + expect(gitlabDatastore.getAllProjects).toHaveBeenCalled() + expect(gitlab.getOrCreateProjectSubGroup).toHaveBeenCalledWith('project-1') + }) + }) +}) diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab.service.ts b/apps/server-nestjs/src/modules/gitlab/gitlab.service.ts new file mode 100644 index 000000000..e7931d367 --- /dev/null +++ b/apps/server-nestjs/src/modules/gitlab/gitlab.service.ts @@ -0,0 +1,478 @@ +import type { CondensedGroupSchema, MemberSchema, ProjectSchema } from '@gitbeaker/core' +import type { VaultSecret } from '../vault/vault-client.service' +import type { ProjectWithDetails } from './gitlab-datastore.service' +import { specificallyEnabled } from '@cpn-console/hooks' +import { AccessLevel } from '@gitbeaker/core' +import { Inject, Injectable, Logger } from '@nestjs/common' +import { OnEvent } from '@nestjs/event-emitter' +import { Cron, CronExpression } from '@nestjs/schedule' +import { trace } from '@opentelemetry/api' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' +import { getAll } from '../../utils/iterable' +import { VaultClientService } from '../vault/vault-client.service' +import { GitlabClientService } from './gitlab-client.service' +import { GitlabDatastoreService } from './gitlab-datastore.service' +import { INFRA_APPS_REPO_NAME, TOPIC_PLUGIN_MANAGED } from './gitlab.constants' +import { DEFAULT_PROJECT_DEVELOPER_GROUP_PATH_SUFFIX, DEFAULT_PROJECT_MAINTAINER_GROUP_PATH_SUFFIX, DEFAULT_PROJECT_REPORTER_GROUP_PATH_SUFFIX } from './gitlab.constants.js' + +const ownedUserRegex = /group_\d+_bot/u + +@Injectable() +export class GitlabService { + private readonly logger = new Logger(GitlabService.name) + + constructor( + @Inject(GitlabDatastoreService) private readonly gitlabDatastore: GitlabDatastoreService, + @Inject(GitlabClientService) private readonly gitlab: GitlabClientService, + @Inject(VaultClientService) private readonly vault: VaultClientService, + @Inject(ConfigurationService) private readonly config: ConfigurationService, + ) { + this.logger.log('GitlabService initialized') + } + + @OnEvent('project.upsert') + @StartActiveSpan() + async handleUpsert(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project upsert for ${project.slug}`) + await this.ensureProjectGroup(project) + } + + @OnEvent('project.delete') + @StartActiveSpan() + async handleDelete(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project delete for ${project.slug}`) + await this.ensureProjectGroup(project) + } + + @Cron(CronExpression.EVERY_HOUR) + @StartActiveSpan() + async handleCron() { + const span = trace.getActiveSpan() + span?.setAttribute('gitlab.projects.count', 0) + this.logger.log('Starting Gitlab reconciliation') + const projects = await this.gitlabDatastore.getAllProjects() + span?.setAttribute('gitlab.projects.count', projects.length) + await this.ensureProjectGroups(projects) + } + + @StartActiveSpan() + private async ensureProjectGroups(projects: ProjectWithDetails[]) { + const span = trace.getActiveSpan() + span?.setAttribute('gitlab.projects.count', projects.length) + await Promise.all(projects.map(p => this.ensureProjectGroup(p))) + } + + @StartActiveSpan() + private async ensureProjectGroup(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + const group = await this.gitlab.getOrCreateProjectSubGroup(project.slug) + const members = await this.gitlab.getGroupMembers(group) + await this.ensureProjectGroupMembers(project, group, members) + await this.ensureProjectRepos(project) + await this.purgeOrphanRepos(project) + await this.ensureSystemRepos(project) + } + + @StartActiveSpan() + private async ensureProjectGroupMembers( + project: ProjectWithDetails, + group: CondensedGroupSchema, + members: MemberSchema[], + ) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + await this.addMissingMembers(project, group, members) + await this.addMissingOwnerMember(project, group, members) + await this.purgeOrphanMembers(project, group, members) + } + + private async addMissingMembers( + project: ProjectWithDetails, + group: CondensedGroupSchema, + members: MemberSchema[], + ) { + const membersById = new Map(members.map(m => [m.id, m])) + const accessLevelByUserId = generateAccessLevelMapping(project) + + await Promise.all(project.members.map(async ({ user }) => { + const gitlabUser = await this.upsertUser(user) + if (!gitlabUser) return + const accessLevel = accessLevelByUserId.get(user.id) ?? AccessLevel.NO_ACCESS + await this.ensureGroupMemberAccessLevel(group, gitlabUser.id, accessLevel, membersById) + })) + } + + private async upsertUser(user: ProjectWithDetails['members'][number]['user']) { + return await this.gitlab.getUserByEmail(user.email) + ?? await this.gitlab.createUser( + user.email, + generateUsername(user.email), + `${user.firstName} ${user.lastName}`, + ) + } + + private async ensureGroupMemberAccessLevel( + group: CondensedGroupSchema, + gitlabUserId: number, + accessLevel: AccessLevel, + membersById: Map, + ) { + const existingMember = membersById.get(gitlabUserId) + + if (accessLevel === AccessLevel.NO_ACCESS) { + if (existingMember) { + await this.gitlab.removeGroupMember(group, gitlabUserId) + } + return + } + + if (!existingMember) { + await this.gitlab.addGroupMember(group, gitlabUserId, accessLevel) + return + } + + if (existingMember.access_level !== accessLevel) { + await this.gitlab.editGroupMember(group, gitlabUserId, accessLevel) + } + } + + private async addMissingOwnerMember( + project: ProjectWithDetails, + group: CondensedGroupSchema, + members: MemberSchema[], + ) { + const gitlabUser = await this.upsertUser(project.owner) + if (!gitlabUser) return + const membersById = new Map(members.map(m => [m.id, m])) + await this.ensureGroupMemberAccessLevel(group, gitlabUser.id, AccessLevel.OWNER, membersById) + } + + @StartActiveSpan() + private async purgeOrphanMembers( + project: ProjectWithDetails, + group: CondensedGroupSchema, + members: MemberSchema[], + ) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': project.slug, + 'group.id': group.id, + 'members.total': members.length, + }) + const purgeConfig = getPluginConfig(project, 'purge') + const usernames = new Set([ + generateUsername(project.owner.email), + ...project.members.map(m => generateUsername(m.user.email)), + ]) + const emails = new Set([ + project.owner.email.toLowerCase(), + ...project.members.map(m => m.user.email.toLowerCase()), + ]) + + const orphans = members.filter((member) => { + if (isOwnedUser(member)) return false + if (usernames.has(member.username)) return false + if (member.email && emails.has(member.email.toLowerCase())) return false + return true + }) + span?.setAttribute('orphans.count', orphans.length) + + if (specificallyEnabled(purgeConfig)) { + span?.setAttribute('purge.enabled', true) + let removedCount = 0 + await Promise.all(orphans.map(async (orphan) => { + await this.gitlab.removeGroupMember(group, orphan.id) + removedCount++ + this.logger.log(`Removed ${orphan.username} from gitlab group ${group.id}`) + })) + span?.setAttribute('orphans.removed.count', removedCount) + } else { + span?.setAttribute('purge.enabled', false) + let warnedCount = 0 + for (const orphan of orphans) { + warnedCount++ + this.logger.warn(`User ${orphan.username} is in Gitlab group but not in project (purge disabled)`) + } + span?.setAttribute('orphans.warned.count', warnedCount) + } + } + + @StartActiveSpan() + private async ensureProjectRepos(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': project.slug, + 'repositories.count': project.repositories.length, + }) + const gitlabRepositories = await getAll(this.gitlab.getRepos(project.slug)) + span?.setAttribute('gitlab.repositories.count', gitlabRepositories.length) + let mirroringEnabledCount = 0 + let mirroringDisabledCount = 0 + for (const repo of project.repositories) { + const externalHost = this.getExternalRepoHost(repo.externalRepoUrl) + span?.addEvent('gitlab.repo.reconcile', { + 'repository.name': repo.internalRepoName, + 'repository.isPrivate': repo.isPrivate, + ...(externalHost ? { 'repository.external.host': externalHost } : {}), + 'repository.external': !!repo.externalRepoUrl, + }) + await this.ensureRepository(project, repo, gitlabRepositories) + + if (repo.externalRepoUrl) { + span?.setAttribute('repository.mirroring', true) + mirroringEnabledCount++ + await this.configureRepositoryMirroring(project, repo) + } else { + span?.setAttribute('repository.mirroring', false) + mirroringDisabledCount++ + await this.vault.deleteGitlabMirrorCreds(project.slug, repo.internalRepoName) + } + } + span?.setAttribute('repositories.mirroring.enabled.count', mirroringEnabledCount) + span?.setAttribute('repositories.mirroring.disabled.count', mirroringDisabledCount) + } + + @StartActiveSpan() + private async purgeOrphanRepos(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + const gitlabRepositories = await getAll(this.gitlab.getRepos(project.slug)) + span?.setAttribute('gitlab.repositories.count', gitlabRepositories.length) + + const orphanRepos = gitlabRepositories.filter(r => isOwnedRepo(r) && !isSystemRepo(project, r)) + span?.setAttribute('orphan.repositories.count', orphanRepos.length) + + if (specificallyEnabled(getPluginConfig(project, 'purge'))) { + span?.setAttribute('purge.enabled', true) + let removedCount = 0 + await Promise.all(orphanRepos.map(async (orphan) => { + await this.gitlab.deleteProjectGroupRepo(project.slug, orphan.name) + removedCount++ + this.logger.log(`Removed ${orphan.name} from gitlab project ${project.slug}`) + })) + + span?.setAttribute('orphan.repositories.removed.count', removedCount) + } else { + span?.setAttribute('purge.enabled', false) + let warnedCount = 0 + for (const orphan of orphanRepos) { + warnedCount++ + this.logger.warn(`Repository ${orphan.name} is in Gitlab project but not in project (purge disabled)`) + } + span?.setAttribute('managed.repositories.warned.count', warnedCount) + } + } + + private async ensureRepository( + project: ProjectWithDetails, + repo: ProjectWithDetails['repositories'][number], + gitlabRepositories: ProjectSchema[], + ) { + return gitlabRepositories.find(r => r.name === repo.internalRepoName) + ?? await this.gitlab.upsertProjectGroupRepo( + project.slug, + repo.internalRepoName, + ) + } + + @StartActiveSpan() + private async configureRepositoryMirroring( + project: ProjectWithDetails, + repo: ProjectWithDetails['repositories'][number], + ) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + span?.setAttribute('repository.internalRepoName', repo.internalRepoName) + span?.setAttribute('repository.isPrivate', repo.isPrivate) + const externalHost = this.getExternalRepoHost(repo.externalRepoUrl) + if (externalHost) span?.setAttribute('repository.external.host', externalHost) + + const currentVaultSecret = await this.vault.readGitlabMirrorCreds(project.slug, repo.internalRepoName) + span?.setAttribute('vault.secret.exists', !!currentVaultSecret) + if (!currentVaultSecret) { + this.logger.warn('No existing mirror credentials found in Vault, rotating new ones', { + projectSlug: project.slug, + repoName: repo.internalRepoName, + }) + } + + const internalRepoUrl = await this.gitlab.getOrCreateProjectGroupInternalRepoUrl(project.slug, repo.internalRepoName) + const externalRepoUrn = repo.externalRepoUrl.split('://')[1] + const internalRepoUrn = internalRepoUrl.split('://')[1] + span?.setAttribute('repository.externalRepoUrn', externalRepoUrn) + span?.setAttribute('repository.internalRepoUrn', internalRepoUrn) + + const projectMirrorCreds = await this.getOrRotateMirrorCreds(project.slug) + + const mirrorSecretData = { + GIT_INPUT_URL: externalRepoUrn, + GIT_INPUT_USER: repo.isPrivate ? repo.externalUserName : undefined, + GIT_INPUT_PASSWORD: currentVaultSecret?.data?.GIT_INPUT_PASSWORD, // Preserve existing password as it's not in DB + GIT_OUTPUT_URL: internalRepoUrn, + GIT_OUTPUT_USER: projectMirrorCreds.MIRROR_USER, + GIT_OUTPUT_PASSWORD: projectMirrorCreds.MIRROR_TOKEN, + } + + // Write to vault if changed + // Using simplified check + await this.vault.writeGitlabMirrorCreds(project.slug, repo.internalRepoName, mirrorSecretData) + span?.setAttribute('vault.secret.written', true) + } + + @StartActiveSpan() + private async ensureSystemRepos(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + await Promise.all([ + this.ensureInfraAppsRepo(project.slug), + this.ensureMirrorRepo(project.slug), + ]) + } + + private async ensureInfraAppsRepo(projectSlug: string) { + await this.gitlab.upsertProjectGroupRepo(projectSlug, INFRA_APPS_REPO_NAME) + } + + private async ensureMirrorRepo(projectSlug: string) { + const mirrorRepo = await this.gitlab.upsertProjectMirrorRepo(projectSlug) + if (mirrorRepo.empty_repo) { + await this.gitlab.commitMirror(mirrorRepo.id) + } + await this.ensureMirrorRepoTriggerToken(projectSlug) + } + + @StartActiveSpan() + private async ensureMirrorRepoTriggerToken(projectSlug: string) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', projectSlug) + const triggerToken = await this.gitlab.getOrCreateMirrorPipelineTriggerToken(projectSlug) + const gitlabSecret = { + PROJECT_SLUG: projectSlug, + GIT_MIRROR_PROJECT_ID: triggerToken.repoId, + GIT_MIRROR_TOKEN: triggerToken.token, + } + await this.vault.writeMirrorTriggerToken(gitlabSecret) + span?.setAttribute('vault.secret.written', true) + } + + @StartActiveSpan() + private async getOrRotateMirrorCreds(projectSlug: string) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', projectSlug) + const vaultSecret = await this.vault.readTechnReadOnlyCreds(projectSlug) + if (!vaultSecret) return this.createMirrorAccessToken(projectSlug) + + const isExpiring = this.isMirrorCredsExpiring(vaultSecret) + span?.setAttribute('mirror.creds.expiring', isExpiring) + if (!isExpiring) { + span?.setAttribute('mirror.creds.rotated', false) + return vaultSecret.data as { MIRROR_USER: string, MIRROR_TOKEN: string } + } + return this.createMirrorAccessToken(projectSlug) + } + + @StartActiveSpan() + private async createMirrorAccessToken(projectSlug: string) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', projectSlug) + span?.setAttribute('mirror.creds.rotated', true) + const token = await this.gitlab.createMirrorAccessToken(projectSlug) + const creds = { + MIRROR_USER: token.name, + MIRROR_TOKEN: token.token, + } + await this.vault.writeTechReadOnlyCreds(projectSlug, creds) + span?.setAttribute('vault.secret.written', true) + return creds + } + + private isMirrorCredsExpiring(vaultSecret: VaultSecret): boolean { + if (!vaultSecret?.metadata?.created_time) return false + const createdTime = new Date(vaultSecret.metadata.created_time) + return daysAgoFromNow(createdTime) > this.config.gitlabMirrorTokenRotationThresholdDays + } + + private getExternalRepoHost(externalRepoUrl: string | null | undefined): string | undefined { + if (!externalRepoUrl) return undefined + try { + return new URL(externalRepoUrl).host + } catch { + return undefined + } + } +} + +function isOwnedUser(member: MemberSchema) { + return ownedUserRegex.test(member.username) +} + +function isOwnedRepo(repo: ProjectSchema) { + return repo.topics?.includes(TOPIC_PLUGIN_MANAGED) ?? false +} + +function isSystemRepo(project: ProjectWithDetails, repo: ProjectSchema) { + return project.repositories.some(r => r.internalRepoName === repo.name) +} + +function generateUsername(email: string) { + return email.replace('@', '.') +} + +function getPluginConfig(project: ProjectWithDetails, key: string) { + return project.plugins?.find(p => p.key === key)?.value +} + +function getGroupPathSuffixes(project: ProjectWithDetails, key: string) { + const value = getPluginConfig(project, key) + if (!value) return null + return value.split(',').map(path => `/${project.slug}${path}`) +} + +function generateAccessLevelMapping(project: ProjectWithDetails) { + const projectReporterGroupPathSuffixes = getProjectReporterGroupPaths(project) + const projectDeveloperGroupPathSuffixes = getProjectDeveloperGroupPaths(project) + const projectMaintainerGroupPathSuffixes = getProjectMaintainerGroupPaths(project) + + const getAccessLevelFromOidcGroup = (oidcGroup: string | null) => { + if (!oidcGroup) return null + if (projectReporterGroupPathSuffixes.includes(oidcGroup)) return AccessLevel.REPORTER + if (projectDeveloperGroupPathSuffixes.includes(oidcGroup)) return AccessLevel.DEVELOPER + if (projectMaintainerGroupPathSuffixes.includes(oidcGroup)) return AccessLevel.MAINTAINER + return null + } + + const roleAccessLevelById = new Map( + project.roles.map(role => [role.id, getAccessLevelFromOidcGroup(role.oidcGroup)]), + ) + + return new Map(project.members.map((membership) => { + let highest = AccessLevel.GUEST + for (const roleId of membership.roleIds) { + const level = roleAccessLevelById.get(roleId) + if (level !== null && level !== undefined && level > highest) highest = level + } + return [membership.user.id, highest] as const + })) +} + +function getProjectMaintainerGroupPaths(project: ProjectWithDetails) { + return getGroupPathSuffixes(project, 'projectMaintainerGroupPathSuffix') ?? DEFAULT_PROJECT_MAINTAINER_GROUP_PATH_SUFFIX.split(',') +} + +function getProjectDeveloperGroupPaths(project: ProjectWithDetails) { + return getGroupPathSuffixes(project, 'projectDeveloperGroupPathSuffix') ?? DEFAULT_PROJECT_DEVELOPER_GROUP_PATH_SUFFIX.split(',') +} + +function getProjectReporterGroupPaths(project: ProjectWithDetails) { + return getGroupPathSuffixes(project, 'projectReporterGroupPathSuffix') ?? DEFAULT_PROJECT_REPORTER_GROUP_PATH_SUFFIX.split(',') +} + +function daysAgoFromNow(date: Date) { + return Math.floor((Date.now() - date.getTime()) / (1000 * 60 * 60 * 24)) +} diff --git a/apps/server-nestjs/src/modules/healthz/healthz.controller.ts b/apps/server-nestjs/src/modules/healthz/healthz.controller.ts index 68724cd78..a81ce3caf 100644 --- a/apps/server-nestjs/src/modules/healthz/healthz.controller.ts +++ b/apps/server-nestjs/src/modules/healthz/healthz.controller.ts @@ -1,6 +1,9 @@ import { Controller, Get, Inject } from '@nestjs/common' import { HealthCheck, HealthCheckService } from '@nestjs/terminus' import { DatabaseHealthService } from '../../cpin-module/infrastructure/database/database-health.service' +import { ArgoCDHealthService } from '../argocd/argocd-health.service' +import { GitlabHealthService } from '../gitlab/gitlab-health.service' +import { KeycloakHealthService } from '../keycloak/keycloak-health.service' import { KeycloakHealthService } from '../keycloak/keycloak-health.service' @Controller('api/v1/healthz') @@ -9,6 +12,9 @@ export class HealthzController { @Inject(HealthCheckService) private readonly health: HealthCheckService, @Inject(DatabaseHealthService) private readonly database: DatabaseHealthService, @Inject(KeycloakHealthService) private readonly keycloak: KeycloakHealthService, + @Inject(GitlabHealthService) private readonly gitlab: GitlabHealthService, + @Inject(VaultHealthService) private readonly vault: VaultHealthService, + @Inject(ArgoCDHealthService) private readonly argocd: ArgoCDHealthService, ) {} @Get() @@ -17,6 +23,9 @@ export class HealthzController { return this.health.check([ () => this.database.check('database'), () => this.keycloak.check('keycloak'), + () => this.gitlab.check('gitlab'), + () => this.vault.check('vault'), + () => this.argocd.check('argocd'), ]) } } diff --git a/apps/server-nestjs/src/modules/healthz/healthz.module.ts b/apps/server-nestjs/src/modules/healthz/healthz.module.ts index 7e7307126..b060843d9 100644 --- a/apps/server-nestjs/src/modules/healthz/healthz.module.ts +++ b/apps/server-nestjs/src/modules/healthz/healthz.module.ts @@ -1,5 +1,8 @@ import { Module } from '@nestjs/common' import { TerminusModule } from '@nestjs/terminus' +import { DatabaseModule } from '../../cpin-module/infrastructure/database/database.module' +import { ArgoCDModule } from '../argocd/argocd.module' +import { GitlabModule } from '../gitlab/gitlab.module' import { KeycloakModule } from '../keycloak/keycloak.module' import { DatabaseModule } from '../../cpin-module/infrastructure/database/database.module' import { HealthzController } from './healthz.controller' @@ -9,6 +12,9 @@ import { HealthzController } from './healthz.controller' TerminusModule, DatabaseModule, KeycloakModule, + GitlabModule, + VaultModule, + ArgoCDModule, ], controllers: [HealthzController], }) diff --git a/apps/server-nestjs/src/modules/vault/vault-client.service.spec.ts b/apps/server-nestjs/src/modules/vault/vault-client.service.spec.ts new file mode 100644 index 000000000..59f96daa3 --- /dev/null +++ b/apps/server-nestjs/src/modules/vault/vault-client.service.spec.ts @@ -0,0 +1,119 @@ +import { Test } from '@nestjs/testing' +import { http, HttpResponse } from 'msw' +import { setupServer } from 'msw/node' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { VaultClientService, VaultError } from './vault-client.service' + +const vaultUrl = 'https://vault.internal' + +const server = setupServer( + http.post(`${vaultUrl}/v1/auth/token/create`, () => { + return HttpResponse.json({ auth: { client_token: 'token' } }) + }), + http.get(`${vaultUrl}/v1/kv/data/:path`, () => { + return HttpResponse.json({ data: { data: { secret: 'value' }, metadata: { created_time: '2023-01-01T00:00:00.000Z', version: 1 } } }) + }), + http.post(`${vaultUrl}/v1/kv/data/:path`, () => { + return HttpResponse.json({}) + }), + http.delete(`${vaultUrl}/v1/kv/metadata/:path`, () => { + return new HttpResponse(null, { status: 204 }) + }), +) + +function createVaultServiceTestingModule() { + return Test.createTestingModule({ + providers: [ + VaultClientService, + { + provide: ConfigurationService, + useValue: { + vaultToken: 'token', + vaultUrl, + vaultInternalUrl: vaultUrl, + vaultKvName: 'kv', + } satisfies Partial, + }, + ], + }) +} + +describe('vault', () => { + let service: VaultClientService + + beforeAll(() => server.listen()) + beforeEach(async () => { + const module = await createVaultServiceTestingModule().compile() + service = module.get(VaultClientService) + }) + afterEach(() => server.resetHandlers()) + afterAll(() => server.close()) + + describe('getProjectValues', () => { + it('should get project values', async () => { + const result = await service.readProjectValues('project-id') + expect(result).toEqual({ secret: 'value' }) + }) + + it('should return empty object if undefined', async () => { + server.use( + http.get(`${vaultUrl}/v1/kv/data/:path`, () => { + return HttpResponse.json({}, { status: 404 }) + }), + ) + + const result = await service.readProjectValues('project-id') + expect(result).toEqual(undefined) + }) + }) + + describe('read', () => { + it('should read secret', async () => { + const result = await service.read('path') + expect(result).toEqual({ + data: { secret: 'value' }, + metadata: { created_time: '2023-01-01T00:00:00.000Z', version: 1 }, + }) + }) + + it('should throw if 404', async () => { + server.use( + http.get(`${vaultUrl}/v1/kv/data/:path`, () => { + return HttpResponse.json({}, { status: 404 }) + }), + ) + + await expect(service.read('path')).rejects.toBeInstanceOf(VaultError) + await expect(service.read('path')).rejects.toMatchObject({ kind: 'NotFound', status: 404 }) + }) + }) + + describe('write', () => { + it('should write secret', async () => { + await expect(service.write({ secret: 'value' }, 'path')).resolves.toBeUndefined() + }) + + it('should expose reasons on error', async () => { + server.use( + http.post(`${vaultUrl}/v1/kv/data/:path`, () => { + return HttpResponse.json({ errors: ['No secret engine mount at test-project/'] }, { status: 400 }) + }), + ) + + await expect(service.write({ secret: 'value' }, 'path')).rejects.toBeInstanceOf(VaultError) + await expect(service.write({ secret: 'value' }, 'path')).rejects.toMatchObject({ + kind: 'HttpError', + status: 400, + reasons: ['No secret engine mount at test-project/'], + }) + await expect(service.write({ secret: 'value' }, 'path')).rejects.toThrow('Request failed') + }) + }) + + describe('delete', () => { + it('should delete secret', async () => { + await expect(service.delete('path')).resolves.toBeUndefined() + }) + }) +}) diff --git a/apps/server-nestjs/src/modules/vault/vault-client.service.ts b/apps/server-nestjs/src/modules/vault/vault-client.service.ts new file mode 100644 index 000000000..8210c2850 --- /dev/null +++ b/apps/server-nestjs/src/modules/vault/vault-client.service.ts @@ -0,0 +1,478 @@ +import { Inject, Injectable } from '@nestjs/common' +import { trace } from '@opentelemetry/api' +import z from 'zod' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' +import { generateGitlabMirrorCredPath, generateProjectPath, generateTechReadOnlyCredPath } from './vault.utils' + +interface VaultSysPoliciesAclUpsertRequest { + policy: string +} + +interface VaultSysMountCreateRequest { + type: string + config: { + force_no_cache: boolean + } + options: { + version: number + } +} + +interface VaultSysMountTuneRequest { + options: { + version: number + } +} + +interface VaultAuthApproleRoleUpsertRequest { + secret_id_num_uses: string + secret_id_ttl: string + token_max_ttl: string + token_num_uses: string + token_ttl: string + token_type: string + token_policies: string[] +} + +interface VaultIdentityGroupUpsertRequest { + name: string + type: string + policies: string[] +} + +interface VaultIdentityGroupAliasCreateRequest { + name: string + mount_accessor: string + canonical_id: string +} + +interface VaultFetchOptions { + method?: string + body?: unknown +} + +interface VaultAuthMethod { + accessor: string + type: string + description?: string +} + +interface VaultSysAuthResponse { + data: Record +} + +interface VaultIdentityGroupResponse { + data: { + id: string + name: string + alias?: { + id?: string + name?: string + } + } +} + +export interface VaultMetadata { + created_time: string + custom_metadata: Record | null + deletion_time: string + destroyed: boolean + version: number +} + +export interface VaultSecret { + data: T + metadata: VaultMetadata +} + +export interface VaultResponse { + data: VaultSecret +} + +export type VaultErrorKind + = | 'NotConfigured' + | 'NotFound' + | 'HttpError' + | 'InvalidResponse' + | 'ParseError' + | 'Unexpected' + +export class VaultError extends Error { + readonly kind: VaultErrorKind + readonly status?: number + readonly method?: string + readonly path?: string + readonly statusText?: string + readonly reasons?: string[] + + constructor( + kind: VaultErrorKind, + message: string, + details: { status?: number, method?: string, path?: string, statusText?: string, reasons?: string[] } = {}, + ) { + super(message) + this.name = 'VaultError' + this.kind = kind + this.status = details.status + this.method = details.method + this.path = details.path + this.statusText = details.statusText + this.reasons = details.reasons + } +} + +interface VaultListResponse { + data: { + keys: string[] + } +} + +interface VaultRoleIdResponse { + data: { + role_id: string + } +} + +interface VaultSecretIdResponse { + data: { + secret_id: string + } +} + +@Injectable() +export class VaultClientService { + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + ) { + } + + @StartActiveSpan() + private async fetch( + path: string, + options: VaultFetchOptions = {}, + ): Promise { + const span = trace.getActiveSpan() + const method = options.method ?? 'GET' + span?.setAttribute('vault.method', method) + span?.setAttribute('vault.path', path) + + const request = this.createRequest(path, method, options.body) + const response = await fetch(request).catch((error) => { + throw new VaultError( + 'Unexpected', + error instanceof Error ? error.message : String(error), + { method, path }, + ) + }) + span?.setAttribute('vault.http.status', response.status) + + return await this.handleResponse(response, method, path) + } + + private get baseUrl() { + if (!this.config.vaultInternalUrl) { + throw new VaultError('NotConfigured', 'VAULT_INTERNAL_URL is required') + } + return this.config.vaultInternalUrl + } + + private get token() { + if (!this.config.vaultToken) { + throw new VaultError('NotConfigured', 'VAULT_TOKEN is required') + } + return this.config.vaultToken + } + + private createRequest(path: string, method: string, body?: unknown): Request { + const url = new URL(path, this.baseUrl).toString() + const headers: Record = { + 'Content-Type': 'application/json', + 'X-Vault-Token': this.token, + } + + return new Request(url, { method, headers, body: body === undefined ? undefined : JSON.stringify(body) }) + } + + private async handleResponse(response: Response, method: string, path: string): Promise { + if (response.status === 204) return null + + if (!response.ok) { + await this.throwForStatus(response, method, path) + } + + return await response.json() + } + + private async throwForStatus(response: Response, method: string, path: string): Promise { + const responseBody = await response.json() + const vaultErrorBody = z.object({ errors: z.array(z.string()) }).safeParse(responseBody) + const reasons = vaultErrorBody.success ? vaultErrorBody.data.errors : undefined + + if (response.status === 404) { + throw new VaultError('NotFound', 'Not Found', { + status: 404, + method, + path, + statusText: response.statusText, + reasons, + }) + } + + throw new VaultError('HttpError', 'Request failed', { + status: response.status, + method, + path, + statusText: response.statusText, + reasons, + }) + } + + @StartActiveSpan() + async getKvData(kvName: string, path: string): Promise> { + const span = trace.getActiveSpan() + span?.setAttribute('vault.kv.name', kvName) + span?.setAttribute('vault.kv.path', path) + const response = await this.fetch>(`/v1/${kvName}/data/${path}`, { method: 'GET' }) + if (!response?.data) { + throw new VaultError('InvalidResponse', 'Missing "data" field', { method: 'GET', path: `/v1/${kvName}/data/${path}` }) + } + return response.data + } + + @StartActiveSpan() + async upsertKvData(kvName: string, path: string, body: { data: T }): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('vault.kv.name', kvName) + span?.setAttribute('vault.kv.path', path) + await this.fetch(`/v1/${kvName}/data/${path}`, { method: 'POST', body }) + } + + @StartActiveSpan() + async read(path: string): Promise> { + return await this.getKvData(this.config.vaultKvName, path) + } + + @StartActiveSpan() + async write(data: T, path: string): Promise { + await this.upsertKvData(this.config.vaultKvName, path, { data }) + } + + @StartActiveSpan() + async delete(path: string): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('vault.kv.path', path) + return await this.deleteKvMetadata(this.config.vaultKvName, path) + } + + @StartActiveSpan() + async readProjectValues(projectId: string): Promise | undefined> { + const path = generateProjectPath(this.config.projectRootPath, projectId) + const secret = await this.read>(path).catch((error) => { + if (error instanceof VaultError && error.kind === 'NotFound') return null + throw error + }) + return secret?.data + } + + @StartActiveSpan() + async readGitlabMirrorCreds(projectSlug: string, repoName: string): Promise { + const vaultCredsPath = generateGitlabMirrorCredPath(this.config.projectRootPath, projectSlug, repoName) + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'repo.name': repoName, + 'vault.kv.path': vaultCredsPath, + }) + return await this.read(vaultCredsPath).catch((error) => { + if (error instanceof VaultError && error.kind === 'NotFound') return null + throw error + }) + } + + @StartActiveSpan() + async writeGitlabMirrorCreds(projectSlug: string, repoName: string, data: Record): Promise { + const vaultCredsPath = generateGitlabMirrorCredPath(this.config.projectRootPath, projectSlug, repoName) + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'repo.name': repoName, + 'vault.kv.path': vaultCredsPath, + }) + await this.write(data, vaultCredsPath) + } + + @StartActiveSpan() + async deleteGitlabMirrorCreds(projectSlug: string, repoName: string): Promise { + const vaultCredsPath = generateGitlabMirrorCredPath(this.config.projectRootPath, projectSlug, repoName) + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'repo.name': repoName, + 'vault.kv.path': vaultCredsPath, + }) + await this.delete(vaultCredsPath).catch((error) => { + if (error instanceof VaultError && error.kind === 'NotFound') return + throw error + }) + } + + @StartActiveSpan() + async readTechnReadOnlyCreds(projectSlug: string): Promise { + const vaultPath = generateTechReadOnlyCredPath(this.config.projectRootPath, projectSlug) + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'vault.kv.path': vaultPath, + }) + return await this.read(vaultPath).catch((error) => { + if (error instanceof VaultError && error.kind === 'NotFound') return null + throw error + }) + } + + @StartActiveSpan() + async writeTechReadOnlyCreds(projectSlug: string, creds: Record): Promise { + const vaultPath = generateTechReadOnlyCredPath(this.config.projectRootPath, projectSlug) + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'vault.kv.path': vaultPath, + }) + await this.write(creds, vaultPath) + } + + @StartActiveSpan() + async writeMirrorTriggerToken(secret: Record): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('vault.kv.path', 'GITLAB') + await this.write(secret, 'GITLAB') + } + + @StartActiveSpan() + async deleteKvMetadata(kvName: string, path: string): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('vault.kv.name', kvName) + span?.setAttribute('vault.kv.path', path) + try { + await this.fetch(`/v1/${kvName}/metadata/${path}`, { method: 'DELETE' }) + } catch (error) { + if (error instanceof VaultError && error.kind === 'NotFound') return + throw error + } + } + + @StartActiveSpan() + async listKvMetadata(kvName: string, path: string): Promise { + try { + const span = trace.getActiveSpan() + span?.setAttribute('vault.kv.name', kvName) + span?.setAttribute('vault.kv.path', path) + const response = await this.fetch(`/v1/${kvName}/metadata/${path}`, { method: 'LIST' }) + if (!response?.data?.keys) { + throw new VaultError('InvalidResponse', 'Missing "data.keys" field', { method: 'LIST', path: `/v1/${kvName}/metadata/${path}` }) + } + return response.data.keys + } catch (error) { + if (error instanceof VaultError && error.kind === 'NotFound') return [] + throw error + } + } + + @StartActiveSpan() + async upsertSysPoliciesAcl(policyName: string, body: VaultSysPoliciesAclUpsertRequest): Promise { + await this.fetch(`/v1/sys/policies/acl/${policyName}`, { method: 'POST', body }) + } + + @StartActiveSpan() + async deleteSysPoliciesAcl(policyName: string): Promise { + await this.fetch(`/v1/sys/policies/acl/${policyName}`, { method: 'DELETE' }) + } + + @StartActiveSpan() + async createSysMount(name: string, body: VaultSysMountCreateRequest): Promise { + await this.fetch(`/v1/sys/mounts/${name}`, { method: 'POST', body }) + } + + @StartActiveSpan() + async tuneSysMount(name: string, body: VaultSysMountTuneRequest): Promise { + await this.fetch(`/v1/sys/mounts/${name}/tune`, { method: 'POST', body }) + } + + @StartActiveSpan() + async deleteSysMounts(name: string): Promise { + await this.fetch(`/v1/sys/mounts/${name}`, { method: 'DELETE' }) + } + + @StartActiveSpan() + async upsertAuthApproleRole(roleName: string, body: VaultAuthApproleRoleUpsertRequest): Promise { + await this.fetch(`/v1/auth/approle/role/${roleName}`, { + method: 'POST', + body, + }) + } + + @StartActiveSpan() + async deleteAuthApproleRole(roleName: string): Promise { + await this.fetch(`/v1/auth/approle/role/${roleName}`, { method: 'DELETE' }) + } + + async getAuthApproleRoleRoleId(roleName: string): Promise { + const path = `/v1/auth/approle/role/${roleName}/role-id` + const response = await this.fetch(path, { method: 'GET' }) + const roleId = response?.data?.role_id + if (!roleId) { + throw new VaultError('InvalidResponse', `Vault role-id not found for role ${roleName}`, { method: 'GET', path }) + } + return roleId + } + + @StartActiveSpan() + async createAuthApproleRoleSecretId(roleName: string): Promise { + const path = `/v1/auth/approle/role/${roleName}/secret-id` + const response = await this.fetch(path, { method: 'POST' }) + const secretId = response?.data?.secret_id + if (!secretId) { + throw new VaultError('InvalidResponse', `Vault secret-id not generated for role ${roleName}`, { method: 'POST', path }) + } + return secretId + } + + async getSysAuth(): Promise> { + const path = '/v1/sys/auth' + const response = await this.fetch(path, { method: 'GET' }) + return response?.data ?? {} + } + + @StartActiveSpan() + async upsertIdentityGroupName(groupName: string, body: VaultIdentityGroupUpsertRequest): Promise { + await this.fetch(`/v1/identity/group/name/${groupName}`, { + method: 'POST', + body, + }) + } + + @StartActiveSpan() + async getIdentityGroupName(groupName: string): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('vault.identity.group.name', groupName) + const path = `/v1/identity/group/name/${groupName}` + const response = await this.fetch(path, { method: 'GET' }) + if (!response) throw new VaultError('InvalidResponse', 'Empty response', { method: 'GET', path }) + return response + } + + @StartActiveSpan() + async deleteIdentityGroupName(groupName: string): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('vault.identity.group.name', groupName) + await this.fetch(`/v1/identity/group/name/${groupName}`, { method: 'DELETE' }) + } + + @StartActiveSpan() + async createIdentityGroupAlias(body: VaultIdentityGroupAliasCreateRequest): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('vault.identity.group.alias', body.name) + await this.fetch('/v1/identity/group-alias', { method: 'POST', body }) + } +} diff --git a/apps/server-nestjs/src/modules/vault/vault-datastore.service.ts b/apps/server-nestjs/src/modules/vault/vault-datastore.service.ts new file mode 100644 index 000000000..932b4bbba --- /dev/null +++ b/apps/server-nestjs/src/modules/vault/vault-datastore.service.ts @@ -0,0 +1,58 @@ +import type { Prisma } from '@prisma/client' +import { Inject, Injectable } from '@nestjs/common' +import { PrismaService } from '../../cpin-module/infrastructure/database/prisma.service' + +export const projectSelect = { + id: true, + name: true, + slug: true, + description: true, + environments: { + select: { + id: true, + name: true, + clusterId: true, + cpu: true, + gpu: true, + memory: true, + autosync: true, + }, + }, +} satisfies Prisma.ProjectSelect + +export type ProjectWithDetails = Prisma.ProjectGetPayload<{ + select: typeof projectSelect +}> + +export const zoneSelect = { + id: true, + slug: true, +} satisfies Prisma.ZoneSelect + +export type ZoneWithDetails = Prisma.ZoneGetPayload<{ + select: typeof zoneSelect +}> + +@Injectable() +export class VaultDatastoreService { + constructor(@Inject(PrismaService) private readonly prisma: PrismaService) {} + + async getAllProjects(): Promise { + return this.prisma.project.findMany({ + select: projectSelect, + }) + } + + async getProject(id: string): Promise { + return this.prisma.project.findUnique({ + where: { id }, + select: projectSelect, + }) + } + + async getAllZones(): Promise { + return this.prisma.zone.findMany({ + select: zoneSelect, + }) + } +} diff --git a/apps/server-nestjs/src/modules/vault/vault-health.service.ts b/apps/server-nestjs/src/modules/vault/vault-health.service.ts new file mode 100644 index 000000000..4748dd642 --- /dev/null +++ b/apps/server-nestjs/src/modules/vault/vault-health.service.ts @@ -0,0 +1,25 @@ +import { Inject, Injectable } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' + +@Injectable() +export class VaultHealthService { + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(HealthIndicatorService) private readonly healthIndicator: HealthIndicatorService, + ) {} + + async check(key: string) { + const indicator = this.healthIndicator.check(key) + if (!this.config.vaultInternalUrl) return indicator.down('Not configured') + + const url = new URL('/v1/sys/health', this.config.vaultInternalUrl).toString() + try { + const response = await fetch(url, { method: 'GET' }) + if (response.status < 500) return indicator.up({ httpStatus: response.status }) + return indicator.down({ httpStatus: response.status }) + } catch (error) { + return indicator.down(error instanceof Error ? error.message : String(error)) + } + } +} diff --git a/apps/server-nestjs/src/modules/vault/vault.module.ts b/apps/server-nestjs/src/modules/vault/vault.module.ts new file mode 100644 index 000000000..9491ee8fb --- /dev/null +++ b/apps/server-nestjs/src/modules/vault/vault.module.ts @@ -0,0 +1,21 @@ +import { Module } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationModule } from '../../cpin-module/infrastructure/configuration/configuration.module' +import { InfrastructureModule } from '../../cpin-module/infrastructure/infrastructure.module' +import { VaultClientService } from './vault-client.service' +import { VaultDatastoreService } from './vault-datastore.service' +import { VaultHealthService } from './vault-health.service' +import { VaultService } from './vault.service' + +@Module({ + imports: [ConfigurationModule, InfrastructureModule], + providers: [ + HealthIndicatorService, + VaultHealthService, + VaultClientService, + VaultService, + VaultDatastoreService, + ], + exports: [VaultClientService, VaultHealthService], +}) +export class VaultModule {} diff --git a/apps/server-nestjs/src/modules/vault/vault.service.spec.ts b/apps/server-nestjs/src/modules/vault/vault.service.spec.ts new file mode 100644 index 000000000..9a3d482b6 --- /dev/null +++ b/apps/server-nestjs/src/modules/vault/vault.service.spec.ts @@ -0,0 +1,94 @@ +import type { TestingModule } from '@nestjs/testing' +import type { Mocked } from 'vitest' +import { Test } from '@nestjs/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { VaultClientService } from './vault-client.service' +import { VaultDatastoreService } from './vault-datastore.service' +import { VaultService } from './vault.service' + +function createVaultControllerServiceTestingModule() { + return Test.createTestingModule({ + providers: [ + VaultService, + { + provide: VaultClientService, + useValue: { + createSysMount: vi.fn().mockResolvedValue(undefined), + tuneSysMount: vi.fn().mockResolvedValue(undefined), + deleteSysMounts: vi.fn().mockResolvedValue(undefined), + upsertSysPoliciesAcl: vi.fn().mockResolvedValue(undefined), + deleteSysPoliciesAcl: vi.fn().mockResolvedValue(undefined), + upsertAuthApproleRole: vi.fn().mockResolvedValue(undefined), + deleteAuthApproleRole: vi.fn().mockResolvedValue(undefined), + upsertIdentityGroupName: vi.fn().mockResolvedValue(undefined), + getIdentityGroupName: vi.fn().mockResolvedValue({ data: { id: 'gid', name: 'p1', alias: { name: '/p1' } } }), + deleteIdentityGroupName: vi.fn().mockResolvedValue(undefined), + getSysAuth: vi.fn().mockResolvedValue({ 'oidc/': { accessor: 'oidc-accessor', type: 'oidc' } }), + createIdentityGroupAlias: vi.fn().mockResolvedValue(undefined), + listKvMetadata: vi.fn().mockResolvedValue([]), + delete: vi.fn().mockResolvedValue(undefined), + } satisfies Partial, + }, + { + provide: VaultDatastoreService, + useValue: { + getAllProjects: vi.fn(), + getAllZones: vi.fn(), + } satisfies Partial, + }, + { + provide: ConfigurationService, + useValue: { + projectRootPath: 'forge', + vaultKvName: 'kv', + } satisfies Partial, + }, + ], + }) +} + +describe('vaultService', () => { + let service: VaultService + let datastore: Mocked + let client: Mocked + + beforeEach(async () => { + const module: TestingModule = await createVaultControllerServiceTestingModule().compile() + service = module.get(VaultService) + datastore = module.get(VaultDatastoreService) + client = module.get(VaultClientService) + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) + it('should reconcile on cron', async () => { + datastore.getAllProjects.mockResolvedValue([{ slug: 'p1' }, { slug: 'p2' }] as any) + datastore.getAllZones.mockResolvedValue([{ slug: 'z1' }] as any) + + await service.handleCron() + + expect(datastore.getAllProjects).toHaveBeenCalled() + expect(datastore.getAllZones).toHaveBeenCalled() + expect(client.createSysMount).toHaveBeenCalledTimes(3) + expect(client.createSysMount).toHaveBeenCalledWith('p1', expect.any(Object)) + expect(client.createSysMount).toHaveBeenCalledWith('p2', expect.any(Object)) + expect(client.createSysMount).toHaveBeenCalledWith('zone-z1', expect.any(Object)) + }) + + it('should upsert project on event', async () => { + await service.handleUpsert({ slug: 'p1' } as any) + expect(client.createSysMount).toHaveBeenCalledWith('p1', expect.any(Object)) + }) + + it('should delete project and destroy secrets on event', async () => { + client.listKvMetadata.mockResolvedValue([]) + await service.handleDelete({ slug: 'p1' } as any) + expect(client.deleteSysMounts).toHaveBeenCalledWith('p1') + expect(client.deleteSysPoliciesAcl).toHaveBeenCalledWith('app--p1--admin') + expect(client.deleteSysPoliciesAcl).toHaveBeenCalledWith('tech--p1--ro') + expect(client.deleteAuthApproleRole).toHaveBeenCalledWith('p1') + expect(client.deleteIdentityGroupName).toHaveBeenCalledWith('p1') + }) +}) diff --git a/apps/server-nestjs/src/modules/vault/vault.service.ts b/apps/server-nestjs/src/modules/vault/vault.service.ts new file mode 100644 index 000000000..88900f350 --- /dev/null +++ b/apps/server-nestjs/src/modules/vault/vault.service.ts @@ -0,0 +1,355 @@ +import type { ProjectWithDetails, ZoneWithDetails } from './vault-datastore.service' +import { Inject, Injectable, Logger } from '@nestjs/common' +import { OnEvent } from '@nestjs/event-emitter' +import { Cron, CronExpression } from '@nestjs/schedule' +import { trace } from '@opentelemetry/api' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' +import { VaultClientService, VaultError } from './vault-client.service' +import { VaultDatastoreService } from './vault-datastore.service' +import { generateProjectPath } from './vault.utils' + +@Injectable() +export class VaultService { + private readonly logger = new Logger(VaultService.name) + + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(VaultDatastoreService) private readonly vaultDatastore: VaultDatastoreService, + @Inject(VaultClientService) private readonly client: VaultClientService, + ) { + this.logger.log('VaultService initialized') + } + + @OnEvent('project.upsert') + @StartActiveSpan() + async handleUpsert(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project upsert for ${project.slug}`) + await this.ensureProject(project) + } + + @OnEvent('project.delete') + @StartActiveSpan() + async handleDelete(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project delete for ${project.slug}`) + await Promise.all([ + this.deleteProject(project.slug), + this.deleteProjectSecrets(project.slug), + ]) + } + + @OnEvent('zone.upsert') + @StartActiveSpan() + async handleUpsertZone(zone: ZoneWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('zone.slug', zone.slug) + this.logger.log(`Handling zone upsert for ${zone.slug}`) + await this.ensureZone(zone) + } + + @OnEvent('zone.delete') + @StartActiveSpan() + async handleDeleteZone(zone: ZoneWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('zone.slug', zone.slug) + this.logger.log(`Handling zone delete for ${zone.slug}`) + await this.deleteZone(zone.slug) + } + + @Cron(CronExpression.EVERY_HOUR) + @StartActiveSpan() + async handleCron() { + const span = trace.getActiveSpan() + this.logger.log('Starting Vault reconciliation') + const [projects, zones] = await Promise.all([ + this.vaultDatastore.getAllProjects(), + this.vaultDatastore.getAllZones(), + ]) + + span?.setAttributes({ + 'vault.projects.count': projects.length, + 'vault.zones.count': zones.length, + }) + await Promise.all([ + this.ensureProjects(projects), + this.ensureZones(zones), + ]) + } + + @StartActiveSpan() + private async ensureProjects(projects: ProjectWithDetails[]) { + const span = trace.getActiveSpan() + span?.setAttribute('vault.projects.count', projects.length) + await Promise.all(projects.map(p => this.ensureProject(p))) + } + + @StartActiveSpan() + private async ensureProject(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + await this.upsertProject(project) + } + + @StartActiveSpan() + private async ensureZones(zones: ZoneWithDetails[]) { + const span = trace.getActiveSpan() + span?.setAttribute('vault.zones.count', zones.length) + await Promise.all(zones.map(z => this.ensureZone(z))) + } + + @StartActiveSpan() + private async ensureZone(zone: ZoneWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('zone.slug', zone.slug) + await this.upsertZone(zone.slug) + } + + private async upsertMount(kvName: string): Promise { + const createBody = { + type: 'kv', + config: { + force_no_cache: true, + }, + options: { + version: 2, + }, + } + const tuneBody = { + options: { + version: 2, + }, + } + try { + await this.client.createSysMount(kvName, createBody) + } catch (error) { + if (error instanceof VaultError && error.kind === 'HttpError' && error.status === 400) { + await this.client.tuneSysMount(kvName, tuneBody) + return + } + throw error + } + } + + private async deleteMount(kvName: string): Promise { + try { + await this.client.deleteSysMounts(kvName) + } catch (error) { + if (error instanceof VaultError && error.kind === 'NotFound') return + throw error + } + } + + @StartActiveSpan() + async upsertZone(zoneName: string): Promise { + const kvName = generateZoneName(zoneName) + const span = trace.getActiveSpan() + span?.setAttribute('zone.name', zoneName) + span?.setAttribute('vault.kv.name', kvName) + const policyName = generateZoneTechReadOnlyPolicyName(zoneName) + + await this.upsertMount(kvName) + await this.client.upsertSysPoliciesAcl(policyName, { + policy: `path "${kvName}/*" { capabilities = ["read"] }`, + }) + await this.client.upsertAuthApproleRole(kvName, generateApproleRoleBody([policyName])) + } + + @StartActiveSpan() + async deleteZone(zoneName: string): Promise { + const kvName = generateZoneName(zoneName) + const span = trace.getActiveSpan() + span?.setAttribute('zone.name', zoneName) + span?.setAttribute('vault.kv.name', kvName) + const policyName = generateZoneTechReadOnlyPolicyName(zoneName) + const roleName = kvName + + await this.deleteMount(kvName) + + const settled = await Promise.allSettled([ + this.client.deleteSysPoliciesAcl(policyName), + this.client.deleteAuthApproleRole(roleName), + ]) + + for (const result of settled) { + if (result.status !== 'rejected') continue + const error = result.reason + if (error instanceof VaultError && error.kind === 'NotFound') continue + throw error + } + } + + @StartActiveSpan() + async upsertProject(project: ProjectWithDetails): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + span?.setAttribute('vault.kv.name', project.slug) + const appPolicyName = generateAppAdminPolicyName(project) + const techPolicyName = generateTechReadOnlyPolicyName(project) + await this.upsertMount(project.slug) + await Promise.all([ + this.createAppAdminPolicy(appPolicyName, project.slug), + this.createTechReadOnlyPolicy(techPolicyName, project.slug), + this.ensureProjectGroup(project.slug, appPolicyName), + this.client.upsertAuthApproleRole(project.slug, generateApproleRoleBody([techPolicyName, appPolicyName])), + ]) + } + + @StartActiveSpan() + async deleteProject(projectSlug: string): Promise { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', projectSlug) + span?.setAttribute('vault.kv.name', projectSlug) + const appPolicyName = generateAppAdminPolicyName({ slug: projectSlug } as ProjectWithDetails) + const techPolicyName = generateTechReadOnlyPolicyName({ slug: projectSlug } as ProjectWithDetails) + + await this.deleteMount(projectSlug) + + const settled = await Promise.allSettled([ + this.client.deleteSysPoliciesAcl(appPolicyName), + this.client.deleteSysPoliciesAcl(techPolicyName), + this.client.deleteAuthApproleRole(projectSlug), + this.client.deleteIdentityGroupName(projectSlug), + ]) + for (const result of settled) { + if (result.status !== 'rejected') continue + const error = result.reason + if (error instanceof VaultError && error.kind === 'NotFound') continue + throw error + } + } + + @StartActiveSpan() + private async ensureProjectGroup(groupName: string, policyName: string): Promise { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'vault.group.name': groupName, + 'vault.policy.name': policyName, + }) + await this.client.upsertIdentityGroupName(groupName, { + name: groupName, + type: 'external', + policies: [policyName], + }) + + const groupResult = await this.client.getIdentityGroupName(groupName) + if (!groupResult.data?.id) { + throw new VaultError('InvalidResponse', `Vault group not found after upsert: ${groupName}`, { method: 'GET', path: `/v1/identity/group/name/${groupName}` }) + } + + const groupAliasName = `/${groupName}` + if (groupResult.data.alias?.name === groupAliasName) return + + const methods = await this.client.getSysAuth() + const oidc = methods['oidc/'] + if (!oidc?.accessor) { + throw new VaultError('InvalidResponse', 'Vault OIDC auth method not found (expected "oidc/")', { method: 'GET', path: '/v1/sys/auth' }) + } + try { + span?.setAttributes({ + 'vault.group.alias.name': groupAliasName, + 'vault.oidc.accessor': oidc.accessor, + }) + await this.client.createIdentityGroupAlias({ + name: groupAliasName, + mount_accessor: oidc.accessor, + canonical_id: groupResult.data.id, + }) + } catch (error) { + if (error instanceof VaultError && error.kind === 'HttpError' && error.status === 400) return + throw error + } + } + + async createAppAdminPolicy(name: string, projectSlug: string): Promise { + await this.client.upsertSysPoliciesAcl(name, { + policy: `path "${projectSlug}/*" { capabilities = ["create", "read", "update", "delete", "list"] }`, + }) + } + + async createTechReadOnlyPolicy(name: string, projectSlug: string): Promise { + await this.client.upsertSysPoliciesAcl(name, { + policy: `path "${this.config.vaultKvName}/data/${projectSlug}/REGISTRY/ro-robot" { capabilities = ["read"] }`, + }) + } + + async listProjectSecrets(projectSlug: string): Promise { + const projectPath = generateProjectPath(this.config.projectRootPath, projectSlug) + return this.listRecursive(this.config.vaultKvName, projectPath, '') + } + + @StartActiveSpan() + async deleteProjectSecrets(projectSlug: string): Promise { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'vault.kv.name': this.config.vaultKvName, + }) + const secrets = await this.listProjectSecrets(projectSlug) + span?.setAttribute('vault.secrets.count', secrets.length) + + const projectPath = generateProjectPath(this.config.projectRootPath, projectSlug) + await Promise.allSettled(secrets.map(async (relativePath) => { + const fullPath = `${projectPath}/${relativePath}` + try { + await this.client.delete(fullPath) + } catch (error) { + if (error instanceof VaultError && error.kind === 'NotFound') return + throw error + } + })) + } + + private async listRecursive( + kvName: string, + basePath: string, + relativePath: string, + ): Promise { + const combined = relativePath.length === 0 ? basePath : `${basePath}/${relativePath}` + const keys = await this.client.listKvMetadata(kvName, combined) + if (keys.length === 0) return [] + + const results: string[] = [] + for (const key of keys) { + if (key.endsWith('/')) { + const nestedRel = relativePath.length === 0 ? key.slice(0, -1) : `${relativePath}/${key.slice(0, -1)}` + const nested = await this.listRecursive(kvName, basePath, nestedRel) + results.push(...nested) + } else { + results.push(relativePath.length === 0 ? key : `${relativePath}/${key}`) + } + } + return results + } +} + +function generateTechReadOnlyPolicyName(project: ProjectWithDetails) { + return `tech--${project.slug}--ro` +} + +function generateAppAdminPolicyName(project: ProjectWithDetails) { + return `app--${project.slug}--admin` +} + +function generateZoneName(name: string) { + return `zone-${name}` +} + +function generateZoneTechReadOnlyPolicyName(zoneName: string) { + return `tech--${generateZoneName(zoneName)}--ro` +} + +function generateApproleRoleBody(policies: string[]) { + return { + secret_id_num_uses: '0', + secret_id_ttl: '0', + token_max_ttl: '0', + token_num_uses: '0', + token_ttl: '0', + token_type: 'batch', + token_policies: policies, + } +} diff --git a/apps/server-nestjs/src/modules/vault/vault.utils.ts b/apps/server-nestjs/src/modules/vault/vault.utils.ts new file mode 100644 index 000000000..0b3572c64 --- /dev/null +++ b/apps/server-nestjs/src/modules/vault/vault.utils.ts @@ -0,0 +1,17 @@ +export function generateProjectPath(projectRootPath: string | undefined, projectSlug: string) { + return projectRootPath + ? `${projectRootPath}/${projectSlug}` + : projectSlug +} + +export function generateGitlabMirrorCredPath(projectRootPath: string | undefined, projectSlug: string, repoName: string) { + return projectRootPath + ? `${generateProjectPath(projectRootPath, projectSlug)}/${repoName}-mirror` + : `${projectSlug}/${repoName}-mirror` +} + +export function generateTechReadOnlyCredPath(projectRootPath: string | undefined, projectSlug: string) { + return projectRootPath + ? `${generateProjectPath(projectRootPath, projectSlug)}/tech/GITLAB_MIRROR` + : `${projectSlug}/tech/GITLAB_MIRROR` +} diff --git a/apps/server-nestjs/src/prisma/schema/project.prisma b/apps/server-nestjs/src/prisma/schema/project.prisma index 833845eee..d45ccf451 100644 --- a/apps/server-nestjs/src/prisma/schema/project.prisma +++ b/apps/server-nestjs/src/prisma/schema/project.prisma @@ -5,6 +5,7 @@ model Environment { memory Float @db.Real cpu Float @db.Real gpu Float @db.Real + autosync Boolean @default(true) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt clusterId String @db.Uuid diff --git a/apps/server-nestjs/src/utils/iterable.ts b/apps/server-nestjs/src/utils/iterable.ts index 04b0a70cc..a46ef8063 100644 --- a/apps/server-nestjs/src/utils/iterable.ts +++ b/apps/server-nestjs/src/utils/iterable.ts @@ -17,3 +17,10 @@ export async function getAll( } return items } + +export async function find(generator: AsyncGenerator, predicate: (item: T) => boolean): Promise { + for await (const item of generator) { + if (predicate(item)) return item + } + return undefined +} diff --git a/apps/server-nestjs/test/argocd.e2e-spec.ts b/apps/server-nestjs/test/argocd.e2e-spec.ts new file mode 100644 index 000000000..e8484d290 --- /dev/null +++ b/apps/server-nestjs/test/argocd.e2e-spec.ts @@ -0,0 +1,308 @@ +import type { CommitAction, Gitlab } from '@gitbeaker/core' +import type { TestingModule } from '@nestjs/testing' +import { faker } from '@faker-js/faker' +import { Test } from '@nestjs/testing' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { parse } from 'yaml' +import { ConfigurationModule } from '../src/cpin-module/infrastructure/configuration/configuration.module' +import { ConfigurationService } from '../src/cpin-module/infrastructure/configuration/configuration.service' +import { PrismaService } from '../src/cpin-module/infrastructure/database/prisma.service' +import { InfrastructureModule } from '../src/cpin-module/infrastructure/infrastructure.module' +import { projectSelect } from '../src/modules/argocd/argocd-datastore.service' +import { ArgoCDModule } from '../src/modules/argocd/argocd.module' +import { ArgoCDService } from '../src/modules/argocd/argocd.service' +import { GITLAB_REST_CLIENT, GitlabClientService } from '../src/modules/gitlab/gitlab-client.service' +import { VaultClientService } from '../src/modules/vault/vault-client.service' + +const canRunArgoCDE2E + = Boolean(process.env.E2E) + && Boolean(process.env.GITLAB_URL) + && Boolean(process.env.GITLAB_TOKEN) + && Boolean(process.env.VAULT_URL) + && Boolean(process.env.VAULT_TOKEN) + && Boolean(process.env.PROJECTS_ROOT_DIR) + && Boolean(process.env.DB_URL) + +const describeWithArgoCD = describe.runIf(canRunArgoCDE2E) + +describeWithArgoCD('ArgoCDController (e2e)', {}, () => { + let moduleRef: TestingModule + let argocdController: ArgoCDService + let gitlab: GitlabClientService + let gitlabClient: Gitlab + let vault: VaultClientService + let prisma: PrismaService + let config: ConfigurationService + + let ownerId: string + let testProjectId: string + let testProjectSlug: string + + let zoneId: string + let zoneSlug: string + let kubeconfigId: string + let clusterId: string + let clusterLabel: string + let stageId: string + let envDevId: string + let envProdId: string + let envDevName: string + let envProdName: string + + let infraRepoId: number | undefined + let infraRepoPath: string + let vaultProjectValuesPath: string | undefined + + beforeAll(async () => { + moduleRef = await Test.createTestingModule({ + imports: [ArgoCDModule, ConfigurationModule, InfrastructureModule], + }).compile() + + await moduleRef.init() + + argocdController = moduleRef.get(ArgoCDService) + gitlab = moduleRef.get(GitlabClientService) + gitlabClient = moduleRef.get(GITLAB_REST_CLIENT) + vault = moduleRef.get(VaultClientService) + prisma = moduleRef.get(PrismaService) + config = moduleRef.get(ConfigurationService) + + ownerId = faker.string.uuid() + testProjectId = faker.string.uuid() + testProjectSlug = faker.helpers.slugify(`test-project-${faker.string.uuid()}`) + + zoneId = faker.string.uuid() + zoneSlug = faker.string.alphanumeric({ length: 10 }).toLowerCase() + kubeconfigId = faker.string.uuid() + clusterId = faker.string.uuid() + clusterLabel = faker.helpers.slugify(`cluster-${faker.string.uuid()}`.slice(0, 40)) + stageId = faker.string.uuid() + envDevId = faker.string.uuid() + envProdId = faker.string.uuid() + envDevName = 'dev' + envProdName = 'prod' + + await prisma.user.create({ + data: { + id: ownerId, + email: faker.internet.email().toLowerCase(), + firstName: 'Test', + lastName: 'Owner', + type: 'human', + }, + }) + + await prisma.zone.create({ + data: { + id: zoneId, + slug: zoneSlug, + label: `Zone ${zoneSlug}`, + argocdUrl: 'https://example.com', + }, + }) + + await prisma.kubeconfig.create({ + data: { + id: kubeconfigId, + user: { token: faker.string.alphanumeric({ length: 16 }) }, + cluster: { server: 'https://example.com' }, + }, + }) + + await prisma.cluster.create({ + data: { + id: clusterId, + label: clusterLabel, + secretName: faker.string.uuid(), + kubeConfigId: kubeconfigId, + infos: null, + memory: 100, + cpu: 100, + gpu: 0, + zoneId, + }, + }) + + await prisma.stage.create({ + data: { + id: stageId, + name: faker.helpers.slugify(`stage-${faker.string.uuid()}`), + }, + }) + + await prisma.project.create({ + data: { + id: testProjectId, + slug: testProjectSlug, + name: testProjectSlug, + ownerId, + description: 'E2E Test Project', + hprodCpu: 0, + hprodGpu: 0, + hprodMemory: 0, + prodCpu: 0, + prodGpu: 0, + prodMemory: 0, + clusters: { + connect: { id: clusterId }, + }, + repositories: { + create: { + internalRepoName: zoneSlug, + isInfra: true, + deployRevision: 'HEAD', + deployPath: '.', + helmValuesFiles: '', + }, + }, + environments: { + create: [ + { + id: envDevId, + name: envDevName, + clusterId, + stageId, + cpu: 1, + gpu: 0, + memory: 1, + autosync: true, + }, + { + id: envProdId, + name: envProdName, + clusterId, + stageId, + cpu: 1, + gpu: 0, + memory: 1, + autosync: true, + }, + ], + }, + }, + }) + + infraRepoPath = `${config.projectRootPath}/infra/${zoneSlug}` + try { + const existing = await gitlabClient.Projects.show(infraRepoPath) + if (existing.empty_repo || existing.default_branch !== 'main') { + await gitlabClient.Projects.remove(existing.id).catch(() => {}) + throw new Error('Recreate infra repo') + } + infraRepoId = existing.id + } catch (error: any) { + const description = error?.cause?.description ?? '' + if ( + !(typeof description === 'string' && description.includes('404')) + && !(error instanceof Error && error.message === 'Recreate infra repo') + ) { + throw error + } + + const infraGroup = await gitlab.getOrCreateProjectSubGroup('infra') + const created = await gitlabClient.Projects.create({ + name: zoneSlug, + path: zoneSlug, + namespaceId: infraGroup.id, + initializeWithReadme: true, + defaultBranch: 'main', + } as any) + infraRepoId = created.id + } + + vaultProjectValuesPath = `${config.projectRootPath}/${testProjectId}` + await vault.write({ e2e: true }, vaultProjectValuesPath) + }) + + afterAll(async () => { + if (vaultProjectValuesPath) { + await vault.delete(vaultProjectValuesPath).catch(() => {}) + } + + if (infraRepoId) { + await gitlabClient.Projects.remove(infraRepoId).catch(() => {}) + } + + if (prisma) { + await prisma.project.deleteMany({ where: { id: testProjectId } }).catch(() => {}) + await prisma.stage.deleteMany({ where: { id: stageId } }).catch(() => {}) + await prisma.cluster.deleteMany({ where: { id: clusterId } }).catch(() => {}) + await prisma.kubeconfig.deleteMany({ where: { id: kubeconfigId } }).catch(() => {}) + await prisma.zone.deleteMany({ where: { id: zoneId } }).catch(() => {}) + await prisma.user.deleteMany({ where: { id: ownerId } }).catch(() => {}) + } + + await moduleRef.close() + + vi.restoreAllMocks() + vi.unstubAllEnvs() + }) + + it('should commit environment values and cleanup stale values in the zone infra repo', async () => { + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + const infraProject = await gitlab.getOrCreateInfraGroupRepo(zoneSlug) + infraRepoId = infraProject.id + + const staleFilePath = `${project.name}/${clusterLabel}/stale/values.yaml` + if (!infraRepoId) throw new Error('Missing infra repo id') + const staleAction = await gitlab.generateCreateOrUpdateAction(infraProject, 'main', staleFilePath, 'stale: true\n') + await gitlab.maybeCreateCommit(infraProject, 'ci: :robot_face: Seed stale values', staleAction ? [staleAction] : []) + + await argocdController.handleUpsert(project) + + const expectedFilePath = `${project.name}/${clusterLabel}/${envDevName}/values.yaml` + const file = await gitlabClient.RepositoryFiles.show(infraRepoId, expectedFilePath, 'main') + const raw = Buffer.from(file.content, 'base64').toString('utf8') + const values = parse(raw) as any + + expect(values?.common?.['dso/project.slug']).toBe(testProjectSlug) + expect(values?.common?.['dso/environment']).toBe(envDevName) + expect(values?.environment?.valueFilePath).toBe(expectedFilePath) + expect(values?.application?.destination?.name).toBe(clusterLabel) + expect(values?.application?.autosync).toBe(true) + + const shouldBeDeleted = await gitlab.getFile(infraProject, staleFilePath, 'main') + expect(shouldBeDeleted).toBeUndefined() + }, 144000) + + it('should update existing values and delete values of a removed environment', async () => { + const before = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + const infraProject = await gitlab.getOrCreateInfraGroupRepo(zoneSlug) + infraRepoId = infraProject.id + + const devFilePath = `${before.name}/${clusterLabel}/${envDevName}/values.yaml` + const prodFilePath = `${before.name}/${clusterLabel}/${envProdName}/values.yaml` + + const seededActions = (await Promise.all([ + gitlab.generateCreateOrUpdateAction(infraProject, 'main', devFilePath, 'old: true\n'), + gitlab.generateCreateOrUpdateAction(infraProject, 'main', prodFilePath, 'old: true\n'), + ])).filter((action): action is NonNullable => action !== null) + await gitlab.maybeCreateCommit(infraProject, 'ci: :robot_face: Seed existing values', seededActions as CommitAction[]) + + await prisma.environment.deleteMany({ where: { id: envProdId } }) + + const after = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + await argocdController.handleUpsert(after) + + const updatedDev = await gitlabClient.RepositoryFiles.show(infraRepoId, devFilePath, 'main') + const devRaw = Buffer.from(updatedDev.content, 'base64').toString('utf8') + const devValues = parse(devRaw) as any + expect(devValues?.common?.['dso/project.slug']).toBe(testProjectSlug) + expect(devValues?.common?.['dso/environment']).toBe(envDevName) + + const prodFile = await gitlab.getFile(infraProject, prodFilePath, 'main') + expect(prodFile).toBeUndefined() + }, 72000) +}) diff --git a/apps/server-nestjs/test/gitlab.e2e-spec.ts b/apps/server-nestjs/test/gitlab.e2e-spec.ts new file mode 100644 index 000000000..6436dccb4 --- /dev/null +++ b/apps/server-nestjs/test/gitlab.e2e-spec.ts @@ -0,0 +1,223 @@ +import type { ExpandedUserSchema, Gitlab } from '@gitbeaker/core' +import type { TestingModule } from '@nestjs/testing' +import { faker } from '@faker-js/faker' +import { Test } from '@nestjs/testing' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import z from 'zod' +import { ConfigurationModule } from '../src/cpin-module/infrastructure/configuration/configuration.module' +import { ConfigurationService } from '../src/cpin-module/infrastructure/configuration/configuration.service' +import { PrismaService } from '../src/cpin-module/infrastructure/database/prisma.service' +import { InfrastructureModule } from '../src/cpin-module/infrastructure/infrastructure.module' +import { GITLAB_REST_CLIENT, GitlabClientService } from '../src/modules/gitlab/gitlab-client.service' +import { projectSelect } from '../src/modules/gitlab/gitlab-datastore.service' +import { GitlabModule } from '../src/modules/gitlab/gitlab.module' +import { GitlabService } from '../src/modules/gitlab/gitlab.service' +import { VaultClientService } from '../src/modules/vault/vault-client.service' + +const canRunGitlabE2E + = Boolean(process.env.E2E) + && Boolean(process.env.GITLAB_URL) + && Boolean(process.env.GITLAB_TOKEN) + && Boolean(process.env.VAULT_URL) + && Boolean(process.env.VAULT_TOKEN) + && Boolean(process.env.PROJECTS_ROOT_DIR) + && Boolean(process.env.DB_URL) + +const describeWithGitLab = describe.runIf(canRunGitlabE2E) + +describeWithGitLab('GitlabController (e2e)', {}, () => { + let moduleRef: TestingModule + let gitlabController: GitlabService + let gitlabService: GitlabClientService + let gitlabClient: Gitlab + let vaultService: VaultClientService + let prisma: PrismaService + let config: ConfigurationService + + let testProjectId: string + let testProjectSlug: string + let ownerId: string + let ownerUser: ExpandedUserSchema + + beforeAll(async () => { + moduleRef = await Test.createTestingModule({ + imports: [GitlabModule, ConfigurationModule, InfrastructureModule], + }).compile() + + await moduleRef.init() + + gitlabController = moduleRef.get(GitlabService) + gitlabService = moduleRef.get(GitlabClientService) + gitlabClient = moduleRef.get(GITLAB_REST_CLIENT) + vaultService = moduleRef.get(VaultClientService) + prisma = moduleRef.get(PrismaService) + config = moduleRef.get(ConfigurationService) + + ownerId = faker.string.uuid() + testProjectId = faker.string.uuid() + testProjectSlug = faker.helpers.slugify(`test-project-${faker.string.uuid()}`) + + const ownerEmail = `test-owner-${ownerId}@example.com` + + // Create owner in GitLab + ownerUser = await gitlabClient.Users.create({ + name: 'Test Owner', + password: faker.internet.password({ length: 24 }), + username: `test-owner-${ownerId}`, + email: ownerEmail, + skipConfirmation: true, + }) + + // Create owner in DB + await prisma.user.create({ + data: { + id: ownerId, + email: ownerUser.email.toLowerCase(), + firstName: 'Test', + lastName: 'Owner', + type: 'human', + }, + }) + }) + + afterAll(async () => { + // Clean GitLab group + if (testProjectSlug && config.projectRootPath) { + const fullPath = `${config.projectRootPath}/${testProjectSlug}` + const group = await gitlabService.getGroupByPath(fullPath) + if (group) { + await gitlabService.deleteGroup(group).catch(() => {}) + } + } + + // Clean Vault + if (testProjectSlug && config.projectRootPath) { + const vaultPath = `${config.projectRootPath}/${testProjectSlug}` + await vaultService.delete(`${vaultPath}/tech/GITLAB_MIRROR`).catch(() => {}) + await vaultService.delete(`${vaultPath}/app-mirror`).catch(() => {}) + } + + // Clean DB + if (prisma) { + await prisma.projectMembers.deleteMany({ where: { projectId: testProjectId } }).catch(() => {}) + await prisma.project.deleteMany({ where: { id: testProjectId } }).catch(() => {}) + await prisma.user.deleteMany({ where: { id: ownerId } }).catch(() => {}) + } + + await moduleRef.close() + + vi.restoreAllMocks() + vi.unstubAllEnvs() + }) + + it('should reconcile and create project group in GitLab and Vault secrets', async () => { + // Create Project in DB + await prisma.project.create({ + data: { + id: testProjectId, + slug: testProjectSlug, + name: testProjectSlug, + ownerId, + description: 'E2E Test Project', + hprodCpu: 0, + hprodGpu: 0, + hprodMemory: 0, + prodCpu: 0, + prodGpu: 0, + prodMemory: 0, + }, + }) + + await prisma.repository.create({ + data: { + projectId: testProjectId, + internalRepoName: 'app', + externalRepoUrl: 'https://example.com/example.git', + isPrivate: false, + }, + }) + + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + // Act + await gitlabController.handleUpsert(project) + + // Assert + const groupPath = `${config.projectRootPath}/${testProjectSlug}` + const group = z.object({ + id: z.number(), + name: z.string(), + full_path: z.string(), + web_url: z.string(), + }).parse(await gitlabService.getGroupByPath(groupPath)) + expect(group.full_path).toBe(groupPath) + + // Check membership + const members = await gitlabService.getGroupMembers(group) + const isMember = members.some(m => m.id === ownerUser.id) + expect(isMember).toBe(true) + + const repoVaultPath = `${config.projectRootPath}/${testProjectSlug}/app-mirror` + const repoSecret = await vaultService.read(repoVaultPath) + expect(repoSecret?.data?.GIT_OUTPUT_USER).toBeTruthy() + expect(repoSecret?.data?.GIT_OUTPUT_PASSWORD).toBeTruthy() + }, 72000) + + it('should add member to GitLab group when added in DB', async () => { + // Create user in GitLab + const newUserId = faker.string.uuid() + const newUser = await gitlabClient.Users.create({ + email: faker.internet.email().toLowerCase(), + username: faker.internet.username(), + name: `${faker.person.firstName()} ${faker.person.lastName()}`, + password: faker.internet.password({ length: 24 }), + skipConfirmation: true, + }) + + // Create user in DB + await prisma.user.create({ + data: { + id: newUserId, + email: newUser.email, + firstName: 'Test', + lastName: 'User', + type: 'human', + }, + }) + + // Add member to project in DB + await prisma.projectMembers.create({ + data: { + projectId: testProjectId, + userId: newUserId, + roleIds: [], // No roles for now + }, + }) + + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + // Act + await gitlabController.handleUpsert(project) + + // Assert + const groupPath = `${config.projectRootPath}/${testProjectSlug}` + const group = z.object({ + id: z.number(), + name: z.string(), + web_url: z.string(), + }).parse(await gitlabService.getGroupByPath(groupPath)) + + const members = await gitlabService.getGroupMembers(group) + const isNewMemberPresent = members.some(m => m.id === newUser.id) + expect(isNewMemberPresent).toBe(true) + + await prisma.projectMembers.deleteMany({ where: { userId: newUserId } }).catch(() => {}) + await prisma.user.delete({ where: { id: newUserId } }).catch(() => {}) + }, 72000) +}) diff --git a/apps/server-nestjs/test/vault.e2e-spec.ts b/apps/server-nestjs/test/vault.e2e-spec.ts new file mode 100644 index 000000000..0fa5941c3 --- /dev/null +++ b/apps/server-nestjs/test/vault.e2e-spec.ts @@ -0,0 +1,102 @@ +import type { TestingModule } from '@nestjs/testing' +import { faker } from '@faker-js/faker' +import { Test } from '@nestjs/testing' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { ConfigurationModule } from '../src/cpin-module/infrastructure/configuration/configuration.module' +import { PrismaService } from '../src/cpin-module/infrastructure/database/prisma.service' +import { InfrastructureModule } from '../src/cpin-module/infrastructure/infrastructure.module' +import { VaultClientService } from '../src/modules/vault/vault-client.service' +import { projectSelect } from '../src/modules/vault/vault-datastore.service' +import { VaultModule } from '../src/modules/vault/vault.module' +import { VaultService } from '../src/modules/vault/vault.service' + +const canRunVaultE2E + = Boolean(process.env.E2E) + && Boolean(process.env.VAULT_URL) + && Boolean(process.env.VAULT_TOKEN) + && Boolean(process.env.DB_URL) + +const describeWithVault = describe.runIf(canRunVaultE2E) + +describeWithVault('VaultController (e2e)', () => { + let moduleRef: TestingModule + let vaultController: VaultService + let vaultClient: VaultClientService + let prisma: PrismaService + + let ownerId: string + let testProjectId: string + let testProjectSlug: string + + beforeAll(async () => { + moduleRef = await Test.createTestingModule({ + imports: [VaultModule, ConfigurationModule, InfrastructureModule], + }).compile() + + await moduleRef.init() + + vaultController = moduleRef.get(VaultService) + vaultClient = moduleRef.get(VaultClientService) + prisma = moduleRef.get(PrismaService) + + ownerId = faker.string.uuid() + testProjectId = faker.string.uuid() + testProjectSlug = faker.helpers.slugify(`test-project-${faker.string.uuid()}`) + + await prisma.user.create({ + data: { + id: ownerId, + email: faker.internet.email().toLowerCase(), + firstName: 'Test', + lastName: 'Owner', + type: 'human', + }, + }) + }) + + afterAll(async () => { + if (testProjectSlug) { + await vaultController.handleDelete({ slug: testProjectSlug } as any).catch(() => {}) + } + + if (prisma) { + await prisma.project.deleteMany({ where: { id: testProjectId } }).catch(() => {}) + await prisma.user.deleteMany({ where: { id: ownerId } }).catch(() => {}) + } + + await moduleRef.close() + + vi.restoreAllMocks() + vi.unstubAllEnvs() + }) + + it('should reconcile project in Vault (mount, group, role)', async () => { + await prisma.project.create({ + data: { + id: testProjectId, + slug: testProjectSlug, + name: testProjectSlug, + ownerId, + description: 'E2E Test Project', + hprodCpu: 0, + hprodGpu: 0, + hprodMemory: 0, + prodCpu: 0, + prodGpu: 0, + prodMemory: 0, + }, + }) + + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + await vaultController.handleUpsert(project) + + const group = await vaultClient.getIdentityGroupName(testProjectSlug) + expect(group.data?.id).toBeTruthy() + expect(group.data?.name).toBe(testProjectSlug) + expect(group.data?.alias?.name).toBe(`/${testProjectSlug}`) + }, 180000) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6b3f3bc50..fe39c8ca7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -385,6 +385,9 @@ importers: '@gitbeaker/core': specifier: ^40.6.0 version: 40.6.0 + '@gitbeaker/requester-utils': + specifier: ^40.6.0 + version: 40.6.0 '@gitbeaker/rest': specifier: ^40.6.0 version: 40.6.0 From a565272d057e484d5881bd4a1cc30f454ce704fd Mon Sep 17 00:00:00 2001 From: William Phetsinorath Date: Wed, 25 Mar 2026 14:16:19 +0100 Subject: [PATCH 3/4] refactor(server-nestjs): migrate package managers to NestJS Signed-off-by: William Phetsinorath --- .../src/modules/healthz/healthz.controller.ts | 8 +- .../src/modules/healthz/healthz.module.ts | 6 +- .../nexus/nexus-client.service.spec.ts | 33 ++ .../src/modules/nexus/nexus-client.service.ts | 404 ++++++++++++++ .../modules/nexus/nexus-datastore.service.ts | 48 ++ .../src/modules/nexus/nexus-health.service.ts | 30 ++ .../src/modules/nexus/nexus-testing.utils.ts | 13 + .../src/modules/nexus/nexus.constants.ts | 13 + .../src/modules/nexus/nexus.module.ts | 16 + .../src/modules/nexus/nexus.service.spec.ts | 138 +++++ .../src/modules/nexus/nexus.service.ts | 421 +++++++++++++++ .../src/modules/nexus/nexus.utils.ts | 68 +++ .../registry/registry-client.service.spec.ts | 48 ++ .../registry/registry-client.service.ts | 491 ++++++++++++++++++ .../registry/registry-health.service.ts | 30 ++ .../src/modules/registry/registry.module.ts | 14 + .../src/modules/registry/registry.utils.ts | 46 ++ apps/server-nestjs/test/nexus.e2e-spec.ts | 148 ++++++ 18 files changed, 1973 insertions(+), 2 deletions(-) create mode 100644 apps/server-nestjs/src/modules/nexus/nexus-client.service.spec.ts create mode 100644 apps/server-nestjs/src/modules/nexus/nexus-client.service.ts create mode 100644 apps/server-nestjs/src/modules/nexus/nexus-datastore.service.ts create mode 100644 apps/server-nestjs/src/modules/nexus/nexus-health.service.ts create mode 100644 apps/server-nestjs/src/modules/nexus/nexus-testing.utils.ts create mode 100644 apps/server-nestjs/src/modules/nexus/nexus.constants.ts create mode 100644 apps/server-nestjs/src/modules/nexus/nexus.module.ts create mode 100644 apps/server-nestjs/src/modules/nexus/nexus.service.spec.ts create mode 100644 apps/server-nestjs/src/modules/nexus/nexus.service.ts create mode 100644 apps/server-nestjs/src/modules/nexus/nexus.utils.ts create mode 100644 apps/server-nestjs/src/modules/registry/registry-client.service.spec.ts create mode 100644 apps/server-nestjs/src/modules/registry/registry-client.service.ts create mode 100644 apps/server-nestjs/src/modules/registry/registry-health.service.ts create mode 100644 apps/server-nestjs/src/modules/registry/registry.module.ts create mode 100644 apps/server-nestjs/src/modules/registry/registry.utils.ts create mode 100644 apps/server-nestjs/test/nexus.e2e-spec.ts diff --git a/apps/server-nestjs/src/modules/healthz/healthz.controller.ts b/apps/server-nestjs/src/modules/healthz/healthz.controller.ts index a81ce3caf..c38944630 100644 --- a/apps/server-nestjs/src/modules/healthz/healthz.controller.ts +++ b/apps/server-nestjs/src/modules/healthz/healthz.controller.ts @@ -4,7 +4,9 @@ import { DatabaseHealthService } from '../../cpin-module/infrastructure/database import { ArgoCDHealthService } from '../argocd/argocd-health.service' import { GitlabHealthService } from '../gitlab/gitlab-health.service' import { KeycloakHealthService } from '../keycloak/keycloak-health.service' -import { KeycloakHealthService } from '../keycloak/keycloak-health.service' +import { NexusHealthService } from '../nexus/nexus-health.service' +import { RegistryHealthService } from '../registry/registry-health.service' +import { VaultHealthService } from '../vault/vault-health.service' @Controller('api/v1/healthz') export class HealthzController { @@ -14,6 +16,8 @@ export class HealthzController { @Inject(KeycloakHealthService) private readonly keycloak: KeycloakHealthService, @Inject(GitlabHealthService) private readonly gitlab: GitlabHealthService, @Inject(VaultHealthService) private readonly vault: VaultHealthService, + @Inject(NexusHealthService) private readonly nexus: NexusHealthService, + @Inject(RegistryHealthService) private readonly registry: RegistryHealthService, @Inject(ArgoCDHealthService) private readonly argocd: ArgoCDHealthService, ) {} @@ -25,6 +29,8 @@ export class HealthzController { () => this.keycloak.check('keycloak'), () => this.gitlab.check('gitlab'), () => this.vault.check('vault'), + () => this.nexus.check('nexus'), + () => this.registry.check('registry'), () => this.argocd.check('argocd'), ]) } diff --git a/apps/server-nestjs/src/modules/healthz/healthz.module.ts b/apps/server-nestjs/src/modules/healthz/healthz.module.ts index b060843d9..5c466de62 100644 --- a/apps/server-nestjs/src/modules/healthz/healthz.module.ts +++ b/apps/server-nestjs/src/modules/healthz/healthz.module.ts @@ -4,7 +4,9 @@ import { DatabaseModule } from '../../cpin-module/infrastructure/database/databa import { ArgoCDModule } from '../argocd/argocd.module' import { GitlabModule } from '../gitlab/gitlab.module' import { KeycloakModule } from '../keycloak/keycloak.module' -import { DatabaseModule } from '../../cpin-module/infrastructure/database/database.module' +import { NexusModule } from '../nexus/nexus.module' +import { RegistryModule } from '../registry/registry.module' +import { VaultModule } from '../vault/vault.module' import { HealthzController } from './healthz.controller' @Module({ @@ -14,6 +16,8 @@ import { HealthzController } from './healthz.controller' KeycloakModule, GitlabModule, VaultModule, + NexusModule, + RegistryModule, ArgoCDModule, ], controllers: [HealthzController], diff --git a/apps/server-nestjs/src/modules/nexus/nexus-client.service.spec.ts b/apps/server-nestjs/src/modules/nexus/nexus-client.service.spec.ts new file mode 100644 index 000000000..e9d13fcdf --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus-client.service.spec.ts @@ -0,0 +1,33 @@ +import type { TestingModule } from '@nestjs/testing' +import { Test } from '@nestjs/testing' +import { beforeEach, describe, expect, it } from 'vitest' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { NexusClientService } from './nexus-client.service' + +function createNexusServiceTestingModule() { + return Test.createTestingModule({ + providers: [ + NexusClientService, + { + provide: ConfigurationService, + useValue: { + nexusSecretExposedUrl: 'https://nexus.example', + projectRootPath: 'forge', + } satisfies Partial, + }, + ], + }) +} + +describe('nexusClientService', () => { + let service: NexusClientService + + beforeEach(async () => { + const module: TestingModule = await createNexusServiceTestingModule().compile() + service = module.get(NexusClientService) + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) +}) diff --git a/apps/server-nestjs/src/modules/nexus/nexus-client.service.ts b/apps/server-nestjs/src/modules/nexus/nexus-client.service.ts new file mode 100644 index 000000000..bf3d4fa83 --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus-client.service.ts @@ -0,0 +1,404 @@ +import { Inject, Injectable } from '@nestjs/common' +import { trace } from '@opentelemetry/api' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' +import { generateMavenHostedRepoName, generateNpmHostedRepoName } from './nexus.utils' + +interface NexusRepositoryStorage { + blobStoreName: string + strictContentTypeValidation: boolean + writePolicy?: string +} + +interface NexusRepositoryCleanup { + policyNames: string[] +} + +interface NexusRepositoryComponent { + proprietaryComponents: boolean +} + +interface NexusRepositoryGroup { + memberNames: string[] +} + +interface NexusMavenHostedRepositoryUpsertRequest { + name: string + online: boolean + storage: NexusRepositoryStorage & { writePolicy: string } + cleanup: NexusRepositoryCleanup + component: NexusRepositoryComponent + maven: { + versionPolicy: string + layoutPolicy: string + contentDisposition: string + } +} + +interface NexusMavenGroupRepositoryUpsertRequest { + name: string + online: boolean + storage: Omit + group: NexusRepositoryGroup +} + +interface NexusNpmHostedRepositoryUpsertRequest { + name: string + online: boolean + storage: NexusRepositoryStorage & { writePolicy: string } + cleanup: NexusRepositoryCleanup + component: NexusRepositoryComponent +} + +interface NexusNpmGroupRepositoryUpsertRequest { + name: string + online: boolean + storage: Omit + group: NexusRepositoryGroup +} + +interface NexusRepositoryViewPrivilegeUpsertRequest { + name: string + description: string + actions: string[] + format: string + repository: string +} + +interface NexusRoleCreateRequest { + id: string + name: string + description: string + privileges: string[] +} + +interface NexusRoleUpdateRequest { + id: string + name: string + privileges: string[] +} + +interface NexusUserCreateRequest { + userId: string + firstName: string + lastName: string + emailAddress: string + password: string + status: string + roles: string[] +} + +interface NexusFetchOptions { + method?: string + body?: unknown + headers?: Record +} + +interface NexusResponse { + status: number + data: T | null +} + +export type NexusErrorKind + = | 'NotConfigured' + | 'HttpError' + | 'Unexpected' + +export class NexusError extends Error { + readonly kind: NexusErrorKind + readonly status?: number + readonly method?: string + readonly path?: string + readonly statusText?: string + + constructor( + kind: NexusErrorKind, + message: string, + details: { status?: number, method?: string, path?: string, statusText?: string } = {}, + ) { + super(message) + this.name = 'NexusError' + this.kind = kind + this.status = details.status + this.method = details.method + this.path = details.path + this.statusText = details.statusText + } +} + +@Injectable() +export class NexusClientService { + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + ) {} + + getProjectSecrets(args: { projectSlug: string, enableMaven: boolean, enableNpm: boolean }) { + const projectSlug = args.projectSlug + const nexusUrl = this.config.nexusSecretExposedUrl! + const secrets: Record = {} + if (args.enableMaven) { + secrets.MAVEN_REPO_RELEASE = `${nexusUrl}/${generateMavenHostedRepoName(projectSlug, 'release')}` + secrets.MAVEN_REPO_SNAPSHOT = `${nexusUrl}/${generateMavenHostedRepoName(projectSlug, 'snapshot')}` + } + if (args.enableNpm) { + secrets.NPM_REPO = `${nexusUrl}/${generateNpmHostedRepoName(projectSlug)}` + } + return secrets + } + + @StartActiveSpan() + private async fetch(path: string, options: NexusFetchOptions = {}): Promise> { + const span = trace.getActiveSpan() + const method = options.method ?? 'GET' + span?.setAttribute('nexus.method', method) + span?.setAttribute('nexus.path', path) + + const request = this.createRequest(path, method, options.body, options.headers) + const response = await fetch(request).catch((error) => { + throw new NexusError( + 'Unexpected', + error instanceof Error ? error.message : String(error), + { method, path }, + ) + }) + span?.setAttribute('nexus.http.status', response.status) + const result = await handleNexusResponse(response) + if (!response.ok) { + throw new NexusError('HttpError', 'Request failed', { + status: result.status, + method, + path, + statusText: response.statusText, + }) + } + return result + } + + private get baseUrl() { + if (!this.config.nexusInternalUrl) { + throw new NexusError('NotConfigured', 'NEXUS_INTERNAL_URL is required') + } + return new URL('service/rest/v1/', this.config.nexusInternalUrl).toString() + } + + private get basicAuth() { + if (!this.config.nexusAdmin) { + throw new NexusError('NotConfigured', 'NEXUS_ADMIN is required') + } + if (!this.config.nexusAdminPassword) { + throw new NexusError('NotConfigured', 'NEXUS_ADMIN_PASSWORD is required') + } + const raw = `${this.config.nexusAdmin}:${this.config.nexusAdminPassword}` + return Buffer.from(raw, 'utf8').toString('base64') + } + + private createRequest(path: string, method: string, body?: unknown, extraHeaders?: Record): Request { + const url = new URL(path, this.baseUrl).toString() + const headers: Record = { + Authorization: `Basic ${this.basicAuth}`, + ...extraHeaders, + } + let requestBody: string | undefined + if (body !== undefined) { + if (typeof body === 'string') { + requestBody = body + headers['Content-Type'] = 'text/plain' + } else { + requestBody = JSON.stringify(body) + headers['Content-Type'] = 'application/json' + } + } + return new Request(url, { method, headers, body: requestBody }) + } + + @StartActiveSpan() + async getRepositoriesMavenHosted(name: string): Promise { + try { + const res = await this.fetch(`/repositories/maven/hosted/${name}`, { method: 'GET' }) + return res.data + } catch (error) { + if (error instanceof NexusError && error.status === 404) return null + throw error + } + } + + @StartActiveSpan() + async createRepositoriesMavenHosted(body: NexusMavenHostedRepositoryUpsertRequest): Promise { + await this.fetch('/repositories/maven/hosted', { method: 'POST', body }) + } + + @StartActiveSpan() + async updateRepositoriesMavenHosted(name: string, body: NexusMavenHostedRepositoryUpsertRequest): Promise { + await this.fetch(`/repositories/maven/hosted/${name}`, { method: 'PUT', body }) + } + + @StartActiveSpan() + async createRepositoriesMavenGroup(body: NexusMavenGroupRepositoryUpsertRequest): Promise { + await this.fetch('/repositories/maven/group', { method: 'POST', body }) + } + + @StartActiveSpan() + async getRepositoriesMavenGroup(name: string): Promise { + try { + const res = await this.fetch(`/repositories/maven/group/${name}`, { method: 'GET' }) + return res.data + } catch (error) { + if (error instanceof NexusError && error.status === 404) return null + throw error + } + } + + @StartActiveSpan() + async getRepositoriesNpmHosted(name: string): Promise { + try { + const res = await this.fetch(`/repositories/npm/hosted/${name}`, { method: 'GET' }) + return res.data + } catch (error) { + if (error instanceof NexusError && error.status === 404) return null + throw error + } + } + + @StartActiveSpan() + async createRepositoriesNpmHosted(body: NexusNpmHostedRepositoryUpsertRequest): Promise { + await this.fetch('/repositories/npm/hosted', { method: 'POST', body }) + } + + @StartActiveSpan() + async updateRepositoriesNpmHosted(name: string, body: NexusNpmHostedRepositoryUpsertRequest): Promise { + await this.fetch(`/repositories/npm/hosted/${name}`, { method: 'PUT', body }) + } + + @StartActiveSpan() + async getRepositoriesNpmGroup(name: string): Promise { + try { + const res = await this.fetch(`/repositories/npm/group/${name}`, { method: 'GET' }) + return res.data + } catch (error) { + if (error instanceof NexusError && error.status === 404) return null + throw error + } + } + + @StartActiveSpan() + async postRepositoriesNpmGroup(body: NexusNpmGroupRepositoryUpsertRequest): Promise { + await this.fetch('/repositories/npm/group', { method: 'POST', body }) + } + + @StartActiveSpan() + async putRepositoriesNpmGroup(name: string, body: NexusNpmGroupRepositoryUpsertRequest): Promise { + await this.fetch(`/repositories/npm/group/${name}`, { method: 'PUT', body }) + } + + @StartActiveSpan() + async getSecurityPrivileges(name: string): Promise { + try { + const res = await this.fetch(`/security/privileges/${name}`, { method: 'GET' }) + return res.data + } catch (error) { + if (error instanceof NexusError && error.status === 404) return null + throw error + } + } + + @StartActiveSpan() + async createSecurityPrivilegesRepositoryView(body: NexusRepositoryViewPrivilegeUpsertRequest): Promise { + await this.fetch('/security/privileges/repository-view', { method: 'POST', body }) + } + + @StartActiveSpan() + async updateSecurityPrivilegesRepositoryView(name: string, body: NexusRepositoryViewPrivilegeUpsertRequest): Promise { + await this.fetch(`/security/privileges/repository-view/${name}`, { method: 'PUT', body }) + } + + @StartActiveSpan() + async deleteSecurityPrivileges(name: string): Promise { + try { + await this.fetch(`/security/privileges/${name}`, { method: 'DELETE' }) + } catch (error) { + if (error instanceof NexusError && error.status === 404) return + throw error + } + } + + @StartActiveSpan() + async getSecurityRoles(id: string): Promise { + try { + const res = await this.fetch(`/security/roles/${id}`, { method: 'GET' }) + return res.data + } catch (error) { + if (error instanceof NexusError && error.status === 404) return null + throw error + } + } + + @StartActiveSpan() + async createSecurityRoles(body: NexusRoleCreateRequest): Promise { + await this.fetch('/security/roles', { method: 'POST', body }) + } + + @StartActiveSpan() + async updateSecurityRoles(id: string, body: NexusRoleUpdateRequest): Promise { + await this.fetch(`/security/roles/${id}`, { method: 'PUT', body }) + } + + @StartActiveSpan() + async deleteSecurityRoles(id: string): Promise { + try { + await this.fetch(`/security/roles/${id}`, { method: 'DELETE' }) + } catch (error) { + if (error instanceof NexusError && error.status === 404) return + throw error + } + } + + @StartActiveSpan() + async getSecurityUsers(userId: string): Promise<{ userId: string }[]> { + const query = new URLSearchParams({ userId }).toString() + const res = await this.fetch<{ userId: string }[]>(`/security/users?${query}`, { method: 'GET' }) + return (res.data as any) ?? [] + } + + @StartActiveSpan() + async updateSecurityUsersChangePassword(userId: string, password: string): Promise { + await this.fetch(`/security/users/${userId}/change-password`, { + method: 'PUT', + body: password, + headers: { 'Content-Type': 'text/plain' }, + }) + } + + @StartActiveSpan() + async createSecurityUsers(body: NexusUserCreateRequest): Promise { + await this.fetch('/security/users', { method: 'POST', body }) + } + + @StartActiveSpan() + async deleteSecurityUsers(userId: string): Promise { + try { + await this.fetch(`/security/users/${userId}`, { method: 'DELETE' }) + } catch (error) { + if (error instanceof NexusError && error.status === 404) return + throw error + } + } + + @StartActiveSpan() + async deleteRepositoriesByName(name: string): Promise { + try { + await this.fetch(`/repositories/${name}`, { method: 'DELETE' }) + } catch (error) { + if (error instanceof NexusError && error.status === 404) return + throw error + } + } +} + +async function handleNexusResponse(response: Response): Promise> { + if (response.status === 204) return { status: response.status, data: null } + const contentType = response.headers.get('content-type') ?? '' + const parsed = contentType.includes('application/json') + ? await response.json() + : await response.text() + return { status: response.status, data: parsed as T } +} diff --git a/apps/server-nestjs/src/modules/nexus/nexus-datastore.service.ts b/apps/server-nestjs/src/modules/nexus/nexus-datastore.service.ts new file mode 100644 index 000000000..3d276d4bc --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus-datastore.service.ts @@ -0,0 +1,48 @@ +import type { Prisma } from '@prisma/client' +import { Inject, Injectable } from '@nestjs/common' +import { PrismaService } from '../../cpin-module/infrastructure/database/prisma.service' +import { NEXUS_PLUGIN_NAME } from './nexus.constants' + +export const projectSelect = { + slug: true, + owner: { + select: { + email: true, + }, + }, + plugins: { + select: { + key: true, + value: true, + }, + }, +} satisfies Prisma.ProjectSelect + +export type ProjectWithDetails = Prisma.ProjectGetPayload<{ + select: typeof projectSelect +}> + +@Injectable() +export class NexusDatastoreService { + constructor(@Inject(PrismaService) private readonly prisma: PrismaService) {} + + async getAllProjects(): Promise { + return this.prisma.project.findMany({ + select: projectSelect, + where: { + plugins: { + some: { + pluginName: NEXUS_PLUGIN_NAME, + }, + }, + }, + }) + } + + async getProject(id: string): Promise { + return this.prisma.project.findUnique({ + where: { id }, + select: projectSelect, + }) + } +} diff --git a/apps/server-nestjs/src/modules/nexus/nexus-health.service.ts b/apps/server-nestjs/src/modules/nexus/nexus-health.service.ts new file mode 100644 index 000000000..2e894a96d --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus-health.service.ts @@ -0,0 +1,30 @@ +import { Inject, Injectable } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' + +@Injectable() +export class NexusHealthService { + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(HealthIndicatorService) private readonly healthIndicator: HealthIndicatorService, + ) {} + + async check(key: string) { + const indicator = this.healthIndicator.check(key) + if (!this.config.nexusInternalUrl) return indicator.down('Not configured') + + const url = new URL('/service/rest/v1/status', this.config.nexusInternalUrl).toString() + const headers: Record = {} + if (this.config.nexusAdmin && this.config.nexusAdminPassword) { + headers.Authorization = `Basic ${Buffer.from(`${this.config.nexusAdmin}:${this.config.nexusAdminPassword}`).toString('base64')}` + } + + try { + const response = await fetch(url, { method: 'GET', headers }) + if (response.status < 500) return indicator.up({ httpStatus: response.status }) + return indicator.down({ httpStatus: response.status }) + } catch (error) { + return indicator.down(error instanceof Error ? error.message : String(error)) + } + } +} diff --git a/apps/server-nestjs/src/modules/nexus/nexus-testing.utils.ts b/apps/server-nestjs/src/modules/nexus/nexus-testing.utils.ts new file mode 100644 index 000000000..7598f97d5 --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus-testing.utils.ts @@ -0,0 +1,13 @@ +import type { ProjectWithDetails } from './nexus-datastore.service' +import { faker } from '@faker-js/faker' + +export function makeProjectWithDetails(overrides: Partial = {}): ProjectWithDetails { + return { + slug: faker.internet.domainWord(), + owner: { + email: faker.internet.email(), + }, + plugins: [], + ...overrides, + } satisfies ProjectWithDetails +} diff --git a/apps/server-nestjs/src/modules/nexus/nexus.constants.ts b/apps/server-nestjs/src/modules/nexus/nexus.constants.ts new file mode 100644 index 000000000..d7fe70438 --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus.constants.ts @@ -0,0 +1,13 @@ +export const NEXUS_PLUGIN_NAME = 'nexus' + +export const NEXUS_CONFIG_KEYS = { + activateMavenRepo: 'activateMavenRepo', + activateNpmRepo: 'activateNpmRepo', + mavenSnapshotWritePolicy: 'mavenSnapshotWritePolicy', + mavenReleaseWritePolicy: 'mavenReleaseWritePolicy', + npmWritePolicy: 'npmWritePolicy', +} as const + +export const DEFAULT_MAVEN_SNAPSHOT_WRITE_POLICY = 'allow' +export const DEFAULT_MAVEN_RELEASE_WRITE_POLICY = 'allow_once' +export const DEFAULT_NPM_WRITE_POLICY = 'allow' diff --git a/apps/server-nestjs/src/modules/nexus/nexus.module.ts b/apps/server-nestjs/src/modules/nexus/nexus.module.ts new file mode 100644 index 000000000..eafcf3809 --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus.module.ts @@ -0,0 +1,16 @@ +import { Module } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationModule } from '../../cpin-module/infrastructure/configuration/configuration.module' +import { InfrastructureModule } from '../../cpin-module/infrastructure/infrastructure.module' +import { VaultModule } from '../vault/vault.module' +import { NexusClientService } from './nexus-client.service' +import { NexusDatastoreService } from './nexus-datastore.service' +import { NexusHealthService } from './nexus-health.service' +import { NexusService } from './nexus.service' + +@Module({ + imports: [ConfigurationModule, InfrastructureModule, VaultModule], + providers: [HealthIndicatorService, NexusHealthService, NexusService, NexusDatastoreService, NexusClientService], + exports: [NexusClientService, NexusHealthService], +}) +export class NexusModule {} diff --git a/apps/server-nestjs/src/modules/nexus/nexus.service.spec.ts b/apps/server-nestjs/src/modules/nexus/nexus.service.spec.ts new file mode 100644 index 000000000..7f5c47525 --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus.service.spec.ts @@ -0,0 +1,138 @@ +import type { Mocked } from 'vitest' +import { ENABLED } from '@cpn-console/shared' +import { Test } from '@nestjs/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { VaultClientService, VaultError } from '../vault/vault-client.service' +import { NexusClientService } from './nexus-client.service' +import { NexusDatastoreService } from './nexus-datastore.service' +import { makeProjectWithDetails } from './nexus-testing.utils' +import { NexusService } from './nexus.service' + +function createNexusControllerServiceTestingModule() { + return Test.createTestingModule({ + providers: [ + NexusService, + { + provide: NexusClientService, + useValue: { + getRepositoriesMavenHosted: vi.fn(), + createRepositoriesMavenHosted: vi.fn(), + updateRepositoriesMavenHosted: vi.fn(), + createRepositoriesMavenGroup: vi.fn(), + getRepositoriesNpmHosted: vi.fn(), + createRepositoriesNpmHosted: vi.fn(), + updateRepositoriesNpmHosted: vi.fn(), + getRepositoriesNpmGroup: vi.fn(), + postRepositoriesNpmGroup: vi.fn(), + putRepositoriesNpmGroup: vi.fn(), + getSecurityPrivileges: vi.fn(), + createSecurityPrivilegesRepositoryView: vi.fn(), + updateSecurityPrivilegesRepositoryView: vi.fn(), + deleteSecurityPrivileges: vi.fn(), + getSecurityRoles: vi.fn(), + createSecurityRoles: vi.fn(), + updateSecurityRoles: vi.fn(), + deleteSecurityRoles: vi.fn(), + getSecurityUsers: vi.fn(), + updateSecurityUsersChangePassword: vi.fn(), + createSecurityUsers: vi.fn(), + deleteSecurityUsers: vi.fn(), + deleteRepositoriesByName: vi.fn(), + } satisfies Partial, + }, + { + provide: NexusDatastoreService, + useValue: { + getAllProjects: vi.fn(), + } satisfies Partial, + }, + { + provide: VaultClientService, + useValue: { + read: vi.fn(), + write: vi.fn(), + delete: vi.fn(), + } satisfies Partial, + }, + { + provide: ConfigurationService, + useValue: { + projectRootPath: 'forge', + } satisfies Partial, + }, + ], + }) +} + +describe('nexusService', () => { + let service: NexusService + let client: Mocked + let nexusDatastore: Mocked + let vault: Mocked + + beforeEach(async () => { + const moduleRef = await createNexusControllerServiceTestingModule().compile() + service = moduleRef.get(NexusService) + client = moduleRef.get(NexusClientService) + nexusDatastore = moduleRef.get(NexusDatastoreService) + vault = moduleRef.get(VaultClientService) + + client.getRepositoriesMavenHosted.mockResolvedValue(null) + client.getRepositoriesNpmHosted.mockResolvedValue(null) + client.getRepositoriesNpmGroup.mockResolvedValue(null) + client.getSecurityPrivileges.mockResolvedValue(null) + client.getSecurityRoles.mockResolvedValue(null) + client.getSecurityUsers.mockResolvedValue([]) + vault.read.mockRejectedValue(new VaultError('NotFound', 'Not Found')) + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) + + it('handleUpsert should reconcile based on computed flags', async () => { + const project = makeProjectWithDetails({ + slug: 'project-1', + owner: { email: 'owner@example.com' }, + plugins: [ + { key: 'activateMavenRepo', value: ENABLED }, + { key: 'activateNpmRepo', value: 'disabled' }, + ], + }) + + await service.handleUpsert(project) + + expect(client.createRepositoriesMavenHosted).toHaveBeenCalled() + expect(client.deleteRepositoriesByName).toHaveBeenCalled() + expect(vault.write).toHaveBeenCalledWith( + expect.objectContaining({ + NEXUS_USERNAME: 'project-1', + NEXUS_PASSWORD: expect.any(String), + }), + 'forge/project-1/tech/NEXUS', + ) + }) + + it('handleDelete should delete project', async () => { + const project = makeProjectWithDetails({ slug: 'project-1' }) + await service.handleDelete(project) + expect(client.deleteSecurityRoles).toHaveBeenCalledWith('project-1-ID') + expect(client.deleteSecurityUsers).toHaveBeenCalledWith('project-1') + expect(vault.delete).toHaveBeenCalledWith('forge/project-1/tech/NEXUS') + }) + + it('handleCron should reconcile all projects', async () => { + const projects = [ + makeProjectWithDetails({ slug: 'project-1', plugins: [{ key: 'activateMavenRepo', value: ENABLED }] }), + makeProjectWithDetails({ slug: 'project-2', plugins: [{ key: 'activateNpmRepo', value: ENABLED }] }), + ] + + nexusDatastore.getAllProjects.mockResolvedValue(projects) + + await service.handleCron() + + expect(client.createSecurityUsers).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/server-nestjs/src/modules/nexus/nexus.service.ts b/apps/server-nestjs/src/modules/nexus/nexus.service.ts new file mode 100644 index 000000000..e0cf9312a --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus.service.ts @@ -0,0 +1,421 @@ +import type { ProjectWithDetails } from './nexus-datastore.service' +import { specificallyEnabled } from '@cpn-console/hooks' +import { Inject, Injectable, Logger } from '@nestjs/common' +import { OnEvent } from '@nestjs/event-emitter' +import { Cron, CronExpression } from '@nestjs/schedule' +import { trace } from '@opentelemetry/api' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' +import { VaultClientService, VaultError } from '../vault/vault-client.service' +import { NexusClientService } from './nexus-client.service' +import { NexusDatastoreService } from './nexus-datastore.service' +import { NEXUS_CONFIG_KEYS } from './nexus.constants' +import { generateMavenGroupPrivilegeName, generateMavenGroupRepoName, generateMavenHostedPrivilegeName, generateMavenHostedRepoName, generateNpmGroupPrivilegeName, generateNpmGroupRepoName, generateNpmHostedPrivilegeName, generateNpmHostedRepoName, generateRandomPassword, getPluginConfig, getProjectVaultPath } from './nexus.utils' + +@Injectable() +export class NexusService { + private readonly logger = new Logger(NexusService.name) + + constructor( + @Inject(NexusDatastoreService) private readonly nexusDatastore: NexusDatastoreService, + @Inject(NexusClientService) private readonly client: NexusClientService, + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(VaultClientService) private readonly vault: VaultClientService, + ) { + this.logger.log('NexusService initialized') + } + + @OnEvent('project.upsert') + @StartActiveSpan() + async handleUpsert(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project upsert for ${project.slug}`) + await this.ensureProject(project) + } + + @OnEvent('project.delete') + @StartActiveSpan() + async handleDelete(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', project.slug) + this.logger.log(`Handling project delete for ${project.slug}`) + await this.deleteProject(project.slug) + } + + @Cron(CronExpression.EVERY_HOUR) + @StartActiveSpan() + async handleCron() { + const span = trace.getActiveSpan() + this.logger.log('Starting Nexus reconciliation') + const projects = await this.nexusDatastore.getAllProjects() + span?.setAttribute('nexus.projects.count', projects.length) + await this.ensureProjects(projects) + } + + @StartActiveSpan() + private async ensureProjects(projects: ProjectWithDetails[]) { + const span = trace.getActiveSpan() + span?.setAttribute('nexus.projects.count', projects.length) + await Promise.all(projects.map(p => this.ensureProject(p))) + } + + @StartActiveSpan() + private async ensureProject(project: ProjectWithDetails) { + const span = trace.getActiveSpan() + const projectSlug = project.slug + span?.setAttribute('project.slug', projectSlug) + + const ownerEmail = project.owner.email + await Promise.all([ + this.ensureMavenRepo(project), + this.ensureNpmRepo(project), + ]) + + await this.ensureRole(projectSlug) + + const vaultPath = getProjectVaultPath(this.config.projectRootPath, projectSlug, 'tech/NEXUS') + let existingPassword: string | undefined + try { + const secret = await this.vault.read(vaultPath) + existingPassword = secret.data?.NEXUS_PASSWORD + } catch (error) { + if (error instanceof VaultError && error.kind === 'NotFound') { + existingPassword = undefined + } else { + throw error + } + } + const password = existingPassword ?? generateRandomPassword(30) + + await this.ensureUser(projectSlug, ownerEmail, password) + await this.vault.write({ + NEXUS_PASSWORD: password, + NEXUS_USERNAME: projectSlug, + }, vaultPath) + } + + private async ensureMavenRepo(project: ProjectWithDetails): Promise { + const enabled = specificallyEnabled(getPluginConfig(project, NEXUS_CONFIG_KEYS.activateMavenRepo)) + const span = trace.getActiveSpan() + span?.setAttribute('nexus.maven.enabled', enabled ?? false) + + if (!enabled) { + await this.deleteMavenRepos(project.slug) + } + + const mavenSnapshotWritePolicy = getPluginConfig(project, NEXUS_CONFIG_KEYS.mavenSnapshotWritePolicy) ?? 'allow' + const mavenReleaseWritePolicy = getPluginConfig(project, NEXUS_CONFIG_KEYS.mavenReleaseWritePolicy) ?? 'allow_once' + await this.ensureMavenRepos(project.slug, { + snapshotWritePolicy: mavenSnapshotWritePolicy, + releaseWritePolicy: mavenReleaseWritePolicy, + }) + } + + private async ensureNpmRepo(project: ProjectWithDetails): Promise { + const enabled = specificallyEnabled(getPluginConfig(project, NEXUS_CONFIG_KEYS.activateNpmRepo)) + const span = trace.getActiveSpan() + span?.setAttribute('nexus.npm.enabled', enabled ?? false) + + if (!enabled) { + await this.deleteNpmRepos(project.slug) + return [] + } + + const npmWritePolicy = getPluginConfig(project, NEXUS_CONFIG_KEYS.npmWritePolicy) ?? 'allow' + await this.createNpmRepos(project.slug, npmWritePolicy) + return [ + generateNpmGroupPrivilegeName(project.slug), + generateNpmHostedPrivilegeName(project.slug), + ] + } + + private async upsertPrivilege(body: { name: string, description: string, actions: string[], format: string, repository: string }) { + const existing = await this.client.getSecurityPrivileges(body.name) + if (!existing) { + await this.client.createSecurityPrivilegesRepositoryView(body) + return + } + await this.client.updateSecurityPrivilegesRepositoryView(body.name, body) + } + + private async ensureMavenHostedRepo(repoName: string, writePolicy: string) { + const existing = await this.client.getRepositoriesMavenHosted(repoName) + const body = { + name: repoName, + online: true, + storage: { + blobStoreName: 'default', + strictContentTypeValidation: true, + writePolicy, + }, + cleanup: { policyNames: ['string'] }, + component: { proprietaryComponents: true }, + maven: { + versionPolicy: 'MIXED', + layoutPolicy: 'STRICT', + contentDisposition: 'ATTACHMENT', + }, + } + if (!existing) { + await this.client.createRepositoriesMavenHosted(body) + return + } + await this.client.updateRepositoriesMavenHosted(repoName, body) + } + + private async ensureNpmHostedRepo(repoName: string, writePolicy: string) { + const existing = await this.client.getRepositoriesNpmHosted(repoName) + const body = { + name: repoName, + online: true, + storage: { + blobStoreName: 'default', + strictContentTypeValidation: true, + writePolicy, + }, + cleanup: { policyNames: ['string'] }, + component: { proprietaryComponents: true }, + } + if (!existing) { + await this.client.createRepositoriesNpmHosted(body) + return + } + await this.client.updateRepositoriesNpmHosted(repoName, body) + } + + private async ensureNpmGroupRepo(repoName: string, memberNames: string[]) { + const existing = await this.client.getRepositoriesNpmGroup(repoName) + const body = { + name: repoName, + online: true, + storage: { + blobStoreName: 'default', + strictContentTypeValidation: true, + }, + group: { + memberNames, + }, + } + if (!existing) { + await this.client.postRepositoriesNpmGroup(body) + return + } + await this.client.putRepositoriesNpmGroup(repoName, body) + } + + private async ensureMavenHostedRepos(args: { + releaseRepoName: string + snapshotRepoName: string + releaseWritePolicy: string + snapshotWritePolicy: string + }) { + await Promise.all([ + this.ensureMavenHostedRepo(args.snapshotRepoName, args.snapshotWritePolicy), + this.ensureMavenHostedRepo(args.releaseRepoName, args.releaseWritePolicy), + ]) + } + + private async ensureMavenRepos(projectSlug: string, options: { snapshotWritePolicy: string, releaseWritePolicy: string }) { + const releaseRepoName = generateMavenHostedRepoName(projectSlug, 'release') + const snapshotRepoName = generateMavenHostedRepoName(projectSlug, 'snapshot') + const groupRepoName = generateMavenGroupRepoName(projectSlug) + + const releasePrivilege = generateMavenHostedPrivilegeName(projectSlug, 'release') + const snapshotPrivilege = generateMavenHostedPrivilegeName(projectSlug, 'snapshot') + const groupPrivilege = generateMavenGroupPrivilegeName(projectSlug) + + await this.ensureMavenHostedRepos({ + releaseRepoName, + snapshotRepoName, + releaseWritePolicy: options.releaseWritePolicy, + snapshotWritePolicy: options.snapshotWritePolicy, + }) + + await this.ensureMavenGroupRepo( + groupRepoName, + [releaseRepoName, snapshotRepoName, 'maven-public'], + ) + + const privilegesToEnsure = [ + { repo: releaseRepoName, privilege: releasePrivilege }, + { repo: snapshotRepoName, privilege: snapshotPrivilege }, + { repo: groupRepoName, privilege: groupPrivilege }, + ] + await this.ensureMavenPrivileges(projectSlug, privilegesToEnsure) + } + + private async ensureMavenGroupRepo(repoName: string, memberNames: string[]) { + try { + await this.client.createRepositoriesMavenGroup({ + name: repoName, + online: true, + storage: { + blobStoreName: 'default', + strictContentTypeValidation: true, + }, + group: { + memberNames, + }, + }) + } catch { + } + } + + private async ensureMavenPrivileges(projectSlug: string, entries: Array<{ repo: string, privilege: string }>) { + for (const entry of entries) { + try { + await this.client.createSecurityPrivilegesRepositoryView({ + name: entry.privilege, + description: `Privilege for organization ${projectSlug} for repo ${entry.repo}`, + actions: ['all'], + format: 'maven2', + repository: entry.repo, + }) + } catch { + } + } + } + + private async deleteMavenRepos(projectSlug: string) { + const repoPaths = [ + { repo: generateMavenGroupRepoName(projectSlug) }, + { repo: generateMavenHostedRepoName(projectSlug, 'release') }, + { repo: generateMavenHostedRepoName(projectSlug, 'snapshot') }, + ] + const privileges = [ + { privilege: generateMavenGroupPrivilegeName(projectSlug) }, + { privilege: generateMavenHostedPrivilegeName(projectSlug, 'release') }, + { privilege: generateMavenHostedPrivilegeName(projectSlug, 'snapshot') }, + ] + const pathsToDelete = [ + ...privileges.map(({ privilege }) => `/security/privileges/${encodeURIComponent(privilege)}`), + ...repoPaths.map(repo => `/repositories/${encodeURIComponent(repo.repo)}`), + ] + for (const path of pathsToDelete) { + if (path.startsWith('/security/privileges/')) { + const name = decodeURIComponent(path.split('/').pop()!) + await this.client.deleteSecurityPrivileges(name) + } else if (path.startsWith('/repositories/')) { + const name = decodeURIComponent(path.split('/').pop()!) + await this.client.deleteRepositoriesByName(name) + } + } + } + + private async createNpmRepos(projectSlug: string, writePolicy: string) { + const hostedRepoName = generateNpmHostedRepoName(projectSlug) + const groupRepoName = generateNpmGroupRepoName(projectSlug) + + const hostedPrivilege = generateNpmHostedPrivilegeName(projectSlug) + const groupPrivilege = generateNpmGroupPrivilegeName(projectSlug) + + await this.ensureNpmHostedRepo(hostedRepoName, writePolicy) + await this.ensureNpmGroupRepo(groupRepoName, [hostedRepoName]) + + for (const name of [ + { repo: hostedRepoName, privilege: hostedPrivilege }, + { repo: groupRepoName, privilege: groupPrivilege }, + ]) { + await this.upsertPrivilege({ + name: name.privilege, + description: `Privilege for organization ${projectSlug} for repo ${name.repo}`, + actions: ['all'], + format: 'npm', + repository: name.repo, + }) + } + } + + private async deleteNpmRepos(projectSlug: string) { + const repoPaths = [ + { repo: generateNpmGroupRepoName(projectSlug) }, + { repo: generateNpmHostedRepoName(projectSlug) }, + ] + const privileges = [ + { privilege: generateNpmGroupPrivilegeName(projectSlug) }, + { privilege: generateNpmHostedPrivilegeName(projectSlug) }, + ] + const pathsToDelete = [ + ...privileges.map(({ privilege }) => `/security/privileges/${encodeURIComponent(privilege)}`), + ...repoPaths.map(repo => `/repositories/${encodeURIComponent(repo.repo)}`), + ] + for (const path of pathsToDelete) { + if (path.startsWith('/security/privileges/')) { + const name = decodeURIComponent(path.split('/').pop()!) + await this.client.deleteSecurityPrivileges(name) + } else if (path.startsWith('/repositories/')) { + const name = decodeURIComponent(path.split('/').pop()!) + await this.client.deleteRepositoriesByName(name) + } + } + } + + private async ensureRole(projectSlug: string) { + const privileges = [ + generateMavenGroupPrivilegeName(projectSlug), + generateMavenHostedPrivilegeName(projectSlug, 'release'), + generateMavenHostedPrivilegeName(projectSlug, 'snapshot'), + generateNpmGroupPrivilegeName(projectSlug), + generateNpmHostedPrivilegeName(projectSlug), + ] + const roleId = `${projectSlug}-ID` + const role = await this.client.getSecurityRoles(roleId) + if (!role) { + await this.client.createSecurityRoles({ + id: roleId, + name: `${projectSlug}-role`, + description: 'desc', + privileges, + }) + return + } + await this.client.updateSecurityRoles(roleId, { + id: roleId, + name: `${projectSlug}-role`, + privileges, + }) + } + + private async ensureUser(projectSlug: string, ownerEmail: string, password: string) { + const users = await this.client.getSecurityUsers(projectSlug) + const existing = users.find(u => u.userId === projectSlug) + if (existing) { + await this.client.updateSecurityUsersChangePassword(projectSlug, password) + return + } + + await this.client.createSecurityUsers({ + userId: projectSlug, + firstName: 'Monkey D.', + lastName: 'Luffy', + emailAddress: ownerEmail, + password, + status: 'active', + roles: [`${projectSlug}-ID`], + }) + } + + @StartActiveSpan() + private async deleteProject(projectSlug: string) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', projectSlug) + await Promise.all([ + this.deleteMavenRepos(projectSlug), + this.deleteNpmRepos(projectSlug), + ]) + + await Promise.all([ + this.client.deleteSecurityRoles(`${projectSlug}-ID`), + this.client.deleteSecurityUsers(projectSlug), + ]) + + const vaultPath = getProjectVaultPath(this.config.projectRootPath, projectSlug, 'tech/NEXUS') + try { + await this.vault.delete(vaultPath) + } catch (error) { + if (error instanceof VaultError && error.kind === 'NotFound') return + throw error + } + } +} diff --git a/apps/server-nestjs/src/modules/nexus/nexus.utils.ts b/apps/server-nestjs/src/modules/nexus/nexus.utils.ts new file mode 100644 index 000000000..8029f7f3c --- /dev/null +++ b/apps/server-nestjs/src/modules/nexus/nexus.utils.ts @@ -0,0 +1,68 @@ +import type { ProjectWithDetails } from './nexus-datastore.service' +import { randomBytes } from 'node:crypto' + +const trailingSlashesRegex = /\/+$/u + +export function removeTrailingSlash(value: string) { + return value.replace(trailingSlashesRegex, '') +} + +export function getPluginConfig(project: ProjectWithDetails, key: string) { + return project.plugins?.find(p => p.key === key)?.value +} + +export type WritePolicy = 'allow' | 'allow_once' | 'deny' | 'replication_only' + +export const writePolicies: WritePolicy[] = ['allow', 'allow_once', 'deny', 'replication_only'] + +export function assertWritePolicy(value: string): asserts value is WritePolicy { + if (!writePolicies.includes(value as WritePolicy)) { + throw new Error(`Invalid writePolicy: ${value}`) + } +} + +export function generateRandomPassword(length: number) { + const raw = randomBytes(Math.ceil(length * 0.75)).toString('base64url') + return raw.slice(0, length) +} + +export function getProjectVaultPath(projectRootPath: string | undefined, projectSlug: string, relativePath: string) { + const normalized = relativePath.startsWith('/') ? relativePath.slice(1) : relativePath + return projectRootPath + ? `${projectRootPath}/${projectSlug}/${normalized}` + : `${projectSlug}/${normalized}` +} + +export type MavenHostedRepoKind = 'release' | 'snapshot' + +export function generateMavenHostedRepoName(projectSlug: string, kind: MavenHostedRepoKind) { + return `${projectSlug}-repository-${kind}` +} + +export function generateMavenHostedPrivilegeName(projectSlug: string, kind: MavenHostedRepoKind) { + return `${projectSlug}-privilege-${kind}` +} + +export function generateMavenGroupRepoName(projectSlug: string) { + return `${projectSlug}-repository-group` +} + +export function generateMavenGroupPrivilegeName(projectSlug: string) { + return `${projectSlug}-privilege-group` +} + +export function generateNpmHostedRepoName(projectSlug: string) { + return `${projectSlug}-npm` +} + +export function generateNpmHostedPrivilegeName(projectSlug: string) { + return `${projectSlug}-npm-privilege` +} + +export function generateNpmGroupRepoName(projectSlug: string) { + return `${projectSlug}-npm-group` +} + +export function generateNpmGroupPrivilegeName(projectSlug: string) { + return `${projectSlug}-npm-group-privilege` +} diff --git a/apps/server-nestjs/src/modules/registry/registry-client.service.spec.ts b/apps/server-nestjs/src/modules/registry/registry-client.service.spec.ts new file mode 100644 index 000000000..34bb175a7 --- /dev/null +++ b/apps/server-nestjs/src/modules/registry/registry-client.service.spec.ts @@ -0,0 +1,48 @@ +import type { TestingModule } from '@nestjs/testing' +import { Test } from '@nestjs/testing' +import { beforeEach, describe, expect, it } from 'vitest' +import { mockDeep, mockReset } from 'vitest-mock-extended' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { VaultClientService } from '../vault/vault-client.service' +import { RegistryClientService } from './registry-client.service' + +const vaultMock = mockDeep() + +function createRegistryServiceTestingModule() { + return Test.createTestingModule({ + providers: [ + RegistryClientService, + { + provide: VaultClientService, + useValue: vaultMock, + }, + { + provide: ConfigurationService, + useValue: { + harborUrl: 'https://harbor.example', + harborInternalUrl: 'https://harbor.example', + harborAdmin: 'admin', + harborAdminPassword: 'password', + harborRuleTemplate: 'latestPushedK', + harborRuleCount: '10', + harborRetentionCron: '0 22 2 * * *', + projectRootPath: 'forge', + } satisfies Partial, + }, + ], + }) +} + +describe('registryService', () => { + let service: RegistryClientService + + beforeEach(async () => { + mockReset(vaultMock) + const module: TestingModule = await createRegistryServiceTestingModule().compile() + service = module.get(RegistryClientService) + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) +}) diff --git a/apps/server-nestjs/src/modules/registry/registry-client.service.ts b/apps/server-nestjs/src/modules/registry/registry-client.service.ts new file mode 100644 index 000000000..0242548d0 --- /dev/null +++ b/apps/server-nestjs/src/modules/registry/registry-client.service.ts @@ -0,0 +1,491 @@ +import type { VaultRobotSecret } from './registry.utils' +import { Inject, Injectable } from '@nestjs/common' +import { trace } from '@opentelemetry/api' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' +import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' +import { VaultClientService, VaultError } from '../vault/vault-client.service' +import { + encodeBasicAuth, + getHostFromUrl, + getProjectVaultPath, + removeTrailingSlash, + toVaultRobotSecret, +} from './registry.utils' + +export interface HarborAccess { + resource: string + action: string +} + +export const roRobotName = 'ro-robot' +export const rwRobotName = 'rw-robot' +export const projectRobotName = 'project-robot' + +export const roAccess: HarborAccess[] = [ + { resource: 'repository', action: 'pull' }, + { resource: 'artifact', action: 'read' }, +] + +export const rwAccess: HarborAccess[] = [ + ...roAccess, + { resource: 'repository', action: 'list' }, + { resource: 'tag', action: 'list' }, + { resource: 'artifact', action: 'list' }, + { resource: 'scan', action: 'create' }, + { resource: 'scan', action: 'stop' }, + { resource: 'repository', action: 'push' }, + { resource: 'artifact-label', action: 'create' }, + { resource: 'artifact-label', action: 'delete' }, + { resource: 'tag', action: 'create' }, + { resource: 'tag', action: 'delete' }, +] + +interface HarborProject { + project_id?: number + metadata?: Record +} + +interface HarborRobot { + id?: number + name?: string +} + +interface HarborRobotCreated { + id?: number + name: string + secret: string +} + +const allowedRuleTemplates = [ + 'always', + 'latestPulledK', + 'latestPushedK', + 'nDaysSinceLastPull', + 'nDaysSinceLastPush', +] as const + +type RuleTemplate = typeof allowedRuleTemplates[number] + +interface HarborRequestOptions { + method?: string + headers?: Record + body?: any +} + +@Injectable() +export class RegistryClientService { + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(VaultClientService) private readonly vault: VaultClientService, + ) {} + + private get baseUrl() { + return `${removeTrailingSlash(this.config.harborInternalUrl!)}/api/v2.0` + } + + private get defaultHeaders() { + return { + Accept: 'application/json', + Authorization: `Basic ${encodeBasicAuth(this.config.harborAdmin!, this.config.harborAdminPassword!)}`, + } as const + } + + async request( + path: string, + options: HarborRequestOptions = {}, + ): Promise<{ status: number, data: T | null }> { + const url = `${this.baseUrl}${path.startsWith('/') ? '' : '/'}${path}` + const headers: Record = { + ...this.defaultHeaders, + ...options.headers, + } + if (options.body) headers['Content-Type'] = 'application/json' + const response = await fetch(url, { + method: options.method ?? 'GET', + headers, + body: options.body ? JSON.stringify(options.body) : undefined, + }) + + if (response.status === 204) return { status: response.status, data: null } + + const contentType = response.headers.get('content-type') ?? '' + const body = contentType.includes('application/json') + ? await response.json() + : await response.text() + + return { status: response.status, data: body as T } + } + + async getProjectByName(projectName: string) { + return this.request(`/projects/${encodeURIComponent(projectName)}`, { + method: 'GET', + headers: { 'X-Is-Resource-Name': 'true' }, + }) + } + + async createProject(projectName: string, storageLimit: number) { + return this.request('/projects', { + method: 'POST', + body: { + project_name: projectName, + metadata: { auto_scan: 'true' }, + storage_limit: storageLimit, + }, + }) + } + + async deleteProjectByName(projectName: string) { + return this.request(`/projects/${encodeURIComponent(projectName)}`, { + method: 'DELETE', + headers: { 'X-Is-Resource-Name': 'true' }, + }) + } + + async listQuotas(projectId: number) { + return this.request(`/quotas?reference_id=${encodeURIComponent(String(projectId))}`, { + method: 'GET', + }) + } + + async updateQuota(projectId: number, storageLimit: number) { + return this.request(`/quotas/${encodeURIComponent(String(projectId))}`, { + method: 'PUT', + body: { + hard: { + storage: storageLimit, + }, + }, + }) + } + + async listProjectMembers(projectName: string) { + return this.request(`/projects/${encodeURIComponent(projectName)}/members`, { + method: 'GET', + headers: { 'X-Is-Resource-Name': 'true' }, + }) + } + + async createProjectMember(projectName: string, body: any) { + return this.request(`/projects/${encodeURIComponent(projectName)}/members`, { + method: 'POST', + headers: { 'X-Is-Resource-Name': 'true' }, + body, + }) + } + + async deleteProjectMember(projectName: string, memberId: number) { + return this.request(`/projects/${encodeURIComponent(projectName)}/members/${encodeURIComponent(String(memberId))}`, { + method: 'DELETE', + headers: { 'X-Is-Resource-Name': 'true' }, + }) + } + + async listProjectRobots(projectName: string) { + return this.request(`/projects/${encodeURIComponent(projectName)}/robots`, { + method: 'GET', + headers: { 'X-Is-Resource-Name': 'true' }, + }) + } + + async createRobotApi(body: any) { + return this.request('/robots', { + method: 'POST', + body, + }) + } + + async deleteRobot(projectName: string, robotId: number) { + const direct = await this.request(`/robots/${encodeURIComponent(String(robotId))}`, { + method: 'DELETE', + }) + if (direct.status < 300 || direct.status === 404) return direct + + return this.request(`/projects/${encodeURIComponent(projectName)}/robots/${encodeURIComponent(String(robotId))}`, { + method: 'DELETE', + headers: { 'X-Is-Resource-Name': 'true' }, + }) + } + + async getRetentionId(projectName: string): Promise { + const project = await this.getProjectByName(projectName) + if (project.status !== 200 || !project.data) return null + const retentionId = Number((project.data as any)?.metadata?.retention_id) + return Number.isFinite(retentionId) ? retentionId : null + } + + async createRetention(body: any) { + return this.request('/retentions', { + method: 'POST', + body, + }) + } + + async updateRetention(retentionId: number, body: any) { + return this.request(`/retentions/${encodeURIComponent(String(retentionId))}`, { + method: 'PUT', + body, + }) + } + + private get harborHost() { + return getHostFromUrl(this.config.harborUrl!) + } + + private getRobotFullName(projectSlug: string, robotName: string) { + return `robot$${projectSlug}+${robotName}` + } + + private async getRobot(projectSlug: string, robotName: string): Promise { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'registry.robot.name': robotName, + }) + const robots = await this.listProjectRobots(projectSlug) + if (robots.status !== 200 || !robots.data) return undefined + const fullName = this.getRobotFullName(projectSlug, robotName) + return (robots.data as any[]).find(r => r?.name === fullName) + } + + private getRobotPermissions(projectSlug: string, robotName: string, access: HarborAccess[]) { + return { + name: robotName, + duration: -1, + description: 'robot for ci builds', + disable: false, + level: 'project', + permissions: [{ + namespace: projectSlug, + kind: 'project', + access, + }], + } + } + + private async createProjectRobot(projectSlug: string, robotName: string, access: HarborAccess[]): Promise { + const created = await this.createRobotApi( + this.getRobotPermissions(projectSlug, robotName, access), + ) + if (created.status >= 300 || !created.data) { + throw new Error(`Harbor create robot failed (${created.status})`) + } + return created.data as HarborRobotCreated + } + + private async regenerateRobot(projectSlug: string, robotName: string, access: HarborAccess[]): Promise { + const existing = await this.getRobot(projectSlug, robotName) + if (existing?.id) { + await this.deleteRobot(projectSlug, existing.id) + } + return this.createProjectRobot(projectSlug, robotName, access) + } + + private async ensureRobot(projectSlug: string, robotName: string, access: HarborAccess[]): Promise { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'registry.robot.name': robotName, + }) + const relativeVaultPath = `REGISTRY/${robotName}` + const vaultPath = getProjectVaultPath(this.config.projectRootPath, projectSlug, relativeVaultPath) + let vaultRobotSecret: VaultRobotSecret | null = null + try { + const secret = await this.vault.read(vaultPath) + vaultRobotSecret = secret.data as VaultRobotSecret + } catch (error) { + if (!(error instanceof VaultError && error.kind === 'NotFound')) { + throw error + } + } + + if (vaultRobotSecret?.HOST === this.harborHost) { + span?.setAttribute('vault.secret.reused', true) + return vaultRobotSecret + } + + const existing = await this.getRobot(projectSlug, robotName) + const created = existing + ? await this.regenerateRobot(projectSlug, robotName, access) + : await this.createProjectRobot(projectSlug, robotName, access) + const fullName = this.getRobotFullName(projectSlug, robotName) + const secret = toVaultRobotSecret(this.harborHost, fullName, created.secret) + await this.vault.write(secret, vaultPath) + span?.setAttribute('vault.secret.written', true) + return secret + } + + @StartActiveSpan() + async addProjectGroupMember(projectSlug: string, groupName: string, accessLevel: number = 3) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'registry.group.name': groupName, + 'registry.group.access_level': accessLevel, + }) + const members = await this.listProjectMembers(projectSlug) + if (members.status !== 200 || !members.data) { + throw new Error(`Harbor list members failed (${members.status})`) + } + const list = members.data as any[] + const existing = list.find(m => m?.entity_name === groupName) + + if (existing?.id) { + if (existing.role_id !== accessLevel && existing.entity_type !== 'g') { + await this.deleteProjectMember(projectSlug, Number(existing.id)) + } else { + span?.setAttribute('registry.member.exists', true) + return + } + } + + const created = await this.createProjectMember(projectSlug, { + role_id: accessLevel, + member_group: { + group_name: groupName, + group_type: 3, + }, + }) + if (created.status >= 300) { + throw new Error(`Harbor create member failed (${created.status})`) + } + span?.setAttribute('registry.member.created', true) + } + + private async createOrUpdateProject(projectSlug: string, storageLimit: number): Promise { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'registry.storage_limit.bytes': storageLimit, + }) + const existing = await this.getProjectByName(projectSlug) + if (existing.status === 200 && existing.data) { + const project = existing.data as HarborProject + const projectId = Number(project.project_id) + if (!Number.isFinite(projectId)) return project + + const quotas = await this.listQuotas(projectId) + if (quotas.status === 200 && quotas.data) { + const hardQuota = (quotas.data as any[]).find(q => q?.ref?.id === projectId) + if (hardQuota?.hard?.storage !== storageLimit) { + await this.updateQuota(projectId, storageLimit) + span?.setAttribute('registry.quota.updated', true) + } + } + return project + } + + const created = await this.createProject(projectSlug, storageLimit) + if (created.status >= 300) { + throw new Error(`Harbor create project failed (${created.status})`) + } + span?.setAttribute('registry.project.created', true) + + const fetched = await this.getProjectByName(projectSlug) + if (fetched.status !== 200 || !fetched.data) { + throw new Error(`Harbor get project failed (${fetched.status})`) + } + return fetched.data as HarborProject + } + + private getRetentionPolicy(projectId: number) { + const template = allowedRuleTemplates.includes(this.config.harborRuleTemplate as RuleTemplate) + ? this.config.harborRuleTemplate as RuleTemplate + : 'latestPushedK' + + const rawCount = Number(this.config.harborRuleCount) + const count = Number.isFinite(rawCount) && rawCount > 0 + ? rawCount + : template === 'always' + ? 1 + : 10 + + const cron = this.config.harborRetentionCron?.trim() || '0 22 2 * * *' + + return { + algorithm: 'or', + scope: { level: 'project', ref: projectId }, + rules: [ + { + disabled: false, + action: 'retain', + template, + params: { [template]: count }, + tag_selectors: [ + { kind: 'doublestar', decoration: 'matches', pattern: '**' }, + ], + scope_selectors: { + repository: [ + { kind: 'doublestar', decoration: 'repoMatches', pattern: '**' }, + ], + }, + }, + ], + trigger: { + kind: 'Schedule', + settings: { cron }, + references: [], + }, + } + } + + private async upsertRetentionPolicy(projectSlug: string, projectId: number) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'registry.project.id': projectId, + }) + const policy = this.getRetentionPolicy(projectId) + const retentionId = await this.getRetentionId(projectSlug) + span?.setAttribute('registry.retention.exists', !!retentionId) + const result = retentionId + ? await this.updateRetention(retentionId, policy) + : await this.createRetention(policy) + if (result.status >= 300) { + throw new Error(`Harbor retention policy failed (${result.status})`) + } + } + + @StartActiveSpan() + async provisionProject(projectSlug: string, options: { storageLimitBytes?: number, publishProjectRobot?: boolean } = {}) { + const span = trace.getActiveSpan() + span?.setAttributes({ + 'project.slug': projectSlug, + 'registry.publish_project_robot': !!options.publishProjectRobot, + }) + const storageLimit = options.storageLimitBytes ?? -1 + const project = await this.createOrUpdateProject(projectSlug, storageLimit) + const projectId = Number(project.project_id) + + const groupName = `/${projectSlug}` + + await Promise.all([ + this.ensureRobot(projectSlug, roRobotName, roAccess), + this.ensureRobot(projectSlug, rwRobotName, rwAccess), + this.addProjectGroupMember(projectSlug, groupName), + Number.isFinite(projectId) ? this.upsertRetentionPolicy(projectSlug, projectId) : Promise.resolve(), + options.publishProjectRobot + ? this.ensureRobot(projectSlug, projectRobotName, roAccess) + : Promise.resolve(), + ]) + + return { + projectId: Number.isFinite(projectId) ? projectId : undefined, + basePath: `${this.harborHost}/${projectSlug}/`, + } + } + + @StartActiveSpan() + async deleteProject(projectSlug: string) { + const span = trace.getActiveSpan() + span?.setAttribute('project.slug', projectSlug) + const existing = await this.getProjectByName(projectSlug) + if (existing.status === 404) { + span?.setAttribute('registry.project.exists', false) + return + } + const deleted = await this.deleteProjectByName(projectSlug) + if (deleted.status >= 300 && deleted.status !== 404) { + throw new Error(`Harbor delete project failed (${deleted.status})`) + } + } +} diff --git a/apps/server-nestjs/src/modules/registry/registry-health.service.ts b/apps/server-nestjs/src/modules/registry/registry-health.service.ts new file mode 100644 index 000000000..4494667af --- /dev/null +++ b/apps/server-nestjs/src/modules/registry/registry-health.service.ts @@ -0,0 +1,30 @@ +import { Inject, Injectable } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' + +@Injectable() +export class RegistryHealthService { + constructor( + @Inject(ConfigurationService) private readonly config: ConfigurationService, + @Inject(HealthIndicatorService) private readonly healthIndicator: HealthIndicatorService, + ) {} + + async check(key: string) { + const indicator = this.healthIndicator.check(key) + if (!this.config.harborInternalUrl) return indicator.down('Not configured') + + const url = new URL('/api/v2.0/ping', this.config.harborInternalUrl).toString() + const headers: Record = {} + if (this.config.harborAdmin && this.config.harborAdminPassword) { + headers.Authorization = `Basic ${Buffer.from(`${this.config.harborAdmin}:${this.config.harborAdminPassword}`).toString('base64')}` + } + + try { + const response = await fetch(url, { method: 'GET', headers }) + if (response.status < 500) return indicator.up({ httpStatus: response.status }) + return indicator.down({ httpStatus: response.status }) + } catch (error) { + return indicator.down(error instanceof Error ? error.message : String(error)) + } + } +} diff --git a/apps/server-nestjs/src/modules/registry/registry.module.ts b/apps/server-nestjs/src/modules/registry/registry.module.ts new file mode 100644 index 000000000..678c35d47 --- /dev/null +++ b/apps/server-nestjs/src/modules/registry/registry.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common' +import { HealthIndicatorService } from '@nestjs/terminus' +import { ConfigurationModule } from '../../cpin-module/infrastructure/configuration/configuration.module' +import { InfrastructureModule } from '../../cpin-module/infrastructure/infrastructure.module' +import { VaultModule } from '../vault/vault.module' +import { RegistryClientService } from './registry-client.service' +import { RegistryHealthService } from './registry-health.service' + +@Module({ + imports: [ConfigurationModule, InfrastructureModule, VaultModule], + providers: [HealthIndicatorService, RegistryHealthService, RegistryClientService, RegistryClientService], + exports: [RegistryHealthService], +}) +export class RegistryModule {} diff --git a/apps/server-nestjs/src/modules/registry/registry.utils.ts b/apps/server-nestjs/src/modules/registry/registry.utils.ts new file mode 100644 index 000000000..890b4e19a --- /dev/null +++ b/apps/server-nestjs/src/modules/registry/registry.utils.ts @@ -0,0 +1,46 @@ +const trailingSlashesRegex = /\/+$/u +const protocolPrefixRegex = /^https?:\/\//u + +export function removeTrailingSlash(value: string) { + return value.replace(trailingSlashesRegex, '') +} + +export function getHostFromUrl(url: string) { + return removeTrailingSlash(url).replace(protocolPrefixRegex, '').split('/')[0] +} + +export function encodeBasicAuth(username: string, password: string) { + return Buffer.from(`${username}:${password}`).toString('base64') +} + +export interface VaultRobotSecret { + DOCKER_CONFIG: string + HOST: string + TOKEN: string + USERNAME: string +} + +export function toVaultRobotSecret(host: string, robotName: string, robotSecret: string): VaultRobotSecret { + const auth = `${robotName}:${robotSecret}` + const b64auth = Buffer.from(auth).toString('base64') + return { + DOCKER_CONFIG: JSON.stringify({ + auths: { + [host]: { + auth: b64auth, + email: '', + }, + }, + }), + HOST: host, + TOKEN: robotSecret, + USERNAME: robotName, + } +} + +export function getProjectVaultPath(projectRootPath: string | undefined, projectSlug: string, relativePath: string) { + const normalized = relativePath.startsWith('/') ? relativePath.slice(1) : relativePath + return projectRootPath + ? `${projectRootPath}/${projectSlug}/${normalized}` + : `${projectSlug}/${normalized}` +} diff --git a/apps/server-nestjs/test/nexus.e2e-spec.ts b/apps/server-nestjs/test/nexus.e2e-spec.ts new file mode 100644 index 000000000..437bde638 --- /dev/null +++ b/apps/server-nestjs/test/nexus.e2e-spec.ts @@ -0,0 +1,148 @@ +import type { TestingModule } from '@nestjs/testing' +import { faker } from '@faker-js/faker' +import { Test } from '@nestjs/testing' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { ConfigurationModule } from '../src/cpin-module/infrastructure/configuration/configuration.module' +import { ConfigurationService } from '../src/cpin-module/infrastructure/configuration/configuration.service' +import { PrismaService } from '../src/cpin-module/infrastructure/database/prisma.service' +import { InfrastructureModule } from '../src/cpin-module/infrastructure/infrastructure.module' +import { NexusClientService } from '../src/modules/nexus/nexus-client.service' +import { projectSelect } from '../src/modules/nexus/nexus-datastore.service' +import { NEXUS_PLUGIN_NAME } from '../src/modules/nexus/nexus.constants' +import { NexusModule } from '../src/modules/nexus/nexus.module' +import { NexusService } from '../src/modules/nexus/nexus.service' +import { getProjectVaultPath } from '../src/modules/nexus/nexus.utils' +import { VaultClientService } from '../src/modules/vault/vault-client.service' +import { VaultModule } from '../src/modules/vault/vault.module' + +const canRunNexusE2E + = Boolean(process.env.E2E) + && Boolean(process.env.NEXUS_URL) + && Boolean(process.env.NEXUS_ADMIN) + && Boolean(process.env.NEXUS_ADMIN_PASSWORD) + && Boolean(process.env.VAULT_URL) + && Boolean(process.env.VAULT_TOKEN) + && Boolean(process.env.DB_URL) + +const describeWithNexus = describe.runIf(canRunNexusE2E) + +describeWithNexus('NexusController (e2e)', () => { + let moduleRef: TestingModule + let nexusController: NexusService + let nexusClient: NexusClientService + let vaultService: VaultClientService + let config: ConfigurationService + let prisma: PrismaService + + let ownerId: string + let testProjectId: string + let testProjectSlug: string + + beforeAll(async () => { + moduleRef = await Test.createTestingModule({ + imports: [NexusModule, VaultModule, ConfigurationModule, InfrastructureModule], + }).compile() + + await moduleRef.init() + + nexusController = moduleRef.get(NexusService) + nexusClient = moduleRef.get(NexusClientService) + vaultService = moduleRef.get(VaultClientService) + config = moduleRef.get(ConfigurationService) + prisma = moduleRef.get(PrismaService) + + ownerId = faker.string.uuid() + testProjectId = faker.string.uuid() + testProjectSlug = faker.helpers.slugify(`test-project-${faker.string.uuid()}`) + + await prisma.user.create({ + data: { + id: ownerId, + email: faker.internet.email().toLowerCase(), + firstName: 'Test', + lastName: 'Owner', + type: 'human', + }, + }) + }) + + afterAll(async () => { + if (testProjectSlug) { + await nexusController.handleDelete({ slug: testProjectSlug } as any).catch(() => {}) + } + + if (prisma) { + await prisma.project.deleteMany({ where: { id: testProjectId } }).catch(() => {}) + await prisma.user.deleteMany({ where: { id: ownerId } }).catch(() => {}) + } + + await moduleRef.close() + + vi.restoreAllMocks() + vi.unstubAllEnvs() + }) + + it('should reconcile project in Nexus (repos, role, user, vault secret)', async () => { + await prisma.project.create({ + data: { + id: testProjectId, + slug: testProjectSlug, + name: testProjectSlug, + ownerId, + description: 'E2E Test Project', + hprodCpu: 0, + hprodGpu: 0, + hprodMemory: 0, + prodCpu: 0, + prodGpu: 0, + prodMemory: 0, + plugins: { + create: [ + { pluginName: NEXUS_PLUGIN_NAME, key: 'activateMavenRepo', value: 'enabled' }, + { pluginName: NEXUS_PLUGIN_NAME, key: 'activateNpmRepo', value: 'enabled' }, + ], + }, + }, + }) + + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + + await nexusController.handleUpsert(project) + + const mavenReleaseRepo = `${testProjectSlug}-repository-release` + const mavenSnapshotRepo = `${testProjectSlug}-repository-snapshot` + const mavenGroupRepo = `${testProjectSlug}-repository-group` + + const npmHostedRepo = `${testProjectSlug}-npm` + const npmGroupRepo = `${testProjectSlug}-npm-group` + + const [releaseRepo, snapshotRepo, groupRepo, npmRepo, npmGroup] = await Promise.all([ + nexusClient.getRepositoriesMavenHosted(mavenReleaseRepo), + nexusClient.getRepositoriesMavenHosted(mavenSnapshotRepo), + nexusClient.getRepositoriesMavenGroup(mavenGroupRepo), + nexusClient.getRepositoriesNpmHosted(npmHostedRepo), + nexusClient.getRepositoriesNpmGroup(npmGroupRepo), + ]) + + expect(releaseRepo).toBeTruthy() + expect(snapshotRepo).toBeTruthy() + expect(groupRepo).toBeTruthy() + expect(npmRepo).toBeTruthy() + expect(npmGroup).toBeTruthy() + + const roleId = `${testProjectSlug}-ID` + const role = await nexusClient.getSecurityRoles(roleId) + expect(role).toBeTruthy() + + const users = await nexusClient.getSecurityUsers(testProjectSlug) + expect(users.some(u => u.userId === testProjectSlug)).toBe(true) + + const vaultPath = getProjectVaultPath(config.projectRootPath, testProjectSlug, 'tech/NEXUS') + const secret = await vaultService.read(vaultPath) + expect(secret.data?.NEXUS_USERNAME).toBe(testProjectSlug) + expect(secret.data?.NEXUS_PASSWORD).toBeTruthy() + }) +}) From 44c9343d9fcaaba80bd93408006e0e2b5fdf6ff7 Mon Sep 17 00:00:00 2001 From: William Phetsinorath Date: Wed, 25 Mar 2026 14:16:19 +0100 Subject: [PATCH 4/4] chore: add reconciler Signed-off-by: William Phetsinorath --- .../reconcile/reconcile.constants.ts | 3 + .../reconcile/reconcile.decorator.ts | 103 ++++++++++++++++++ .../src/modules/nexus/nexus.service.ts | 1 + 3 files changed, 107 insertions(+) create mode 100644 apps/server-nestjs/src/cpin-module/infrastructure/reconcile/reconcile.constants.ts create mode 100644 apps/server-nestjs/src/cpin-module/infrastructure/reconcile/reconcile.decorator.ts diff --git a/apps/server-nestjs/src/cpin-module/infrastructure/reconcile/reconcile.constants.ts b/apps/server-nestjs/src/cpin-module/infrastructure/reconcile/reconcile.constants.ts new file mode 100644 index 000000000..84489806a --- /dev/null +++ b/apps/server-nestjs/src/cpin-module/infrastructure/reconcile/reconcile.constants.ts @@ -0,0 +1,3 @@ +export const DEFAULT_RECONCILE_MAX_RETRIES = 5 +export const DEFAULT_RECONCILE_REQUEUE_AFTER_MS = 0 +export const DEFAULT_RECONCILE_ERROR_REQUEUE_AFTER_MS = 0 diff --git a/apps/server-nestjs/src/cpin-module/infrastructure/reconcile/reconcile.decorator.ts b/apps/server-nestjs/src/cpin-module/infrastructure/reconcile/reconcile.decorator.ts new file mode 100644 index 000000000..490c4a0d9 --- /dev/null +++ b/apps/server-nestjs/src/cpin-module/infrastructure/reconcile/reconcile.decorator.ts @@ -0,0 +1,103 @@ +import { setTimeout } from 'node:timers/promises' +import { Logger } from '@nestjs/common' +import { + DEFAULT_RECONCILE_ERROR_REQUEUE_AFTER_MS, + DEFAULT_RECONCILE_MAX_RETRIES, + DEFAULT_RECONCILE_REQUEUE_AFTER_MS, +} from './reconcile.constants' + +export interface RequeueResult { + requeueAfterMs?: number +} + +export type ReconcileResult = undefined | RequeueResult + +export function requeue(options: RequeueResult = {}): RequeueResult { + return options +} + +export interface ReconcileOptions { + maxRetries?: number + defaultRequeueAfterMs?: number + defaultErrorRequeueAfterMs?: number + shouldRetry?: (error: unknown) => boolean + onError?: (error: unknown) => void +} + +async function reconcile(handler: () => Promise | T, options: ReconcileOptions = {}): Promise { + const { + maxRetries = DEFAULT_RECONCILE_MAX_RETRIES, + defaultRequeueAfterMs = DEFAULT_RECONCILE_REQUEUE_AFTER_MS, + defaultErrorRequeueAfterMs = DEFAULT_RECONCILE_ERROR_REQUEUE_AFTER_MS, + shouldRetry, + onError, + } = options + + const run = async (attempt: number): Promise => { + try { + const result = await handler() + const requeueResult = toRequeueResult(result) + + if (requeueResult) { + if (attempt >= maxRetries) return result + const delayMs = Math.max(0, requeueResult.requeueAfterMs ?? defaultRequeueAfterMs) + await setTimeout(delayMs) + return await run(attempt + 1) + } + + return result + } catch (error) { + onError?.(error) + const canRetry = attempt < maxRetries && (shouldRetry?.(error) ?? true) + if (!canRetry) throw error + + await setTimeout(Math.max(0, defaultErrorRequeueAfterMs)) + return await run(attempt + 1) + } + } + + return await run(0) +} + +export type TypedMethodDecorator = any>( + target: object, + propertyKey: string | symbol, + descriptor: TypedPropertyDescriptor, +) => void + +export function Reconcile(options: ReconcileOptions = {}): TypedMethodDecorator { + return any>( + _target: object, + propertyKey: string | symbol, + descriptor: TypedPropertyDescriptor, + ): void => { + const original = descriptor.value + if (!original) return + + descriptor.value = (async function (this: ThisParameterType, ...args: Parameters): Promise>> { + const logger: Logger = this?.logger instanceof Logger + ? this.logger + : new Logger(this?.constructor?.name ?? 'Reconcile') + + try { + return await reconcile( + () => original.apply(this, args), + options, + ) as Awaited> + } catch (error) { + logger.error( + `Handler ${String(propertyKey)} failed permanently`, + error instanceof Error ? error.stack : undefined, + ) + throw error + } + }) as T + } +} + +function toRequeueResult(value: unknown): RequeueResult | undefined { + if (value && typeof value === 'object' && 'requeueAfterMs' in value) { + const ms = (value as RequeueResult).requeueAfterMs + return ms === undefined || typeof ms === 'number' ? { requeueAfterMs: ms } : undefined + } +} diff --git a/apps/server-nestjs/src/modules/nexus/nexus.service.ts b/apps/server-nestjs/src/modules/nexus/nexus.service.ts index e0cf9312a..3eaf92050 100644 --- a/apps/server-nestjs/src/modules/nexus/nexus.service.ts +++ b/apps/server-nestjs/src/modules/nexus/nexus.service.ts @@ -4,6 +4,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common' import { OnEvent } from '@nestjs/event-emitter' import { Cron, CronExpression } from '@nestjs/schedule' import { trace } from '@opentelemetry/api' + import { ConfigurationService } from '../../cpin-module/infrastructure/configuration/configuration.service' import { StartActiveSpan } from '../../cpin-module/infrastructure/telemetry/telemetry.decorator' import { VaultClientService, VaultError } from '../vault/vault-client.service'