From 987e19e020c70ab94e01b9e87866561fda35ce79 Mon Sep 17 00:00:00 2001 From: SuperEwald Date: Wed, 2 Jul 2025 15:58:45 +0200 Subject: [PATCH 1/5] feat: option to start local docker registry --- src/argv.ts | 4 + src/handler.ts | 12 +++ src/index.ts | 5 ++ src/job.ts | 9 +++ src/utils.ts | 78 +++++++++++++++++++ .../test-cases/local-registry/.gitlab-ci.yml | 16 ++++ .../local-registry/integration.test.ts | 53 +++++++++++++ 7 files changed, 177 insertions(+) create mode 100644 tests/test-cases/local-registry/.gitlab-ci.yml create mode 100644 tests/test-cases/local-registry/integration.test.ts diff --git a/src/argv.ts b/src/argv.ts index 6c0ccaeb..e73c17b5 100644 --- a/src/argv.ts +++ b/src/argv.ts @@ -337,4 +337,8 @@ export class Argv { get childPipelineDepth (): number { return this.map.get("childPipelineDepth"); } + + get registry (): boolean { + return this.map.get("registry") ?? false; + } } diff --git a/src/handler.ts b/src/handler.ts index bdbbe530..37a5673d 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -64,6 +64,9 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ Commander.runCsv(parser, writeStreams, argv.listCsvAll); } else if (argv.job.length > 0) { assert(argv.stage === null, "You cannot use --stage when starting individual jobs"); + if (argv.registry) { + await Utils.startDockerRegistry(argv); + } generateGitIgnore(cwd, stateDir); const time = process.hrtime(); if (argv.needs || argv.onlyNeeds) { @@ -77,6 +80,9 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`); } } else if (argv.stage) { + if (argv.registry) { + await Utils.startDockerRegistry(argv); + } generateGitIgnore(cwd, stateDir); const time = process.hrtime(); const pipelineIid = await state.getPipelineIid(cwd, stateDir); @@ -85,6 +91,9 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ await Commander.runJobsInStage(argv, parser, writeStreams); writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`); } else { + if (argv.registry) { + await Utils.startDockerRegistry(argv); + } generateGitIgnore(cwd, stateDir); const time = process.hrtime(); await state.incrementPipelineIid(cwd, stateDir); @@ -96,5 +105,8 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ } writeStreams.flush(); + if (argv.registry) { + await Utils.stopDockerRegistry(argv.containerExecutable); + } return cleanupJobResources(jobs); } diff --git a/src/index.ts b/src/index.ts index 379f9734..9df828ad 100644 --- a/src/index.ts +++ b/src/index.ts @@ -308,6 +308,11 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs)); default: true, description: "Enables color", }) + .option("registry", { + type: "boolean", + requiresArg: false, + description: "Start a local docker registry and configure gitlab-ci-local containers to use that by default", + }) .completion("completion", false, (current: string, yargsArgv: any, completionFilter: any, done: (completions: string[]) => any) => { try { if (current.startsWith("-")) { diff --git a/src/job.ts b/src/job.ts index e428fa4f..7584e3e6 100644 --- a/src/job.ts +++ b/src/job.ts @@ -893,6 +893,15 @@ export class Job { dockerCmd += `--network ${this._serviceNetworkId} --network-alias build `; } + if (this.argv.registry) { + dockerCmd += `--network ${Utils.gclRegistryPrefix}.net `; + dockerCmd += `--volume ${Utils.gclRegistryPrefix}.certs:/etc/containers/certs.d:ro `; + dockerCmd += `--volume ${Utils.gclRegistryPrefix}.certs:/etc/docker/certs.d:ro `; + expanded["CI_REGISTRY"] = Utils.gclRegistryPrefix; + expanded["CI_REGISTRY_USER"] = expanded["CI_REGISTRY_USER"] ?? `${Utils.gclRegistryPrefix}.user`; + expanded["CI_REGISTRY_PASSWORD"] = expanded["CI_REGISTRY_PASSWORD"] ?? `${Utils.gclRegistryPrefix}.password`; + } + dockerCmd += `--volume ${buildVolumeName}:${this.ciProjectDir} `; dockerCmd += `--volume ${tmpVolumeName}:${this.fileVariablesDir} `; dockerCmd += `--workdir ${this.ciProjectDir} `; diff --git a/src/utils.ts b/src/utils.ts index e61524be..fafec012 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -434,4 +434,82 @@ export class Utils { // https://dev.to/babak/exhaustive-type-checking-with-typescript-4l3f throw new Error(`Unhandled case ${param}`); } + + static async dockerVolumeFileExists (containerExecutable: string, path: string, volume: string): Promise { + try { + await Utils.spawn([containerExecutable, "run", "--rm", "-v", `${volume}:/mnt/vol`, "alpine", "ls", `/mnt/vol/${path}`]); + return true; + } catch { + return false; + } + } + + static gclRegistryPrefix: string = "registry.gcl.local"; + static async startDockerRegistry (argv: Argv): Promise { + const gclRegistryCertVol = `${this.gclRegistryPrefix}.certs`; + const gclRegistryDataVol = `${this.gclRegistryPrefix}.data`; + const gclRegistryNet = `${this.gclRegistryPrefix}.net`; + + // create cert volume + try { + await Utils.spawn(`${argv.containerExecutable} volume create ${gclRegistryCertVol}`.split(" ")); + } catch (err) { + if (err instanceof Error && "exitCode" in err && err.exitCode !== 125) + throw err; + } + + // create self-signed cert/key files for https support + if (!await this.dockerVolumeFileExists(argv.containerExecutable, `${this.gclRegistryPrefix}.crt`, gclRegistryCertVol)) { + const opensslArgs = [ + "req", "-newkey", "rsa:4096", "-nodes", "-sha256", + "-keyout", `/certs/${this.gclRegistryPrefix}.key`, + "-x509", "-days", "365", + "-out", `/certs/${this.gclRegistryPrefix}.crt`, + "-subj", `/CN=${this.gclRegistryPrefix}`, + "-addext", `subjectAltName=DNS:${this.gclRegistryPrefix}` + ]; + const generateCertsInPlace = [ + argv.containerExecutable, "run", "--rm", "-v", `${gclRegistryCertVol}:/certs`, "--entrypoint", "sh", "alpine/openssl", "-c", + [ + "openssl", ...opensslArgs, + "&&", "mkdir", "-p", `/certs/${this.gclRegistryPrefix}`, + "&&", "cp", `/certs/${this.gclRegistryPrefix}.crt`, `/certs/${this.gclRegistryPrefix}/ca.crt`, + ].join(" ") + ]; + await Utils.spawn(generateCertsInPlace); + } + + // create data volume + try { + await Utils.spawn([argv.containerExecutable, "volume", "create", gclRegistryDataVol]); + } catch (err) { + // rethrow error if not 'already exists' (exitCode 125) + if (err instanceof Error && "exitCode" in err && err.exitCode !== 125) + throw err; + } + + // create network + try { + await Utils.spawn([argv.containerExecutable, "network", "create", gclRegistryNet]); + } catch (err) { + if (err instanceof Error && "exitCode" in err && err.exitCode !== 125) + throw err; + } + + await Utils.spawn([argv.containerExecutable, "rm", "-f", this.gclRegistryPrefix]); + await Utils.spawn([ + argv.containerExecutable, "run", "-d", "--name", this.gclRegistryPrefix, + "--network", gclRegistryNet, + "--volume", `${gclRegistryDataVol}:/var/lib/registry`, + "--volume", `${gclRegistryCertVol}:/certs:ro`, + "-e", "REGISTRY_HTTP_ADDR=0.0.0.0:443", + "-e", `REGISTRY_HTTP_TLS_CERTIFICATE=/certs/${this.gclRegistryPrefix}.crt`, + "-e", `REGISTRY_HTTP_TLS_KEY=/certs/${this.gclRegistryPrefix}.key`, + "registry" + ]); + } + + static async stopDockerRegistry (containerExecutable: string): Promise { + await Utils.spawn([containerExecutable, "rm", "-f", this.gclRegistryPrefix]); + } } diff --git a/tests/test-cases/local-registry/.gitlab-ci.yml b/tests/test-cases/local-registry/.gitlab-ci.yml new file mode 100644 index 00000000..6736a487 --- /dev/null +++ b/tests/test-cases/local-registry/.gitlab-ci.yml @@ -0,0 +1,16 @@ +registry-variables: + image: alpine:latest + script: + - echo "CI_REGISTRY=$CI_REGISTRY" + - echo "CI_REGISTRY_USER=$CI_REGISTRY_USER" + - echo "CI_REGISTRY_PASSWORD=$CI_REGISTRY_PASSWORD" + +registry-login-docker: + image: docker:dind + script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + +registry-login-oci: + image: quay.io/podman/stable + script: + - echo "$CI_REGISTRY_PASSWORD" | podman login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY diff --git a/tests/test-cases/local-registry/integration.test.ts b/tests/test-cases/local-registry/integration.test.ts new file mode 100644 index 00000000..93a3898b --- /dev/null +++ b/tests/test-cases/local-registry/integration.test.ts @@ -0,0 +1,53 @@ +import {WriteStreamsMock} from "../../../src/write-streams.js"; +import {handler} from "../../../src/handler.js"; +import {Utils} from "../../../src/utils.js"; +import chalk from "chalk"; + +test("local-registry ci variables", async () => { + const writeStreams = new WriteStreamsMock; + await handler({ + cwd: "tests/test-cases/local-registry", + job: ["registry-variables"], + registry: true + }, writeStreams); + + const expected = [ + chalk`{blueBright registry-variables} {greenBright >} CI_REGISTRY=${Utils.gclRegistryPrefix}`, + chalk`{blueBright registry-variables} {greenBright >} CI_REGISTRY_USER=${Utils.gclRegistryPrefix}.user`, + chalk`{blueBright registry-variables} {greenBright >} CI_REGISTRY_PASSWORD=${Utils.gclRegistryPrefix}.password`, + ]; + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); +}); + +test("local-registry login ", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/local-registry", + job: ["registry-login-docker"], + registry: true + }, writeStreams); + + + const expected = [ + chalk`{blueBright registry-login-docker} {greenBright >} Login Succeeded`, + ]; + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); +}); + +test("local-registry login ", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/local-registry", + job: ["registry-login-oci"], + registry: true + }, writeStreams); + + + const expected = [ + chalk`{blueBright registry-login-oci} {greenBright >} Login Succeeded!`, + ]; + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); +}); From 2eaff16cc811b258d072f9b9ddba7200512a7aec Mon Sep 17 00:00:00 2001 From: SuperEwald Date: Wed, 2 Jul 2025 16:03:38 +0200 Subject: [PATCH 2/5] fix linting errors --- src/utils.ts | 6 +++--- tests/test-cases/local-registry/.gitlab-ci.yml | 1 + tests/test-cases/local-registry/integration.test.ts | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index fafec012..7fc9a312 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -466,7 +466,7 @@ export class Utils { "-x509", "-days", "365", "-out", `/certs/${this.gclRegistryPrefix}.crt`, "-subj", `/CN=${this.gclRegistryPrefix}`, - "-addext", `subjectAltName=DNS:${this.gclRegistryPrefix}` + "-addext", `subjectAltName=DNS:${this.gclRegistryPrefix}`, ]; const generateCertsInPlace = [ argv.containerExecutable, "run", "--rm", "-v", `${gclRegistryCertVol}:/certs`, "--entrypoint", "sh", "alpine/openssl", "-c", @@ -474,7 +474,7 @@ export class Utils { "openssl", ...opensslArgs, "&&", "mkdir", "-p", `/certs/${this.gclRegistryPrefix}`, "&&", "cp", `/certs/${this.gclRegistryPrefix}.crt`, `/certs/${this.gclRegistryPrefix}/ca.crt`, - ].join(" ") + ].join(" "), ]; await Utils.spawn(generateCertsInPlace); } @@ -505,7 +505,7 @@ export class Utils { "-e", "REGISTRY_HTTP_ADDR=0.0.0.0:443", "-e", `REGISTRY_HTTP_TLS_CERTIFICATE=/certs/${this.gclRegistryPrefix}.crt`, "-e", `REGISTRY_HTTP_TLS_KEY=/certs/${this.gclRegistryPrefix}.key`, - "registry" + "registry", ]); } diff --git a/tests/test-cases/local-registry/.gitlab-ci.yml b/tests/test-cases/local-registry/.gitlab-ci.yml index 6736a487..158a0b6a 100644 --- a/tests/test-cases/local-registry/.gitlab-ci.yml +++ b/tests/test-cases/local-registry/.gitlab-ci.yml @@ -1,3 +1,4 @@ +--- registry-variables: image: alpine:latest script: diff --git a/tests/test-cases/local-registry/integration.test.ts b/tests/test-cases/local-registry/integration.test.ts index 93a3898b..e65f5c1d 100644 --- a/tests/test-cases/local-registry/integration.test.ts +++ b/tests/test-cases/local-registry/integration.test.ts @@ -8,7 +8,7 @@ test("local-registry ci variables", async () => { await handler({ cwd: "tests/test-cases/local-registry", job: ["registry-variables"], - registry: true + registry: true, }, writeStreams); const expected = [ @@ -25,7 +25,7 @@ test("local-registry login ", async () => { await handler({ cwd: "tests/test-cases/local-registry", job: ["registry-login-docker"], - registry: true + registry: true, }, writeStreams); @@ -41,7 +41,7 @@ test("local-registry login ", async () => { await handler({ cwd: "tests/test-cases/local-registry", job: ["registry-login-oci"], - registry: true + registry: true, }, writeStreams); From 8f84b339a3d6739b3433561ba76b1a308ba876e0 Mon Sep 17 00:00:00 2001 From: SuperEwald Date: Wed, 2 Jul 2025 16:09:44 +0200 Subject: [PATCH 3/5] fix already exist check for container errors --- src/utils.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 7fc9a312..bf459f92 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -454,7 +454,7 @@ export class Utils { try { await Utils.spawn(`${argv.containerExecutable} volume create ${gclRegistryCertVol}`.split(" ")); } catch (err) { - if (err instanceof Error && "exitCode" in err && err.exitCode !== 125) + if (err instanceof Error && !err.message.endsWith("already exists")) throw err; } @@ -483,8 +483,7 @@ export class Utils { try { await Utils.spawn([argv.containerExecutable, "volume", "create", gclRegistryDataVol]); } catch (err) { - // rethrow error if not 'already exists' (exitCode 125) - if (err instanceof Error && "exitCode" in err && err.exitCode !== 125) + if (err instanceof Error && !err.message.endsWith("already exists")) throw err; } @@ -492,7 +491,7 @@ export class Utils { try { await Utils.spawn([argv.containerExecutable, "network", "create", gclRegistryNet]); } catch (err) { - if (err instanceof Error && "exitCode" in err && err.exitCode !== 125) + if (err instanceof Error && !err.message.includes("already exists")) throw err; } From 55765a415166cb0d084a8b1314c2a3b29c816306 Mon Sep 17 00:00:00 2001 From: SuperEwald Date: Wed, 2 Jul 2025 16:32:18 +0200 Subject: [PATCH 4/5] run oci test container in privileged mode --- tests/test-cases/local-registry/integration.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test-cases/local-registry/integration.test.ts b/tests/test-cases/local-registry/integration.test.ts index e65f5c1d..7f850ed4 100644 --- a/tests/test-cases/local-registry/integration.test.ts +++ b/tests/test-cases/local-registry/integration.test.ts @@ -42,6 +42,7 @@ test("local-registry login ", async () => { cwd: "tests/test-cases/local-registry", job: ["registry-login-oci"], registry: true, + privileged: true, }, writeStreams); From d3679ebcea342eae98b1dc041bc8d0c7aa7bcd7d Mon Sep 17 00:00:00 2001 From: SuperEwald Date: Thu, 3 Jul 2025 14:54:37 +0200 Subject: [PATCH 5/5] fix $CI_REGISTRY[..] variable scopes --- src/job.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/job.ts b/src/job.ts index 7584e3e6..8afd04be 100644 --- a/src/job.ts +++ b/src/job.ts @@ -323,7 +323,7 @@ export class Job { predefinedVariables["CI_NODE_INDEX"] = `${opt.nodeIndex}`; } predefinedVariables["CI_NODE_TOTAL"] = `${opt.nodesTotal}`; - predefinedVariables["CI_REGISTRY"] = `local-registry.${this.gitData.remote.host}`; + predefinedVariables["CI_REGISTRY"] = predefinedVariables["CI_REGISTRY"] = this.argv.registry ? Utils.gclRegistryPrefix : `local-registry.${this.gitData.remote.host}`; predefinedVariables["CI_REGISTRY_IMAGE"] = `$CI_REGISTRY/${predefinedVariables["CI_PROJECT_PATH"].toLowerCase()}`; return predefinedVariables; } @@ -829,6 +829,11 @@ export class Job { }); } + if (this.argv.registry) { + expanded["CI_REGISTRY_USER"] = expanded["CI_REGISTRY_USER"] ?? `${Utils.gclRegistryPrefix}.user`; + expanded["CI_REGISTRY_PASSWORD"] = expanded["CI_REGISTRY_PASSWORD"] ?? `${Utils.gclRegistryPrefix}.password`; + } + this.refreshLongRunningSilentTimeout(writeStreams); if (imageName && !this._containerId) { @@ -897,9 +902,6 @@ export class Job { dockerCmd += `--network ${Utils.gclRegistryPrefix}.net `; dockerCmd += `--volume ${Utils.gclRegistryPrefix}.certs:/etc/containers/certs.d:ro `; dockerCmd += `--volume ${Utils.gclRegistryPrefix}.certs:/etc/docker/certs.d:ro `; - expanded["CI_REGISTRY"] = Utils.gclRegistryPrefix; - expanded["CI_REGISTRY_USER"] = expanded["CI_REGISTRY_USER"] ?? `${Utils.gclRegistryPrefix}.user`; - expanded["CI_REGISTRY_PASSWORD"] = expanded["CI_REGISTRY_PASSWORD"] ?? `${Utils.gclRegistryPrefix}.password`; } dockerCmd += `--volume ${buildVolumeName}:${this.ciProjectDir} `;