Bug 2026200 - Migrate code-review CI from all tasks defined in .taskcluster.yml to taskgraph

backend/build.sh

@@ -14,4 +14,4 @@ printf '{"commit": "%s", "version": "%s", "source": "%s", "build": "%s"}\n' \
 # Run 'taskboot build' with our local copy of the Git repository where we updated the version.json with correct values.
 # To do so, we use '--target /path/to/existing/clone' instead of passing environment variables (GIT_REPOSITORY, GIT_REVISION)
 # to taskboot that would activate an automated clone.
-taskboot --target /code-review build --image mozilla/code-review --tag "$CHANNEL" --tag "$COMMIT_SHA" --write /backend.tar backend/Dockerfile
+taskboot --target . build --image mozilla/code-review --tag "$CHANNEL" --tag "$COMMIT_SHA" --write /builds/worker/artifacts/backend.tar backend/Dockerfile

pyproject.toml

@@ -12,3 +12,4 @@ ignore = [
 
 [tool.ruff.lint.isort]
 known-first-party = ["code_review_backend", "code_review_bot", "code_review_tools"]
+known-third-party = ["taskcluster"]

taskcluster/code_review_taskgraph/__init__.py

@@ -0,0 +1,10 @@
+from importlib import import_module
+
+
+def register(graph_config):
+    _import_modules(["parameters"])
+
+
+def _import_modules(modules):
+    for module in modules:
+        import_module(f".{module}", package=__name__)

taskcluster/code_review_taskgraph/parameters.py

@@ -0,0 +1,28 @@
+from taskgraph.parameters import extend_parameters_schema
+from voluptuous import Optional
+
+extend_parameters_schema(
+    {
+        Optional("channel"): str,
+        Optional("backend_url"): str,
+    },
+)
+
+
+def decision_parameters(graph_config, parameters):
+    short_head_ref = parameters["head_ref"]
+    for prefix in ("refs/heads/", "refs/tags/"):
+        if short_head_ref.startswith(prefix):
+            short_head_ref = short_head_ref[len(prefix) :]
+            break
+    parameters["head_ref"] = short_head_ref
+
+    if short_head_ref == "testing":
+        parameters["channel"] = "testing"
+        parameters["backend_url"] = "https://api.code-review.testing.moz.tools"
+    elif short_head_ref == "production":
+        parameters["channel"] = "production"
+        parameters["backend_url"] = "https://api.code-review.moz.tools"
+    else:
+        parameters["channel"] = "dev"
+        parameters["backend_url"] = "https://api.code-review.testing.moz.tools"

taskcluster/code_review_taskgraph/transforms/build.py

@@ -0,0 +1,26 @@
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_index_routes(config, tasks):
+    for task in tasks:
+        params = config.params
+        head_rev = params["head_rev"]
+        head_ref = params["head_ref"]
+
+        if params["tasks_for"] == "github-pull-request":
+            index_prefix = "code-review-pr"
+        else:
+            index_prefix = "code-review"
+
+        trust_domain = config.graph_config["trust-domain"]
+        task.setdefault("routes", []).extend(
+            [
+                f"index.{trust_domain}.v2.{index_prefix}.{task['name']}.revision.{head_rev}",
+                f"index.{trust_domain}.v2.{index_prefix}.{task['name']}.branch.{head_ref}",
+            ]
+        )
+
+        yield task

taskcluster/config.yml

@@ -0,0 +1,21 @@
+---
+trust-domain: code-analysis
+task-priority: low
+taskgraph:
+  register: code_review_taskgraph:register
+  decision-parameters: code_review_taskgraph.parameters:decision_parameters
+  repositories:
+    code_review:
+      name: code-review
+workers:
+  aliases:
+    b-linux:
+      provisioner: "{trust-domain}-{level}"
+      implementation: docker-worker
+      os: linux
+      worker-type: linux-docker
+    images:
+      provisioner: "{trust-domain}-{level}"
+      implementation: docker-worker
+      os: linux
+      worker-type: linux-gcp

taskcluster/docker/node/Dockerfile

@@ -0,0 +1,22 @@
+FROM node:16-alpine
+
+# Add worker user
+RUN mkdir -p /builds/worker/artifacts && \
+    delgroup $(grep ":1000:" /etc/group | cut -d: -f1) 2>/dev/null || true && \
+    deluser $(grep ":1000:" /etc/passwd | cut -d: -f1) 2>/dev/null || true && \
+    addgroup -g 1000 worker && \
+    adduser -u 1000 -G worker -h /builds/worker -s /bin/bash -D worker && \
+    chown -R worker:worker /builds/worker
+
+RUN apk add --no-cache git bash lcms2-dev libpng-dev autoconf build-base coreutils shadow python3
+
+# %include-run-task
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+CMD ["/bin/bash"]

taskcluster/docker/python/Dockerfile

@@ -0,0 +1,42 @@
+FROM python:3.12-slim
+
+# Add worker user
+RUN mkdir -p /builds/worker/artifacts && \
+    groupadd --gid 1000 worker && \
+    useradd --uid 1000 --gid worker --home-dir /builds/worker --shell /bin/bash worker && \
+    chown -R worker:worker /builds/worker
+
+# Install build deps (needed for mercurial setup)
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y \
+        git \
+        gcc \
+        python3-dev \
+        libatomic1 && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install mercurial
+RUN pip install --disable-pip-version-check --quiet --no-cache-dir mercurial==7.2
+
+# Clone version-control-tools
+RUN hg clone -r 5b6e8298d035 https://hg.mozilla.org/hgcustom/version-control-tools /src/version-control-tools/ && \
+    rm -rf /src/version-control-tools/.hg \
+           /src/version-control-tools/ansible \
+           /src/version-control-tools/docs \
+           /src/version-control-tools/testing
+
+# %include-run-task
+
+# Cleanup build-only deps
+RUN apt-get purge -y gcc python3-dev && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+CMD ["/bin/bash"]

taskcluster/docker/taskboot/Dockerfile

@@ -0,0 +1,21 @@
+FROM mozilla/taskboot:0.4.5
+
+RUN apk add --no-cache bash coreutils shadow
+
+RUN mkdir -p /builds/worker/artifacts && \
+    delgroup $(grep ":1000:" /etc/group | cut -d: -f1) 2>/dev/null || true && \
+    deluser $(grep ":1000:" /etc/passwd | cut -d: -f1) 2>/dev/null || true && \
+    addgroup -g 1000 worker && \
+    adduser -u 1000 -G worker -h /builds/worker -s /bin/bash -D worker && \
+    chown -R worker:worker /builds/worker
+
+# %include-run-task
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+CMD ["/bin/bash"]

taskcluster/kinds/build/kind.yml

@@ -0,0 +1,68 @@
+loader: taskgraph.loader.transform:loader
+
+transforms:
+  - code_review_taskgraph.transforms.build
+  - taskgraph.transforms.task_context
+  - taskgraph.transforms.run
+  - taskgraph.transforms.task
+
+kind-dependencies:
+  - lint
+  - test
+
+task-defaults:
+  description: Build {name} Docker image
+  run-on-tasks-for:
+    - github-pull-request
+    - github-push
+  worker-type: b-linux
+  worker:
+    docker-image: { in-tree: taskboot }
+    max-run-time: 3600
+    artifacts:
+      - type: file
+        name: public/code-review-{name}.tar.zst
+        path: /builds/worker/artifacts/{name}.tar.zst
+  run:
+    using: run-task
+    cwd: "{checkout}"
+    run-as-root: true
+  dependencies:
+    lint: lint-pre-commit
+  task-context:
+    from-parameters:
+      channel: channel
+      head_rev: head_rev
+      head_ref: head_ref
+      head_repository: head_repository
+    substitution-fields:
+      - description
+      - run.command
+      - worker.artifacts
+
+tasks:
+  bot:
+    run:
+      command: >-
+        taskboot --target /builds/worker/checkouts/vcs
+        build --image mozilla/code-review
+        --tag {channel} --tag {head_rev}
+        --write /builds/worker/artifacts/bot.tar bot/docker/Dockerfile
+    dependencies:
+      test: test-bot
+
+  backend:
+    run:
+      command: backend/build.sh {head_rev} {head_ref} {head_repository} {channel}
+    dependencies:
+      test: test-backend
+
+  integration:
+    run:
+      command: >-
+        taskboot --target /builds/worker/checkouts/vcs
+        build --image mozilla/code-review
+        --tag integration-{channel} --tag integration-{head_rev}
+        --write /builds/worker/artifacts/integration.tar integration/docker/Dockerfile
+    dependencies:
+      test: test-integration

taskcluster/kinds/deploy/kind.yml

@@ -0,0 +1,64 @@
+loader: taskgraph.loader.transform:loader
+
+transforms:
+  - taskgraph.transforms.task_context
+  - taskgraph.transforms.run
+  - taskgraph.transforms.task
+
+kind-dependencies:
+  - build
+  - frontend
+
+task-defaults:
+  run-on-tasks-for:
+    - github-push
+  run-on-git-branches:
+    - testing
+    - production
+  worker-type: b-linux
+  worker:
+    docker-image: { in-tree: taskboot }
+    max-run-time: 3600
+    taskcluster-proxy: true
+    env:
+      TASKCLUSTER_SECRET: "project/relman/code-review/deploy-{channel}"
+  scopes:
+    - "secrets:get:project/relman/code-review/deploy-{channel}"
+  run:
+    using: run-task
+    checkout: false
+  task-context:
+    from-parameters:
+      channel: channel
+    substitution-fields:
+      - run.command
+      - worker.env.TASKCLUSTER_SECRET
+      - scopes
+
+tasks:
+  frontend:
+    description: Deploy frontend SPA to S3
+    run:
+      command: >-
+        taskboot deploy-s3
+        --artifact-folder public/frontend
+        --bucket relman-{channel}-code-review-{channel}-static-website
+    dependencies:
+      frontend: frontend-build
+
+  backend:
+    description: Deploy backend to Heroku
+    run:
+      command: >-
+        taskboot deploy-heroku
+        --heroku-app code-review-backend-{channel}
+        web:public/code-review-backend.tar.zst
+    dependencies:
+      build: build-backend
+
+  integration:
+    description: Push integration Docker image
+    run:
+      command: taskboot push-artifact --artifact-filter public/code-review-integration.tar.zst
+    dependencies:
+      build: build-integration

taskcluster/kinds/docker-image/kind.yml

@@ -0,0 +1,13 @@
+loader: taskgraph.loader.transform:loader
+
+transforms:
+  - taskgraph.transforms.docker_image
+  - taskgraph.transforms.cached_tasks
+  - taskgraph.transforms.task
+
+tasks:
+  python: {}
+
+  taskboot: {}
+
+  node: {}

taskcluster/kinds/frontend/kind.yml

@@ -0,0 +1,35 @@
+loader: taskgraph.loader.transform:loader
+
+transforms:
+  - taskgraph.transforms.task_context
+  - taskgraph.transforms.run
+  - taskgraph.transforms.task
+
+kind-dependencies:
+  - docker-image
+
+tasks:
+  build:
+    description: Build frontend SPA
+    worker-type: b-linux
+    worker:
+      docker-image: { in-tree: node }
+      max-run-time: 3600
+      env:
+        BACKEND_URL: "{backend_url}"
+      artifacts:
+        - type: directory
+          name: public/frontend
+          path: /builds/worker/artifacts/frontend
+    run:
+      using: run-task
+      cwd: "{checkout}/frontend"
+      command: >-
+        npm install &&
+        npm run build &&
+        cp -r build /builds/worker/artifacts/frontend
+    task-context:
+      from-parameters:
+        backend_url: backend_url
+      substitution-fields:
+        - worker.env