diff --git a/.taskcluster.yml b/.taskcluster.yml index 9d81ef86b..19da54fd5 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,462 +1,189 @@ +# yamllint disable rule:line-length +# This file is rendered via JSON-e by +# - github events - https://github.com/taskcluster/taskcluster/tree/main/services/github +--- version: 1 +reporting: checks-v1 +autoCancelPreviousChecks: true policy: pullRequests: public -reporting: checks-v1 tasks: - $let: - head_branch: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.ref} - else: - $if: 'tasks_for == "github-push"' + - $let: + trustDomain: code-analysis + ownerEmail: + $switch: + 'tasks_for == "github-push"': "${event.pusher.email}" + 'tasks_for == "github-release"': "${event.sender.login}@users.noreply.github.com" + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.user.login}@users.noreply.github.com" + baseRepoUrl: + $switch: + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.base.repo.html_url}" + $default: "${event.repository.html_url}" + repoUrl: + $switch: + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.head.repo.html_url}" + $default: "${event.repository.html_url}" + project: + $switch: + 'tasks_for in ["github-push", "github-release"]': "${event.repository.name}" + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.base.repo.name}" + head_ref: + $switch: + 'tasks_for[:19] == "github-pull-request"': ${event.pull_request.head.ref} + 'tasks_for == "github-push"': ${event.ref} + 'tasks_for == "github-release"': ${event.release.tag_name} + base_ref: + $switch: + 'tasks_for[:19] == "github-pull-request"': ${event.pull_request.base.ref} + 'tasks_for == "github-push" && event.base_ref': ${event.base_ref} + 'tasks_for == "github-push" && !(event.base_ref)': ${event.ref} + 'tasks_for == "github-release"': "" + base_sha: + $switch: + 'tasks_for == "github-push"': "${event.before}" + 'tasks_for == "github-release"': "${event.release.target_commitish}" + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.base.sha}" + head_sha: + $switch: + 'tasks_for == "github-push"': "${event.after}" + 'tasks_for == "github-release"': "${event.release.tag_name}" + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.head.sha}" + ownTaskId: + $eval: as_slugid("decision_task") + pullRequestAction: + $switch: + 'tasks_for[:19] == "github-pull-request"': ${event.action} + $default: "UNDEFINED" + releaseAction: + $if: 'tasks_for == "github-release"' + then: ${event.action} + else: "UNDEFINED" + isPullRequest: + $eval: 'tasks_for[:19] == "github-pull-request"' + in: + $let: + short_head_ref: + $switch: + 'head_ref[:10] == "refs/tags/"': "${head_ref[10:]}" + 'head_ref[:11] == "refs/heads/"': "${head_ref[11:]}" + $default: "${head_ref}" + level: + $if: 'tasks_for in ["github-release", "github-push"] && repoUrl == "https://github.com/mozilla/code-review"' + then: 3 + else: 1 + in: + $if: > + (tasks_for == "github-push" && short_head_ref in ["master", "production", "testing"]) + || (tasks_for == "github-release" && releaseAction == "published") + || (isPullRequest && pullRequestAction in ["opened", "reopened", "synchronize"]) then: - # Strip ref branch prefix - $if: 'event.ref[0:11] == "refs/heads/"' - then: ${event.ref[11:]} - else: ${event.ref} - else: ${event.release.target_commitish} - - head_rev: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.sha} - else: - $if: 'tasks_for == "github-push"' - then: ${event.after} - else: ${event.release.tag_name} - - repository: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.repo.html_url} - else: ${event.repository.html_url} - - channel: - $if: 'tasks_for == "github-push"' - then: - $if: 'event.ref in ["refs/heads/testing", "refs/heads/production"]' - then: ${event.ref[11:]} - else: "dev" - else: "dev" - - backend_url: - $if: 'tasks_for == "github-push"' - then: - $if: 'event.ref == "refs/heads/testing"' - then: "https://api.code-review.testing.moz.tools" - else: "https://api.code-review.moz.tools" - else: - $if: 'tasks_for == "github-pull-request"' - then: "https://api.code-review.testing.moz.tools" - else: "https://api.code-review.moz.tools" - - taskboot_image: "mozilla/taskboot:0.4.1" - - pip_install: "pip install --disable-pip-version-check --no-cache-dir --quiet" - python_version: "3.12" - - provisionerId: - $if: 'taskcluster_root_url == "https://firefox-ci-tc.services.mozilla.com"' - then: - $if: 'tasks_for == "github-push"' - then: "code-analysis-3" - else: "code-analysis-1" - else: proj-relman - - workerType: - $if: 'taskcluster_root_url == "https://firefox-ci-tc.services.mozilla.com"' - then: linux-gw-gcp - else: generic-worker-ubuntu-24-04 - in: - $if: '(tasks_for == "github-push" && (head_branch == "master" || head_branch == "production" || head_branch == "testing")) || (tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"])' - then: - $flatten: - $match: - # Always run those tasks - "true": - - taskId: { $eval: as_slugid("check_lint") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: "python:${python_version}" - command: - - sh - - -lxce - - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - cd /src/bot && ${pip_install} -r requirements-dev.txt && - cd /src && pre-commit run -a --show-diff-on-failure" - metadata: - name: "Code Review Bot checks: linting" - description: Check code style with pre-commit hooks - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - # Run only on firefoxci Taskcluster - 'taskcluster_root_url == "https://firefox-ci-tc.services.mozilla.com"': - - taskId: { $eval: as_slugid("bot_check_tests") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: "python:${python_version}-slim" - command: - - sh - - -lxce - - "apt-get update -q && apt-get install -q -y --no-install-recommends git && - git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - /src/tools/docker/bootstrap-mercurial.sh && - cd /src/bot && ${pip_install} . && ${pip_install} -r requirements-dev.txt && - pytest -v" - metadata: - name: "Code Review Bot checks: unit tests" - description: Check python code with pytest - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("bot_build_dind") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("check_lint") } - - { $eval: as_slugid("bot_check_tests") } - payload: - maxRunTime: 3600 - image: "${taskboot_image}" - env: - GIT_REPOSITORY: ${repository} - GIT_REVISION: ${head_rev} - command: - - taskboot - - build - - --image - - mozilla/code-review - - --tag - - "${channel}" - - --tag - - "${head_rev}" - - --write - - /bot.tar - - bot/docker/Dockerfile - artifacts: - public/code-review-bot.tar.zst: - expires: { $fromNow: "6 months" } - path: /bot.tar.zst - type: file - routes: - $if: 'tasks_for == "github-pull-request"' + taskId: "${ownTaskId}" + taskGroupId: "${ownTaskId}" + schedulerId: "${trustDomain}-level-${level}" + + created: { $fromNow: "" } + deadline: { $fromNow: "1 day" } + expires: { $fromNow: "1 year 1 second" } + + metadata: + owner: "${ownerEmail}" + source: "${repoUrl}/raw/${head_sha}/.taskcluster.yml" + name: "Decision Task" + description: "The task that creates all of the other tasks in the task graph" + + provisionerId: "${trustDomain}-${level}" + workerType: "decision" + + tags: + createdForUser: "${ownerEmail}" + kind: decision-task + + routes: + $flatten: + - checks + - $if: 'tasks_for == "github-push"' then: - - "index.code-analysis.v2.code-review-pr.revision.${head_rev}" - - "index.code-analysis.v2.code-review-pr.branch.${head_branch}" - else: - - "index.code-analysis.v2.code-review.revision.${head_rev}" - - "index.code-analysis.v2.code-review.branch.${head_branch}" - metadata: - name: Code Review Bot docker in docker build - description: Build docker image of code review bot, using a remote docker daemon - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - # Run only on community Taskcluster - 'taskcluster_root_url == "https://community-tc.services.mozilla.com"': - - taskId: { $eval: as_slugid("backend_check_tests") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: "python:${python_version}" - command: - - sh - - -lxce - - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - cd /src/backend && ${pip_install} . && ${pip_install} -r requirements-dev.txt && - ./ci/setup_postgres.sh && - ./manage.py test && ./manage.py makemigrations --check --noinput --dry-run -v 3" - env: - DATABASE_URL: postgres://tester@127.0.0.1/code-review - metadata: - name: "Code Review Backend checks: unit tests" - description: Check python code with Django tests - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("frontend_build") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: node:16-alpine - env: - BACKEND_URL: "${backend_url}" - command: - - sh - - -lxce - - "apk add git lcms2-dev bash libpng-dev autoconf build-base --quiet && - git clone --quiet ${repository} /src && cd /src/frontend && git checkout ${head_rev} -b build && - npm install && npm run build" - artifacts: - public/frontend: - expires: { $fromNow: "2 weeks" } - path: /src/frontend/build - type: directory - metadata: - name: Code Review Frontend build - description: Build web single page application - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("backend_build") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("check_lint") } - - { $eval: as_slugid("backend_check_tests") } - payload: - capabilities: - privileged: true - maxRunTime: 3600 - image: "${taskboot_image}" - command: - - /bin/sh - - -lxce - - "git clone --quiet ${repository} /code-review && cd /code-review && git checkout ${head_rev} -b build - && backend/build.sh ${head_rev} ${head_branch} ${repository} ${channel}" - artifacts: - public/code-review-backend.tar.zst: - expires: { $fromNow: "2 weeks" } - path: /backend.tar.zst - type: file - scopes: - - docker-worker:capability:privileged - metadata: - name: Code Review Backend docker build - description: Build docker image of code review backend - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("integration_check_tests") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: "python:${python_version}" - command: - - sh - - -lxce - - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - cd /src/integration && ${pip_install} -r requirements.txt -r requirements-dev.txt && - pytest -v" - metadata: - name: "Code Review Integration checks: unit tests" - description: Check python code with pytest - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("integration_build") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("check_lint") } - - { $eval: as_slugid("integration_check_tests") } - payload: - capabilities: - privileged: true - maxRunTime: 3600 - image: "${taskboot_image}" - env: - GIT_REPOSITORY: ${repository} - GIT_REVISION: ${head_rev} - command: - - taskboot - - build - - --image - - mozilla/code-review - - --tag - - "integration-${channel}" - - --tag - - "integration-${head_rev}" - - --write - - /integration.tar - - integration/docker/Dockerfile - artifacts: - public/code-review-integration.tar.zst: - expires: { $fromNow: "2 weeks" } - path: /integration.tar.zst - type: file - scopes: - - docker-worker:capability:privileged - metadata: - name: Code Review Integration docker build - description: Build docker image of code review integration tests - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'channel in ["testing", "production"]' - then: - taskId: { $eval: as_slugid("frontend_deploy") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("frontend_build") } - payload: - features: - # Needed for access to secret - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - env: - TASKCLUSTER_SECRET: "project/relman/code-review/deploy-${channel}" - command: - - taskboot - - deploy-s3 - - --artifact-folder - - public/frontend - - --bucket - - "relman-${channel}-code-review-${channel}-static-website" - scopes: - - "secrets:get:project/relman/code-review/deploy-${channel}" - metadata: - name: "Code Review Frontend deploy (${channel})" - description: Deploy frontend build on environment - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'channel in ["testing", "production"]' - then: - taskId: { $eval: as_slugid("backend_deploy") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("backend_build") } - payload: - features: - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - command: - - taskboot - - deploy-heroku - - --heroku-app - - "code-review-backend-${channel}" - - web:public/code-review-backend.tar.zst - env: - TASKCLUSTER_SECRET: "project/relman/code-review/deploy-${channel}" - scopes: - - "secrets:get:project/relman/code-review/deploy-${channel}" - metadata: - name: "Code Review Backend deployment (${channel})" - description: Deploy docker image on Heroku - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'channel in ["testing", "production"]' - then: - taskId: { $eval: as_slugid("integration_deploy") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("integration_build") } - payload: - features: - # Needed for access to secret - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - env: - TASKCLUSTER_SECRET: "project/relman/code-review/deploy-${channel}" - command: - - taskboot - - push-artifact - scopes: - - "secrets:get:project/relman/code-review/deploy-${channel}" - metadata: - name: "Code Review Integration test push (${channel})" - description: Push integration's docker image on repository - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'channel in ["testing", "production"]' - then: - taskId: { $eval: as_slugid("integration_hook") } - dependencies: - - { $eval: as_slugid("integration_deploy") } - scopes: - - "assume:hook-id:project-relman/code-review-integration-${channel}" - - "hooks:modify-hook:project-relman/code-review-integration-${channel}" - created: { $fromNow: "" } - deadline: { $fromNow: "5 hours" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - payload: - features: - # Needed for access to hook api - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - command: - - "/bin/sh" - - "-lcxe" - - "git clone --quiet ${repository} && - cd code-review && - git checkout ${head_rev} && - sed -i -e 's/CHANNEL/${channel}/g' -e 's/REVISION/${head_rev}/g' integration/taskcluster-hook.json && - taskboot --target . build-hook integration/taskcluster-hook.json project-relman code-review-integration-${channel}" - metadata: - name: "Code Review Bot integration test hook update (${channel})" - description: Update Taskcluster hook triggering the code-review integration tests - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"' - then: - $let: - version: { $eval: "head_branch[10:]" } - in: - taskId: { $eval: as_slugid("release") } - dependencies: - - { $eval: as_slugid("backend_build") } - - { $eval: as_slugid("frontend_build") } - - { $eval: as_slugid("integration_build") } - created: { $fromNow: "" } - deadline: { $fromNow: "2 hours" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - scopes: - - secrets:get:project/relman/code-review/release - payload: - features: - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - env: - TASKCLUSTER_SECRET: project/relman/code-review/release - command: - - taskboot - - github-release - - mozilla/code-review - - "${version}" - metadata: - name: "Code Review Bot release ${version}" - description: Publish a new GitHub release for code-review platform - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review + - "index.${trustDomain}.v2.${project}.latest.taskgraph.decision" + - "index.${trustDomain}.v2.${project}.revision.${head_sha}.taskgraph.decision" + else: [] + + scopes: + $switch: + 'tasks_for == "github-push"': + - "assume:repo:${repoUrl[8:]}:branch:${short_head_ref}" + 'tasks_for == "github-release"': + - "assume:repo:${repoUrl[8:]}:release:${releaseAction}" + "isPullRequest": + - "assume:repo:github.com/${event.pull_request.base.repo.full_name}:${tasks_for[7:]}" + + dependencies: [] + requires: all-completed + + priority: + $if: 'tasks_for == "github-push" || isPullRequest' + then: very-low + else: lowest + + retries: 5 + + payload: + env: + $merge: + - CODE_REVIEW_BASE_REPOSITORY: "${baseRepoUrl}" + CODE_REVIEW_BASE_REF: "${base_ref}" + CODE_REVIEW_BASE_REV: "${base_sha}" + CODE_REVIEW_HEAD_REPOSITORY: "${repoUrl}" + CODE_REVIEW_HEAD_REF: "${head_ref}" + CODE_REVIEW_HEAD_REV: "${head_sha}" + CODE_REVIEW_REPOSITORY_TYPE: git + REPOSITORIES: { $json: { code_review: code-review } } + - $if: "isPullRequest" + then: + CODE_REVIEW_PULL_REQUEST_NUMBER: "${event.pull_request.number}" + + cache: + "${trustDomain}-level-${level}-checkouts-v2": /builds/worker/checkouts + + features: + taskclusterProxy: true + + image: mozillareleases/taskgraph:decision-v20.0.0@sha256:afae5e5d3caaeb10a93fae56bc3176d1e73faf9b616c01facf811e488db4eafe + + maxRunTime: 1800 + + command: + - run-task + - "--code_review-checkout=/builds/worker/checkouts/src" + - "--" + - bash + - -cx + - > + cd /builds/worker/checkouts/src && + ln -s /builds/worker/artifacts artifacts && + taskgraph decision + --pushlog-id='0' + --pushdate='0' + --project='${project}' + --owner='${ownerEmail}' + --level='${level}' + --repository-type=git + --tasks-for='${tasks_for}' + --base-repository='${baseRepoUrl}' + --base-ref='${base_ref}' + --base-rev='${base_sha}' + --head-repository='${repoUrl}' + --head-ref='${head_ref}' + --head-rev='${head_sha}' + + artifacts: + "public": + type: "directory" + path: "/builds/worker/artifacts" + expires: { $fromNow: "1 year" } + "public/docker-contexts": + type: "directory" + path: "/builds/worker/checkouts/src/docker-contexts" + expires: { $fromNow: "7 day" } + + extra: + tasks_for: "${tasks_for}" diff --git a/backend/build.sh b/backend/build.sh index 496495a95..159ea6a05 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -14,4 +14,4 @@ printf '{"commit": "%s", "version": "%s", "source": "%s", "build": "%s"}\n' \ # Run 'taskboot build' with our local copy of the Git repository where we updated the version.json with correct values. # To do so, we use '--target /path/to/existing/clone' instead of passing environment variables (GIT_REPOSITORY, GIT_REVISION) # to taskboot that would activate an automated clone. -taskboot --target /code-review build --image mozilla/code-review --tag "$CHANNEL" --tag "$COMMIT_SHA" --write /backend.tar backend/Dockerfile +taskboot --target . build --image mozilla/code-review --tag "$CHANNEL" --tag "$COMMIT_SHA" --write /builds/worker/artifacts/backend.tar backend/Dockerfile diff --git a/pyproject.toml b/pyproject.toml index 93fa1fef6..3fa99ac4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,3 +12,4 @@ ignore = [ [tool.ruff.lint.isort] known-first-party = ["code_review_backend", "code_review_bot", "code_review_tools"] +known-third-party = ["taskcluster"] diff --git a/taskcluster/code_review_taskgraph/__init__.py b/taskcluster/code_review_taskgraph/__init__.py new file mode 100644 index 000000000..ca58addde --- /dev/null +++ b/taskcluster/code_review_taskgraph/__init__.py @@ -0,0 +1,10 @@ +from importlib import import_module + + +def register(graph_config): + _import_modules(["parameters"]) + + +def _import_modules(modules): + for module in modules: + import_module(f".{module}", package=__name__) diff --git a/taskcluster/code_review_taskgraph/parameters.py b/taskcluster/code_review_taskgraph/parameters.py new file mode 100644 index 000000000..477dad182 --- /dev/null +++ b/taskcluster/code_review_taskgraph/parameters.py @@ -0,0 +1,28 @@ +from taskgraph.parameters import extend_parameters_schema +from voluptuous import Optional + +extend_parameters_schema( + { + Optional("channel"): str, + Optional("backend_url"): str, + }, +) + + +def decision_parameters(graph_config, parameters): + short_head_ref = parameters["head_ref"] + for prefix in ("refs/heads/", "refs/tags/"): + if short_head_ref.startswith(prefix): + short_head_ref = short_head_ref[len(prefix) :] + break + parameters["head_ref"] = short_head_ref + + if short_head_ref == "testing": + parameters["channel"] = "testing" + parameters["backend_url"] = "https://api.code-review.testing.moz.tools" + elif short_head_ref == "production": + parameters["channel"] = "production" + parameters["backend_url"] = "https://api.code-review.moz.tools" + else: + parameters["channel"] = "dev" + parameters["backend_url"] = "https://api.code-review.testing.moz.tools" diff --git a/taskcluster/code_review_taskgraph/transforms/__init__.py b/taskcluster/code_review_taskgraph/transforms/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/taskcluster/code_review_taskgraph/transforms/build.py b/taskcluster/code_review_taskgraph/transforms/build.py new file mode 100644 index 000000000..d9bcbcea3 --- /dev/null +++ b/taskcluster/code_review_taskgraph/transforms/build.py @@ -0,0 +1,26 @@ +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def add_index_routes(config, tasks): + for task in tasks: + params = config.params + head_rev = params["head_rev"] + head_ref = params["head_ref"] + + if params["tasks_for"] == "github-pull-request": + index_prefix = "code-review-pr" + else: + index_prefix = "code-review" + + trust_domain = config.graph_config["trust-domain"] + task.setdefault("routes", []).extend( + [ + f"index.{trust_domain}.v2.{index_prefix}.{task['name']}.revision.{head_rev}", + f"index.{trust_domain}.v2.{index_prefix}.{task['name']}.branch.{head_ref}", + ] + ) + + yield task diff --git a/taskcluster/config.yml b/taskcluster/config.yml new file mode 100644 index 000000000..9fa5c26e3 --- /dev/null +++ b/taskcluster/config.yml @@ -0,0 +1,21 @@ +--- +trust-domain: code-analysis +task-priority: low +taskgraph: + register: code_review_taskgraph:register + decision-parameters: code_review_taskgraph.parameters:decision_parameters + repositories: + code_review: + name: code-review +workers: + aliases: + b-linux: + provisioner: "{trust-domain}-{level}" + implementation: docker-worker + os: linux + worker-type: linux-docker + images: + provisioner: "{trust-domain}-{level}" + implementation: docker-worker + os: linux + worker-type: linux-gcp diff --git a/taskcluster/docker/node/Dockerfile b/taskcluster/docker/node/Dockerfile new file mode 100644 index 000000000..7aef7a18d --- /dev/null +++ b/taskcluster/docker/node/Dockerfile @@ -0,0 +1,22 @@ +FROM node:16-alpine + +# Add worker user +RUN mkdir -p /builds/worker/artifacts && \ + delgroup $(grep ":1000:" /etc/group | cut -d: -f1) 2>/dev/null || true && \ + deluser $(grep ":1000:" /etc/passwd | cut -d: -f1) 2>/dev/null || true && \ + addgroup -g 1000 worker && \ + adduser -u 1000 -G worker -h /builds/worker -s /bin/bash -D worker && \ + chown -R worker:worker /builds/worker + +RUN apk add --no-cache git bash lcms2-dev libpng-dev autoconf build-base coreutils shadow python3 + +# %include-run-task + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +CMD ["/bin/bash"] diff --git a/taskcluster/docker/python/Dockerfile b/taskcluster/docker/python/Dockerfile new file mode 100644 index 000000000..5d721c592 --- /dev/null +++ b/taskcluster/docker/python/Dockerfile @@ -0,0 +1,42 @@ +FROM python:3.12-slim + +# Add worker user +RUN mkdir -p /builds/worker/artifacts && \ + groupadd --gid 1000 worker && \ + useradd --uid 1000 --gid worker --home-dir /builds/worker --shell /bin/bash worker && \ + chown -R worker:worker /builds/worker + +# Install build deps (needed for mercurial setup) +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + git \ + gcc \ + python3-dev \ + libatomic1 && \ + rm -rf /var/lib/apt/lists/* + +# Install mercurial +RUN pip install --disable-pip-version-check --quiet --no-cache-dir mercurial==7.2 + +# Clone version-control-tools +RUN hg clone -r 5b6e8298d035 https://hg.mozilla.org/hgcustom/version-control-tools /src/version-control-tools/ && \ + rm -rf /src/version-control-tools/.hg \ + /src/version-control-tools/ansible \ + /src/version-control-tools/docs \ + /src/version-control-tools/testing + +# %include-run-task + +# Cleanup build-only deps +RUN apt-get purge -y gcc python3-dev && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +CMD ["/bin/bash"] diff --git a/taskcluster/docker/taskboot/Dockerfile b/taskcluster/docker/taskboot/Dockerfile new file mode 100644 index 000000000..d640fa2f5 --- /dev/null +++ b/taskcluster/docker/taskboot/Dockerfile @@ -0,0 +1,21 @@ +FROM mozilla/taskboot:0.4.5 + +RUN apk add --no-cache bash coreutils shadow + +RUN mkdir -p /builds/worker/artifacts && \ + delgroup $(grep ":1000:" /etc/group | cut -d: -f1) 2>/dev/null || true && \ + deluser $(grep ":1000:" /etc/passwd | cut -d: -f1) 2>/dev/null || true && \ + addgroup -g 1000 worker && \ + adduser -u 1000 -G worker -h /builds/worker -s /bin/bash -D worker && \ + chown -R worker:worker /builds/worker + +# %include-run-task + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +CMD ["/bin/bash"] diff --git a/taskcluster/kinds/build/kind.yml b/taskcluster/kinds/build/kind.yml new file mode 100644 index 000000000..a56325b79 --- /dev/null +++ b/taskcluster/kinds/build/kind.yml @@ -0,0 +1,68 @@ +loader: taskgraph.loader.transform:loader + +transforms: + - code_review_taskgraph.transforms.build + - taskgraph.transforms.task_context + - taskgraph.transforms.run + - taskgraph.transforms.task + +kind-dependencies: + - lint + - test + +task-defaults: + description: Build {name} Docker image + run-on-tasks-for: + - github-pull-request + - github-push + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + artifacts: + - type: file + name: public/code-review-{name}.tar.zst + path: /builds/worker/artifacts/{name}.tar.zst + run: + using: run-task + cwd: "{checkout}" + run-as-root: true + dependencies: + lint: lint-pre-commit + task-context: + from-parameters: + channel: channel + head_rev: head_rev + head_ref: head_ref + head_repository: head_repository + substitution-fields: + - description + - run.command + - worker.artifacts + +tasks: + bot: + run: + command: >- + taskboot --target /builds/worker/checkouts/vcs + build --image mozilla/code-review + --tag {channel} --tag {head_rev} + --write /builds/worker/artifacts/bot.tar bot/docker/Dockerfile + dependencies: + test: test-bot + + backend: + run: + command: backend/build.sh {head_rev} {head_ref} {head_repository} {channel} + dependencies: + test: test-backend + + integration: + run: + command: >- + taskboot --target /builds/worker/checkouts/vcs + build --image mozilla/code-review + --tag integration-{channel} --tag integration-{head_rev} + --write /builds/worker/artifacts/integration.tar integration/docker/Dockerfile + dependencies: + test: test-integration diff --git a/taskcluster/kinds/deploy/kind.yml b/taskcluster/kinds/deploy/kind.yml new file mode 100644 index 000000000..e89a9cb2e --- /dev/null +++ b/taskcluster/kinds/deploy/kind.yml @@ -0,0 +1,64 @@ +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.task_context + - taskgraph.transforms.run + - taskgraph.transforms.task + +kind-dependencies: + - build + - frontend + +task-defaults: + run-on-tasks-for: + - github-push + run-on-git-branches: + - testing + - production + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + taskcluster-proxy: true + env: + TASKCLUSTER_SECRET: "project/relman/code-review/deploy-{channel}" + scopes: + - "secrets:get:project/relman/code-review/deploy-{channel}" + run: + using: run-task + checkout: false + task-context: + from-parameters: + channel: channel + substitution-fields: + - run.command + - worker.env.TASKCLUSTER_SECRET + - scopes + +tasks: + frontend: + description: Deploy frontend SPA to S3 + run: + command: >- + taskboot deploy-s3 + --artifact-folder public/frontend + --bucket relman-{channel}-code-review-{channel}-static-website + dependencies: + frontend: frontend-build + + backend: + description: Deploy backend to Heroku + run: + command: >- + taskboot deploy-heroku + --heroku-app code-review-backend-{channel} + web:public/code-review-backend.tar.zst + dependencies: + build: build-backend + + integration: + description: Push integration Docker image + run: + command: taskboot push-artifact --artifact-filter public/code-review-integration.tar.zst + dependencies: + build: build-integration diff --git a/taskcluster/kinds/docker-image/kind.yml b/taskcluster/kinds/docker-image/kind.yml new file mode 100644 index 000000000..276338a19 --- /dev/null +++ b/taskcluster/kinds/docker-image/kind.yml @@ -0,0 +1,13 @@ +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.docker_image + - taskgraph.transforms.cached_tasks + - taskgraph.transforms.task + +tasks: + python: {} + + taskboot: {} + + node: {} diff --git a/taskcluster/kinds/frontend/kind.yml b/taskcluster/kinds/frontend/kind.yml new file mode 100644 index 000000000..8e8d5d62e --- /dev/null +++ b/taskcluster/kinds/frontend/kind.yml @@ -0,0 +1,35 @@ +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.task_context + - taskgraph.transforms.run + - taskgraph.transforms.task + +kind-dependencies: + - docker-image + +tasks: + build: + description: Build frontend SPA + worker-type: b-linux + worker: + docker-image: { in-tree: node } + max-run-time: 3600 + env: + BACKEND_URL: "{backend_url}" + artifacts: + - type: directory + name: public/frontend + path: /builds/worker/artifacts/frontend + run: + using: run-task + cwd: "{checkout}/frontend" + command: >- + npm install && + npm run build && + cp -r build /builds/worker/artifacts/frontend + task-context: + from-parameters: + backend_url: backend_url + substitution-fields: + - worker.env diff --git a/taskcluster/kinds/hook/kind.yml b/taskcluster/kinds/hook/kind.yml new file mode 100644 index 000000000..ba77dd6a7 --- /dev/null +++ b/taskcluster/kinds/hook/kind.yml @@ -0,0 +1,48 @@ +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.task_context + - taskgraph.transforms.run + - taskgraph.transforms.task + +kind-dependencies: + - deploy + +tasks: + integration: + description: Update Taskcluster hook for integration tests + run-on-tasks-for: + - github-push + run-on-git-branches: + - testing + - production + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + taskcluster-proxy: true + env: + TASKCLUSTER_SECRET: "project/relman/code-review/deploy-{channel}" + scopes: + - "secrets:get:project/relman/code-review/deploy-{channel}" + - "assume:hook-id:project-relman/code-review-integration-{channel}" + - "hooks:modify-hook:project-relman/code-review-integration-{channel}" + run: + using: run-task + cwd: "{checkout}" + command: >- + sed -i -e 's/CHANNEL/{channel}/g' -e 's/REVISION/{head_rev}/g' + /builds/worker/checkouts/vcs/integration/taskcluster-hook.json && + taskboot --target /builds/worker/checkouts/vcs + build-hook integration/taskcluster-hook.json + project-relman code-review-integration-{channel} + task-context: + from-parameters: + channel: channel + head_rev: head_rev + substitution-fields: + - run.command + - worker.env.TASKCLUSTER_SECRET + - scopes + dependencies: + deploy: deploy-integration diff --git a/taskcluster/kinds/lint/kind.yml b/taskcluster/kinds/lint/kind.yml new file mode 100644 index 000000000..25ba17997 --- /dev/null +++ b/taskcluster/kinds/lint/kind.yml @@ -0,0 +1,27 @@ +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.run + - taskgraph.transforms.task + +kind-dependencies: + - docker-image + +task-defaults: + worker-type: b-linux + worker: + docker-image: { in-tree: python } + max-run-time: 3600 + run: + using: run-task + cwd: "{checkout}" + +tasks: + pre-commit: + description: Check code style with pre-commit hooks + run: + command: >- + cd bot && + pip install --disable-pip-version-check --no-cache-dir --quiet -r requirements-dev.txt && + cd .. && + pre-commit run -a --show-diff-on-failure diff --git a/taskcluster/kinds/release/kind.yml b/taskcluster/kinds/release/kind.yml new file mode 100644 index 000000000..ce30fa488 --- /dev/null +++ b/taskcluster/kinds/release/kind.yml @@ -0,0 +1,41 @@ +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.task_context + - taskgraph.transforms.run + - taskgraph.transforms.task + +kind-dependencies: + - build + - frontend + +tasks: + github: + description: Create GitHub release + run-on-tasks-for: + - github-release + run-on-git-branches: + - "v*" + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + taskcluster-proxy: true + env: + TASKCLUSTER_SECRET: "project/relman/code-review/release" + scopes: + - "secrets:get:project/relman/code-review/release" + run: + using: run-task + checkout: false + command: taskboot github-release mozilla/code-review {head_ref} + task-context: + from-parameters: + head_ref: head_ref + substitution-fields: + - run.command + dependencies: + backend: build-backend + bot: build-bot + integration: build-integration + frontend: frontend-build diff --git a/taskcluster/kinds/test/kind.yml b/taskcluster/kinds/test/kind.yml new file mode 100644 index 000000000..89a1e850d --- /dev/null +++ b/taskcluster/kinds/test/kind.yml @@ -0,0 +1,51 @@ +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.run + - taskgraph.transforms.task + +kind-dependencies: + - docker-image + +task-defaults: + worker-type: b-linux + worker: + docker-image: { in-tree: python } + max-run-time: 3600 + run: + using: run-task + cwd: "{checkout}" + +tasks: + bot: + description: Check bot code with pytest + run: + command: >- + ln -sf /builds/worker/checkouts/vcs/tools/docker/hgrc ~/.hgrc && + cd bot && + pip install --disable-pip-version-check --no-cache-dir --quiet . && + pip install --disable-pip-version-check --no-cache-dir --quiet -r requirements-dev.txt && + pytest -v + + backend: + description: Check backend with Django tests + worker: + env: + DATABASE_URL: postgres://tester@127.0.0.1/code-review + run: + run-as-root: true + command: >- + cd backend && + pip install --disable-pip-version-check --no-cache-dir --quiet . && + pip install --disable-pip-version-check --no-cache-dir --quiet -r requirements-dev.txt && + ./ci/setup_postgres.sh && + ./manage.py test && + ./manage.py makemigrations --check --noinput --dry-run -v 3 + + integration: + description: Check integration code with pytest + run: + command: >- + cd integration && + pip install --disable-pip-version-check --no-cache-dir --quiet -r requirements.txt -r requirements-dev.txt && + pytest -v diff --git a/taskcluster/test/params/github-pull-request.yml b/taskcluster/test/params/github-pull-request.yml new file mode 100644 index 000000000..13131f243 --- /dev/null +++ b/taskcluster/test/params/github-pull-request.yml @@ -0,0 +1,29 @@ +base_ref: master +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: dev +backend_url: "https://api.code-review.testing.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: some-feature-branch +head_repository: https://github.com/user/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "1" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-pull-request +version: null +next_version: null diff --git a/taskcluster/test/params/github-push-master.yml b/taskcluster/test/params/github-push-master.yml new file mode 100644 index 000000000..543bcb5ad --- /dev/null +++ b/taskcluster/test/params/github-push-master.yml @@ -0,0 +1,29 @@ +base_ref: master +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: dev +backend_url: "https://api.code-review.testing.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: refs/heads/master +head_repository: https://github.com/mozilla/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "3" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-push +version: null +next_version: null diff --git a/taskcluster/test/params/github-push-production.yml b/taskcluster/test/params/github-push-production.yml new file mode 100644 index 000000000..0cd875c95 --- /dev/null +++ b/taskcluster/test/params/github-push-production.yml @@ -0,0 +1,29 @@ +base_ref: production +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: production +backend_url: "https://api.code-review.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: refs/heads/production +head_repository: https://github.com/mozilla/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "3" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-push +version: null +next_version: null diff --git a/taskcluster/test/params/github-push-testing.yml b/taskcluster/test/params/github-push-testing.yml new file mode 100644 index 000000000..5708642ad --- /dev/null +++ b/taskcluster/test/params/github-push-testing.yml @@ -0,0 +1,29 @@ +base_ref: testing +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: testing +backend_url: "https://api.code-review.testing.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: refs/heads/testing +head_repository: https://github.com/mozilla/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "3" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-push +version: null +next_version: null diff --git a/taskcluster/test/params/github-release.yml b/taskcluster/test/params/github-release.yml new file mode 100644 index 000000000..6286a9613 --- /dev/null +++ b/taskcluster/test/params/github-release.yml @@ -0,0 +1,29 @@ +base_ref: "" +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: dev +backend_url: "https://api.code-review.testing.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: v1.0.0 +head_repository: https://github.com/mozilla/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "3" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-release +version: null +next_version: null