diff --git a/.github/actions/run-integration-test/action.yml b/.github/actions/run-integration-test/action.yml new file mode 100644 index 0000000000..07f63b39a4 --- /dev/null +++ b/.github/actions/run-integration-test/action.yml @@ -0,0 +1,45 @@ +name: "Run Integration Test" +description: "Run integration tests with retry on infra failures" + +inputs: + make-args: + description: "Extra arguments to pass to make (e.g. FIREFOX_CHANNEL=beta)" + default: "" + retries: + description: "Number of retry attempts after the first failure" + default: "1" + artifact-name: + description: "Name for the uploaded test report artifact" + required: true + +runs: + using: composite + steps: + - name: Run integration tests + shell: bash + env: + BUILDKIT_PROGRESS: plain + COMPOSE_ANSI: never + run: | + cp .env.integration-tests .env + attempts=$(( ${{ inputs.retries }} + 1 )) + for attempt in $(seq 1 "$attempts"); do + echo "::group::Attempt $attempt of $attempts" + if make refresh SKIP_DUMMY=1 ${{ inputs.make-args }} up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="$PYTEST_ARGS"; then + echo "::endgroup::" + exit 0 + fi + echo "::endgroup::" + if [ "$attempt" -eq "$attempts" ]; then + echo "::error::All $attempts attempts failed" + exit 1 + fi + echo "::warning::Attempt $attempt failed, retrying..." + done + + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: experimenter/tests/integration/test-reports/report.htm diff --git a/.github/workflows/integration-desktop-enrollment.yml b/.github/workflows/integration-desktop-enrollment.yml new file mode 100644 index 0000000000..5a9bc4821d --- /dev/null +++ b/.github/workflows/integration-desktop-enrollment.yml @@ -0,0 +1,46 @@ +name: Desktop Enrollment Integration Tests + +on: + push: + branches: + - main + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-desktop-enrollment.yml" + pull_request: + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-desktop-enrollment.yml" + merge_group: + types: [checks_requested] + +jobs: + test: + strategy: + fail-fast: false + matrix: + split: [0, 1] + name: "Desktop Enrollment (${{ matrix.split }})" + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + env: + FIREFOX_CHANNEL: release + PYTEST_ARGS: -k FIREFOX_DESKTOP -m desktop_enrollment --reruns 1 --splits 2 --split ${{ matrix.split }} --base-url https://nginx/nimbus/ + PYTEST_BASE_URL: https://nginx/nimbus/ + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup-cached-build + with: + arch: x86_64 + gcp-project-number: ${{ vars.GCPV2_WORKLOAD_IDENTITY_POOL_PROJECT_NUMBER }} + + - uses: ./.github/actions/run-integration-test + with: + artifact-name: desktop-enrollment-${{ matrix.split }}-test-report diff --git a/.github/workflows/integration-desktop-targeting.yml b/.github/workflows/integration-desktop-targeting.yml new file mode 100644 index 0000000000..646bef105b --- /dev/null +++ b/.github/workflows/integration-desktop-targeting.yml @@ -0,0 +1,48 @@ +name: Desktop Targeting Integration Tests + +on: + push: + branches: + - main + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-desktop-targeting.yml" + pull_request: + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-desktop-targeting.yml" + merge_group: + types: [checks_requested] + +jobs: + test: + strategy: + fail-fast: false + matrix: + channel: [release, beta, nightly] + split: [0, 1] + name: "Desktop Targeting (${{ matrix.channel }} ${{ matrix.split }})" + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + env: + FIREFOX_CHANNEL: ${{ matrix.channel }} + PYTEST_ARGS: -k FIREFOX_DESKTOP -m run_targeting -n 4 --reruns 1 --splits 2 --split ${{ matrix.split }} --base-url https://nginx/nimbus/ + PYTEST_BASE_URL: https://nginx/nimbus/ + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup-cached-build + with: + arch: x86_64 + gcp-project-number: ${{ vars.GCPV2_WORKLOAD_IDENTITY_POOL_PROJECT_NUMBER }} + + - uses: ./.github/actions/run-integration-test + with: + make-args: FIREFOX_CHANNEL=${{ matrix.channel }} + artifact-name: desktop-targeting-${{ matrix.channel }}-${{ matrix.split }}-test-report diff --git a/.github/workflows/integration-nimbus-ui.yml b/.github/workflows/integration-nimbus-ui.yml new file mode 100644 index 0000000000..e7ccf56f2b --- /dev/null +++ b/.github/workflows/integration-nimbus-ui.yml @@ -0,0 +1,42 @@ +name: Nimbus UI Integration Tests + +on: + push: + branches: + - main + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-nimbus-ui.yml" + pull_request: + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-nimbus-ui.yml" + merge_group: + types: [checks_requested] + +jobs: + test: + name: "Desktop Nimbus UI (Release Firefox)" + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + env: + FIREFOX_CHANNEL: release + PYTEST_ARGS: -m nimbus_ui -n 4 --reruns 1 --base-url https://nginx/nimbus/ + PYTEST_BASE_URL: https://nginx/nimbus/ + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup-cached-build + with: + arch: x86_64 + gcp-project-number: ${{ vars.GCPV2_WORKLOAD_IDENTITY_POOL_PROJECT_NUMBER }} + + - uses: ./.github/actions/run-integration-test + with: + artifact-name: nimbus-ui-test-report diff --git a/.github/workflows/integration-remote-settings-all.yml b/.github/workflows/integration-remote-settings-all.yml new file mode 100644 index 0000000000..749020725e --- /dev/null +++ b/.github/workflows/integration-remote-settings-all.yml @@ -0,0 +1,52 @@ +name: Remote Settings All Workflows Integration Tests + +on: + push: + branches: + - main + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-remote-settings-all.yml" + pull_request: + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-remote-settings-all.yml" + merge_group: + types: [checks_requested] + +jobs: + test: + name: "Remote Settings All Workflows (${{ matrix.name }})" + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + include: + - name: Experiments + marker: remote_settings_experiments + - name: Rollouts + marker: remote_settings_rollouts + - name: Live Updates + marker: remote_settings_live_updates + env: + FIREFOX_CHANNEL: release + PYTEST_ARGS: -k FIREFOX_DESKTOP -m ${{ matrix.marker }} --reruns 1 --base-url https://nginx/nimbus/ + PYTEST_BASE_URL: https://nginx/nimbus/ + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup-cached-build + with: + arch: x86_64 + gcp-project-number: ${{ vars.GCPV2_WORKLOAD_IDENTITY_POOL_PROJECT_NUMBER }} + + - uses: ./.github/actions/run-integration-test + with: + artifact-name: remote-settings-all-${{ matrix.marker }}-test-report diff --git a/.github/workflows/integration-remote-settings-launch.yml b/.github/workflows/integration-remote-settings-launch.yml new file mode 100644 index 0000000000..1b8bd0acf4 --- /dev/null +++ b/.github/workflows/integration-remote-settings-launch.yml @@ -0,0 +1,42 @@ +name: Remote Settings Launch Integration Tests + +on: + push: + branches: + - main + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-remote-settings-launch.yml" + pull_request: + paths: + - "experimenter/**" + - "application-services/**" + - ".github/actions/run-integration-test/**" + - ".github/workflows/integration-remote-settings-launch.yml" + merge_group: + types: [checks_requested] + +jobs: + test: + name: "Remote Settings Launch (All Applications)" + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + env: + FIREFOX_CHANNEL: release + PYTEST_ARGS: -m remote_settings_launch --reruns 1 --base-url https://nginx/nimbus/ + PYTEST_BASE_URL: https://nginx/nimbus/ + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup-cached-build + with: + arch: x86_64 + gcp-project-number: ${{ vars.GCPV2_WORKLOAD_IDENTITY_POOL_PROJECT_NUMBER }} + + - uses: ./.github/actions/run-integration-test + with: + artifact-name: remote-settings-launch-test-report diff --git a/Makefile b/Makefile index 1de8b4888c..15c472b92d 100644 --- a/Makefile +++ b/Makefile @@ -127,7 +127,7 @@ build_ui: ssl $(DOCKER_BUILD) --target ui -f experimenter/Dockerfile -t experimenter:ui experimenter/ build_prod: ssl build_megazords - $(DOCKER_BUILD) --target deploy -f experimenter/Dockerfile -t experimenter:deploy experimenter/ + $(DOCKER_BUILD) $(EXPERIMENTER_BUILD_FLAGS) --target deploy -f experimenter/Dockerfile -t experimenter:deploy experimenter/ compose_stop: $(CIRRUS_ENABLE) $(COMPOSE) kill || true @@ -265,7 +265,7 @@ CIRRUS_PYTHON_TYPECHECK_CREATESTUB = pyright -p . --createstub cirrus CIRRUS_GENERATE_DOCS = python cirrus/generate_docs.py cirrus_build: build_megazords - $(CIRRUS_ENABLE) $(DOCKER_BUILD) --target deploy -f cirrus/server/Dockerfile -t cirrus:deploy --build-context=fml=experimenter/experimenter/features/manifests/ cirrus/server/ + $(CIRRUS_ENABLE) $(DOCKER_BUILD) $(CIRRUS_BUILD_FLAGS) --target deploy -f cirrus/server/Dockerfile -t cirrus:deploy --build-context=fml=experimenter/experimenter/features/manifests/ cirrus/server/ cirrus_build_dev: build_megazords $(CIRRUS_ENABLE) $(DOCKER_BUILD) --target dev -f cirrus/server/Dockerfile -t cirrus:dev --build-context=fml=experimenter/experimenter/features/manifests/ cirrus/server/ diff --git a/experimenter/tests/integration/nimbus/conftest.py b/experimenter/tests/integration/nimbus/conftest.py index dd808be2f8..796b0e2923 100755 --- a/experimenter/tests/integration/nimbus/conftest.py +++ b/experimenter/tests/integration/nimbus/conftest.py @@ -30,6 +30,30 @@ from nimbus.pages.experimenter.home import HomePage from nimbus.utils import helpers + +def pytest_addoption(parser): + parser.addoption( + "--split", + type=int, + default=None, + help="This node's split index (0-based)", + ) + parser.addoption( + "--splits", + type=int, + default=None, + help="Total number of split nodes", + ) + + +def pytest_collection_modifyitems(config, items): + split = config.getoption("--split") + splits = config.getoption("--splits") + if split is not None and splits is not None: + items.sort(key=lambda item: item.nodeid) + items[:] = [item for i, item in enumerate(items) if i % splits == split] + + APPLICATION_KINTO_REVIEW_PATH = { BaseExperimentApplications.FIREFOX_DESKTOP.value: ( "#/buckets/main-workspace/collections/nimbus-desktop-experiments/simple-review"