From b87d7bb960a319ba8b35e5b77c34f0cd5aa83651 Mon Sep 17 00:00:00 2001 From: Albert Li Date: Wed, 22 Oct 2025 22:36:43 -0700 Subject: [PATCH 1/5] Add smoketest for bpt secrets and gracefully shutdown devboxes when test fails --- tests/smoketests/test_blueprints.py | 59 +++++++++++++++++++++++------ tests/smoketests/test_snapshots.py | 16 +++++--- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/tests/smoketests/test_blueprints.py b/tests/smoketests/test_blueprints.py index e5d0fd2f4..4591aefc5 100644 --- a/tests/smoketests/test_blueprints.py +++ b/tests/smoketests/test_blueprints.py @@ -45,21 +45,58 @@ def test_create_blueprint_and_await_build(client: Runloop) -> None: @pytest.mark.timeout(30) def test_start_devbox_from_base_blueprint_by_id(client: Runloop) -> None: assert _blueprint_id - devbox = client.devboxes.create_and_await_running( + devbox = None + try: + devbox = client.devboxes.create_and_await_running( blueprint_id=_blueprint_id, polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60), ) - assert devbox.blueprint_id == _blueprint_id - assert devbox.status == "running" - client.devboxes.shutdown(devbox.id) + assert devbox.blueprint_id == _blueprint_id + assert devbox.status == "running" + finally: + if devbox: + client.devboxes.shutdown(devbox.id) @pytest.mark.timeout(30) def test_start_devbox_from_base_blueprint_by_name(client: Runloop) -> None: - devbox = client.devboxes.create_and_await_running( - blueprint_name=_blueprint_name, - polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60), - ) - assert devbox.blueprint_id - assert devbox.status == "running" - client.devboxes.shutdown(devbox.id) + devbox = None + try: + devbox = client.devboxes.create_and_await_running( + blueprint_name=_blueprint_name, + polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60), + ) + assert devbox.blueprint_id + assert devbox.status == "running" + finally: + if devbox: + client.devboxes.shutdown(devbox.id) + + +@pytest.mark.timeout(60) +def test_create_blueprint_with_secret_and_await_build(client: Runloop) -> None: + bpt = None + try: + bpt = client.blueprints.create( + name=unique_name("bp-secrets"), + dockerfile=( + "FROM runloop:runloop/starter-arm64\n" + "ARG GITHUB_TOKEN\n" + "RUN git config --global credential.helper '!f() { echo \"username=x-access-token\"; echo \"password=$GITHUB_TOKEN\"; }; f' " + "&& git clone https://github.com/runloopai/runloop-fe.git /workspace/runloop-fe " + "&& git config --global --unset credential.helper\n" + "WORKDIR /workspace/runloop-fe" + ), + secrets={"GITHUB_TOKEN": "GITHUB_TOKEN_FOR_SMOKETESTS"}, + ) + # Wait for build to complete + completed = client.blueprints.await_build_complete( + bpt.id, + polling_config=PollingConfig(max_attempts=180, interval_seconds=5.0, timeout_seconds=30 * 60), + ) + assert completed.status == "build_complete" + assert completed.parameters.secrets is not None + assert completed.parameters.secrets.get("GITHUB_TOKEN") == "GITHUB_TOKEN_FOR_SMOKETESTS" + finally: + if bpt: + client.blueprints.delete(bpt.id) diff --git a/tests/smoketests/test_snapshots.py b/tests/smoketests/test_snapshots.py index a7864cc9a..4b1a27b9f 100644 --- a/tests/smoketests/test_snapshots.py +++ b/tests/smoketests/test_snapshots.py @@ -48,9 +48,13 @@ def test_snapshot_devbox(client: Runloop) -> None: @pytest.mark.timeout(30) def test_launch_devbox_from_snapshot(client: Runloop) -> None: assert _snapshot_id - launched = client.devboxes.create_and_await_running( - snapshot_id=_snapshot_id, - polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60), - ) - assert launched.snapshot_id == _snapshot_id - client.devboxes.shutdown(launched.id) + launched = None + try: + launched = client.devboxes.create_and_await_running( + snapshot_id=_snapshot_id, + polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60), + ) + assert launched.snapshot_id == _snapshot_id + finally: + if launched: + client.devboxes.shutdown(launched.id) From 363f351aae5d13905bca121bc020deb56f2bf98c Mon Sep 17 00:00:00 2001 From: Albert Li Date: Wed, 22 Oct 2025 22:47:43 -0700 Subject: [PATCH 2/5] format --- tests/smoketests/test_blueprints.py | 16 ++++++++-------- tests/smoketests/test_snapshots.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/smoketests/test_blueprints.py b/tests/smoketests/test_blueprints.py index 4591aefc5..9ee7b2ca6 100644 --- a/tests/smoketests/test_blueprints.py +++ b/tests/smoketests/test_blueprints.py @@ -46,11 +46,11 @@ def test_create_blueprint_and_await_build(client: Runloop) -> None: def test_start_devbox_from_base_blueprint_by_id(client: Runloop) -> None: assert _blueprint_id devbox = None - try: + try: devbox = client.devboxes.create_and_await_running( - blueprint_id=_blueprint_id, - polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60), - ) + blueprint_id=_blueprint_id, + polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60), + ) assert devbox.blueprint_id == _blueprint_id assert devbox.status == "running" finally: @@ -61,7 +61,7 @@ def test_start_devbox_from_base_blueprint_by_id(client: Runloop) -> None: @pytest.mark.timeout(30) def test_start_devbox_from_base_blueprint_by_name(client: Runloop) -> None: devbox = None - try: + try: devbox = client.devboxes.create_and_await_running( blueprint_name=_blueprint_name, polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60), @@ -76,13 +76,13 @@ def test_start_devbox_from_base_blueprint_by_name(client: Runloop) -> None: @pytest.mark.timeout(60) def test_create_blueprint_with_secret_and_await_build(client: Runloop) -> None: bpt = None - try: + try: bpt = client.blueprints.create( name=unique_name("bp-secrets"), dockerfile=( "FROM runloop:runloop/starter-arm64\n" "ARG GITHUB_TOKEN\n" - "RUN git config --global credential.helper '!f() { echo \"username=x-access-token\"; echo \"password=$GITHUB_TOKEN\"; }; f' " + 'RUN git config --global credential.helper \'!f() { echo "username=x-access-token"; echo "password=$GITHUB_TOKEN"; }; f\' ' "&& git clone https://github.com/runloopai/runloop-fe.git /workspace/runloop-fe " "&& git config --global --unset credential.helper\n" "WORKDIR /workspace/runloop-fe" @@ -97,6 +97,6 @@ def test_create_blueprint_with_secret_and_await_build(client: Runloop) -> None: assert completed.status == "build_complete" assert completed.parameters.secrets is not None assert completed.parameters.secrets.get("GITHUB_TOKEN") == "GITHUB_TOKEN_FOR_SMOKETESTS" - finally: + finally: if bpt: client.blueprints.delete(bpt.id) diff --git a/tests/smoketests/test_snapshots.py b/tests/smoketests/test_snapshots.py index 4b1a27b9f..09797f02c 100644 --- a/tests/smoketests/test_snapshots.py +++ b/tests/smoketests/test_snapshots.py @@ -49,7 +49,7 @@ def test_snapshot_devbox(client: Runloop) -> None: def test_launch_devbox_from_snapshot(client: Runloop) -> None: assert _snapshot_id launched = None - try: + try: launched = client.devboxes.create_and_await_running( snapshot_id=_snapshot_id, polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60), From 1fdac4179fc85e1dd853c47edf2421ed1db853c7 Mon Sep 17 00:00:00 2001 From: Albert Li Date: Thu, 23 Oct 2025 11:09:34 -0700 Subject: [PATCH 3/5] Disable secrets test if local dev --- tests/smoketests/test_blueprints.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/smoketests/test_blueprints.py b/tests/smoketests/test_blueprints.py index 9ee7b2ca6..b980c7165 100644 --- a/tests/smoketests/test_blueprints.py +++ b/tests/smoketests/test_blueprints.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from typing import Iterator import pytest @@ -74,6 +75,10 @@ def test_start_devbox_from_base_blueprint_by_name(client: Runloop) -> None: @pytest.mark.timeout(60) +@pytest.mark.skipif( + os.getenv("RUN_SMOKETESTS") != "1", + reason="Skip blueprint secrets test in local testing (requires RUN_SMOKETESTS=1)", +) def test_create_blueprint_with_secret_and_await_build(client: Runloop) -> None: bpt = None try: @@ -89,7 +94,7 @@ def test_create_blueprint_with_secret_and_await_build(client: Runloop) -> None: ), secrets={"GITHUB_TOKEN": "GITHUB_TOKEN_FOR_SMOKETESTS"}, ) - # Wait for build to complete + completed = client.blueprints.await_build_complete( bpt.id, polling_config=PollingConfig(max_attempts=180, interval_seconds=5.0, timeout_seconds=30 * 60), From ed3c49257ee35d2ce258401b9b779d92076d9c66 Mon Sep 17 00:00:00 2001 From: Albert Li Date: Thu, 23 Oct 2025 12:14:42 -0700 Subject: [PATCH 4/5] fix tests --- tests/smoketests/test_blueprints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/smoketests/test_blueprints.py b/tests/smoketests/test_blueprints.py index b980c7165..283765c82 100644 --- a/tests/smoketests/test_blueprints.py +++ b/tests/smoketests/test_blueprints.py @@ -32,7 +32,7 @@ def _cleanup(client: Runloop) -> Iterator[None]: # pyright: ignore[reportUnused _blueprint_name = unique_name("bp") -@pytest.mark.timeout(30) +@pytest.mark.timeout(60) def test_create_blueprint_and_await_build(client: Runloop) -> None: global _blueprint_id created = client.blueprints.create_and_await_build_complete( @@ -43,7 +43,7 @@ def test_create_blueprint_and_await_build(client: Runloop) -> None: _blueprint_id = created.id -@pytest.mark.timeout(30) +@pytest.mark.timeout(60) def test_start_devbox_from_base_blueprint_by_id(client: Runloop) -> None: assert _blueprint_id devbox = None @@ -59,7 +59,7 @@ def test_start_devbox_from_base_blueprint_by_id(client: Runloop) -> None: client.devboxes.shutdown(devbox.id) -@pytest.mark.timeout(30) +@pytest.mark.timeout(60) def test_start_devbox_from_base_blueprint_by_name(client: Runloop) -> None: devbox = None try: From e59d472c0c2302c28da8083c8d94dc90b4859dfd Mon Sep 17 00:00:00 2001 From: Albert Li Date: Thu, 23 Oct 2025 13:42:55 -0700 Subject: [PATCH 5/5] extend test timeouts --- tests/smoketests/test_blueprints.py | 8 ++++---- tests/smoketests/test_devboxes.py | 4 ++-- tests/smoketests/test_snapshots.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/smoketests/test_blueprints.py b/tests/smoketests/test_blueprints.py index 283765c82..6aa5a3d17 100644 --- a/tests/smoketests/test_blueprints.py +++ b/tests/smoketests/test_blueprints.py @@ -32,7 +32,7 @@ def _cleanup(client: Runloop) -> Iterator[None]: # pyright: ignore[reportUnused _blueprint_name = unique_name("bp") -@pytest.mark.timeout(60) +@pytest.mark.timeout(120) # 2 minutes def test_create_blueprint_and_await_build(client: Runloop) -> None: global _blueprint_id created = client.blueprints.create_and_await_build_complete( @@ -43,7 +43,7 @@ def test_create_blueprint_and_await_build(client: Runloop) -> None: _blueprint_id = created.id -@pytest.mark.timeout(60) +@pytest.mark.timeout(120) def test_start_devbox_from_base_blueprint_by_id(client: Runloop) -> None: assert _blueprint_id devbox = None @@ -59,7 +59,7 @@ def test_start_devbox_from_base_blueprint_by_id(client: Runloop) -> None: client.devboxes.shutdown(devbox.id) -@pytest.mark.timeout(60) +@pytest.mark.timeout(120) def test_start_devbox_from_base_blueprint_by_name(client: Runloop) -> None: devbox = None try: @@ -74,7 +74,7 @@ def test_start_devbox_from_base_blueprint_by_name(client: Runloop) -> None: client.devboxes.shutdown(devbox.id) -@pytest.mark.timeout(60) +@pytest.mark.timeout(120) @pytest.mark.skipif( os.getenv("RUN_SMOKETESTS") != "1", reason="Skip blueprint secrets test in local testing (requires RUN_SMOKETESTS=1)", diff --git a/tests/smoketests/test_devboxes.py b/tests/smoketests/test_devboxes.py index db1df5acf..b26207a41 100644 --- a/tests/smoketests/test_devboxes.py +++ b/tests/smoketests/test_devboxes.py @@ -37,7 +37,7 @@ def test_create_devbox(client: Runloop) -> None: client.devboxes.shutdown(created.id) -@pytest.mark.timeout(30) +@pytest.mark.timeout(120) def test_await_running_create_and_await_running(client: Runloop) -> None: global _devbox_id created = client.devboxes.create_and_await_running( @@ -67,7 +67,7 @@ def test_shutdown_devbox(client: Runloop) -> None: assert view.status == "shutdown" -@pytest.mark.timeout(90) +@pytest.mark.timeout(120) def test_create_and_await_running_long_set_up(client: Runloop) -> None: created = client.devboxes.create_and_await_running( name=unique_name("smoketest-devbox-await-running-long-set-up"), diff --git a/tests/smoketests/test_snapshots.py b/tests/smoketests/test_snapshots.py index 09797f02c..71b592320 100644 --- a/tests/smoketests/test_snapshots.py +++ b/tests/smoketests/test_snapshots.py @@ -45,7 +45,7 @@ def test_snapshot_devbox(client: Runloop) -> None: _snapshot_id = snap.id -@pytest.mark.timeout(30) +@pytest.mark.timeout(120) def test_launch_devbox_from_snapshot(client: Runloop) -> None: assert _snapshot_id launched = None