diff --git a/README.md b/README.md index 7270a5281..5e04aa7e5 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,7 @@ Error codes are as follows: Certain errors are automatically retried 5 times by default, with a short exponential backoff. Connection errors (for example, due to a network connectivity problem), 408 Request Timeout, 409 Conflict, -429 Rate Limit, and >=500 Internal errors are all retried by default for GET requests. For POST requests, only +429 Rate Limit, and >=500 Internal errors are all retried by default for GET requests. For POST requests, only 429 errors will be retried. You can use the `max_retries` option to configure or disable retry settings: @@ -265,6 +265,35 @@ client = Runloop( client.with_options(max_retries=10).devboxes.create() ``` +#### File Write Rate Limiting + +File write operations (`write_file_contents` and `upload_file`) are subject to rate limiting on the backend +(approximately 80 chunks per second, equivalent to ~10MB/sec per connection). When rate limits are exceeded, +the API returns a `429 Too Many Requests` error, which is automatically retried with exponential backoff. + +The retry behavior includes: +- **Default retries**: 5 attempts (initial request + 4 retries) +- **Backoff strategy**: Exponential with jitter to prevent thundering herd +- **Retry-After header**: Respected when provided by the API +- **Configurable**: Use `max_retries` parameter to adjust retry behavior + +Example: +```python +from runloop_api_client import Runloop + +client = Runloop( + bearer_token="your-api-key", + max_retries=5 # Recommended for file operations +) + +# Automatically retries on rate limit errors +result = client.devboxes.write_file_contents( + id="devbox-id", + contents="large file content...", + file_path="/path/to/file.txt" +) +``` + ### Timeouts By default requests time out after 30 seconds. You can configure this with a `timeout` option, diff --git a/src/runloop_api_client/resources/devboxes/devboxes.py b/src/runloop_api_client/resources/devboxes/devboxes.py index 4a8bffe9e..78d15b510 100644 --- a/src/runloop_api_client/resources/devboxes/devboxes.py +++ b/src/runloop_api_client/resources/devboxes/devboxes.py @@ -718,7 +718,7 @@ def execute( id: str, *, command: str, - command_id: str, + command_id: str = str(uuid7()), optimistic_timeout: Optional[int] | Omit = omit, shell_name: Optional[str] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -897,6 +897,7 @@ def execute_async( ) @typing_extensions.deprecated("deprecated") + # Use execute, executeAsync, or executeAndAwaitCompletion instead def execute_sync( self, id: str, @@ -915,6 +916,9 @@ def execute_sync( Execute a bash command in the Devbox shell, await the command completion and return the output. + .. deprecated:: + Use execute, executeAsync, or executeAndAwaitCompletion instead. + Args: command: The command to execute via the Devbox shell. By default, commands are run from the user home directory unless shell_name is specified. If shell_name is @@ -2161,7 +2165,7 @@ async def execute( id: str, *, command: str, - command_id: str, + command_id: str = str(uuid7()), optimistic_timeout: Optional[int] | Omit = omit, shell_name: Optional[str] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -2341,6 +2345,7 @@ async def execute_async( ) @typing_extensions.deprecated("deprecated") + # Use execute, executeAsync, or executeAndAwaitCompletion instead async def execute_sync( self, id: str, @@ -2359,6 +2364,9 @@ async def execute_sync( Execute a bash command in the Devbox shell, await the command completion and return the output. + .. deprecated:: + Use execute, executeAsync, or executeAndAwaitCompletion instead. + Args: command: The command to execute via the Devbox shell. By default, commands are run from the user home directory unless shell_name is specified. If shell_name is diff --git a/tests/test_rate_limit_retry.py b/tests/test_rate_limit_retry.py new file mode 100644 index 000000000..82eceb555 --- /dev/null +++ b/tests/test_rate_limit_retry.py @@ -0,0 +1,343 @@ +"""Tests for rate limit retry behavior on file write operations.""" + +from __future__ import annotations + +import time +from typing import Any +from unittest.mock import Mock, patch + +import httpx +import pytest +import respx + +from runloop_api_client import Runloop, AsyncRunloop +from runloop_api_client._exceptions import RateLimitError +from runloop_api_client.types import DevboxExecutionDetailView + +base_url = "http://127.0.0.1:4010" + + +class TestRateLimitRetry: + """Test rate limit retry behavior for file write operations.""" + + def test_write_file_contents_retries_on_429(self, respx_mock: respx.MockRouter) -> None: + """Test that write_file_contents retries when encountering 429 errors.""" + client = Runloop(base_url=base_url, bearer_token="test-token", max_retries=3) + + # Mock the first two requests to return 429, then succeed + route = respx_mock.post(f"{base_url}/v1/devboxes/test-devbox-id/write_file_contents") + + route.side_effect = [ + # First attempt: 429 + httpx.Response( + status_code=429, + json={ + "error": { + "message": "Write operations for this devbox are currently rate limited. Please retry in a few seconds." + } + }, + ), + # Second attempt: 429 + httpx.Response( + status_code=429, + json={ + "error": { + "message": "Write operations for this devbox are currently rate limited. Please retry in a few seconds." + } + }, + ), + # Third attempt: success + httpx.Response( + status_code=200, + json={ + "id": "exec-123", + "devbox_id": "test-devbox-id", + "status": "completed", + "exit_status": 0, + }, + ), + ] + + start_time = time.time() + result = client.devboxes.write_file_contents( + id="test-devbox-id", + contents="test content", + file_path="/tmp/test.txt", + ) + elapsed_time = time.time() - start_time + + # Verify the request succeeded + assert result.id == "exec-123" + assert result.status == "completed" + + # Verify retry happened (should have taken at least some time due to backoff) + assert elapsed_time > 0.1 # At least some delay from retries + + # Verify all three requests were made + assert route.call_count == 3 + + def test_write_file_contents_respects_retry_after_header(self, respx_mock: respx.MockRouter) -> None: + """Test that write_file_contents respects Retry-After header.""" + client = Runloop(base_url=base_url, bearer_token="test-token", max_retries=2) + + route = respx_mock.post(f"{base_url}/v1/devboxes/test-devbox-id/write_file_contents") + + route.side_effect = [ + # First attempt: 429 with Retry-After header + httpx.Response( + status_code=429, + headers={"Retry-After": "1"}, # Retry after 1 second + json={ + "error": { + "message": "Write operations for this devbox are currently rate limited." + } + }, + ), + # Second attempt: success + httpx.Response( + status_code=200, + json={ + "id": "exec-456", + "devbox_id": "test-devbox-id", + "status": "completed", + "exit_status": 0, + }, + ), + ] + + start_time = time.time() + result = client.devboxes.write_file_contents( + id="test-devbox-id", + contents="test content", + file_path="/tmp/test.txt", + ) + elapsed_time = time.time() - start_time + + # Verify the request succeeded + assert result.id == "exec-456" + + # Verify it waited approximately 1 second (Retry-After value) + assert elapsed_time >= 0.9 # Allow slight timing variance + assert elapsed_time < 2.0 # Should not take too long + + # Verify two requests were made + assert route.call_count == 2 + + def test_write_file_contents_exhausts_retries(self, respx_mock: respx.MockRouter) -> None: + """Test that write_file_contents fails after exhausting retries.""" + client = Runloop(base_url=base_url, bearer_token="test-token", max_retries=2) + + route = respx_mock.post(f"{base_url}/v1/devboxes/test-devbox-id/write_file_contents") + + # All attempts return 429 + for _ in range(3): # max_retries + 1 + route.mock( + return_value=httpx.Response( + status_code=429, + json={ + "error": { + "message": "Write operations for this devbox are currently rate limited." + } + }, + ) + ) + + # Should raise RateLimitError after exhausting retries + with pytest.raises(RateLimitError) as exc_info: + client.devboxes.write_file_contents( + id="test-devbox-id", + contents="test content", + file_path="/tmp/test.txt", + ) + + # Verify error message + assert "rate limit" in str(exc_info.value).lower() + + # Verify all retry attempts were made + assert route.call_count == 3 # initial + 2 retries + + def test_upload_file_retries_on_429(self, respx_mock: respx.MockRouter) -> None: + """Test that upload_file retries when encountering 429 errors.""" + client = Runloop(base_url=base_url, bearer_token="test-token", max_retries=3) + + route = respx_mock.post(f"{base_url}/v1/devboxes/test-devbox-id/upload_file") + + route.side_effect = [ + # First attempt: 429 + httpx.Response( + status_code=429, + json={ + "error": { + "message": "Write operations for this devbox are currently rate limited." + } + }, + ), + # Second attempt: success + httpx.Response( + status_code=200, + json={"success": True}, + ), + ] + + result = client.devboxes.upload_file( + id="test-devbox-id", + path="/tmp/test.bin", + file=b"binary content", + ) + + # Verify the request succeeded + assert result == {"success": True} + + # Verify retry happened + assert route.call_count == 2 + + def test_write_file_contents_with_custom_max_retries(self, respx_mock: respx.MockRouter) -> None: + """Test that custom max_retries configuration is respected.""" + client = Runloop(base_url=base_url, bearer_token="test-token", max_retries=5) + + route = respx_mock.post(f"{base_url}/v1/devboxes/test-devbox-id/write_file_contents") + + # Return 429 for first 4 attempts, then succeed + route.side_effect = [ + httpx.Response(status_code=429, json={"error": {"message": "Rate limited"}}), + httpx.Response(status_code=429, json={"error": {"message": "Rate limited"}}), + httpx.Response(status_code=429, json={"error": {"message": "Rate limited"}}), + httpx.Response(status_code=429, json={"error": {"message": "Rate limited"}}), + httpx.Response( + status_code=200, + json={ + "id": "exec-789", + "devbox_id": "test-devbox-id", + "status": "completed", + "exit_status": 0, + }, + ), + ] + + result = client.devboxes.write_file_contents( + id="test-devbox-id", + contents="test content", + file_path="/tmp/test.txt", + ) + + # Verify success after 5 attempts (initial + 4 retries) + assert result.id == "exec-789" + assert route.call_count == 5 + + def test_write_file_contents_no_retry_when_disabled(self, respx_mock: respx.MockRouter) -> None: + """Test that retries can be disabled by setting max_retries=0.""" + client = Runloop(base_url=base_url, bearer_token="test-token", max_retries=0) + + route = respx_mock.post(f"{base_url}/v1/devboxes/test-devbox-id/write_file_contents") + route.mock( + return_value=httpx.Response( + status_code=429, + json={"error": {"message": "Rate limited"}}, + ) + ) + + # Should fail immediately without retry + with pytest.raises(RateLimitError): + client.devboxes.write_file_contents( + id="test-devbox-id", + contents="test content", + file_path="/tmp/test.txt", + ) + + # Verify only one request was made (no retries) + assert route.call_count == 1 + + +class TestAsyncRateLimitRetry: + """Test async rate limit retry behavior for file write operations.""" + + @pytest.mark.asyncio + async def test_write_file_contents_retries_on_429(self, respx_mock: respx.MockRouter) -> None: + """Test that async write_file_contents retries when encountering 429 errors.""" + client = AsyncRunloop(base_url=base_url, bearer_token="test-token", max_retries=3) + + route = respx_mock.post(f"{base_url}/v1/devboxes/test-devbox-id/write_file_contents") + + route.side_effect = [ + # First two attempts: 429 + httpx.Response(status_code=429, json={"error": {"message": "Rate limited"}}), + httpx.Response(status_code=429, json={"error": {"message": "Rate limited"}}), + # Third attempt: success + httpx.Response( + status_code=200, + json={ + "id": "exec-async-123", + "devbox_id": "test-devbox-id", + "status": "completed", + "exit_status": 0, + }, + ), + ] + + start_time = time.time() + result = await client.devboxes.write_file_contents( + id="test-devbox-id", + contents="test content", + file_path="/tmp/test.txt", + ) + elapsed_time = time.time() - start_time + + # Verify the request succeeded + assert result.id == "exec-async-123" + + # Verify retry happened (should have taken some time) + assert elapsed_time > 0.1 + + # Verify all three requests were made + assert route.call_count == 3 + + @pytest.mark.asyncio + async def test_upload_file_retries_on_429(self, respx_mock: respx.MockRouter) -> None: + """Test that async upload_file retries when encountering 429 errors.""" + client = AsyncRunloop(base_url=base_url, bearer_token="test-token", max_retries=2) + + route = respx_mock.post(f"{base_url}/v1/devboxes/test-devbox-id/upload_file") + + route.side_effect = [ + # First attempt: 429 + httpx.Response(status_code=429, json={"error": {"message": "Rate limited"}}), + # Second attempt: success + httpx.Response(status_code=200, json={"success": True}), + ] + + result = await client.devboxes.upload_file( + id="test-devbox-id", + path="/tmp/test.bin", + file=b"binary content", + ) + + # Verify the request succeeded + assert result == {"success": True} + assert route.call_count == 2 + + @pytest.mark.asyncio + async def test_write_file_contents_exhausts_retries(self, respx_mock: respx.MockRouter) -> None: + """Test that async write_file_contents fails after exhausting retries.""" + client = AsyncRunloop(base_url=base_url, bearer_token="test-token", max_retries=2) + + route = respx_mock.post(f"{base_url}/v1/devboxes/test-devbox-id/write_file_contents") + + # All attempts return 429 + for _ in range(3): + route.mock( + return_value=httpx.Response( + status_code=429, + json={"error": {"message": "Rate limited"}}, + ) + ) + + # Should raise RateLimitError + with pytest.raises(RateLimitError): + await client.devboxes.write_file_contents( + id="test-devbox-id", + contents="test content", + file_path="/tmp/test.txt", + ) + + # Verify all retry attempts were made + assert route.call_count == 3