From fc600634c94859e6e9cd17d17226dead9f08381c Mon Sep 17 00:00:00 2001 From: GMorris-professional Date: Wed, 30 Apr 2025 11:34:53 -0400 Subject: [PATCH] feat: improved handling of retries --- openfga_sdk/api_client.py | 42 +++++- openfga_sdk/configuration.py | 23 +++- openfga_sdk/exceptions.py | 4 +- openfga_sdk/sync/api_client.py | 40 +++++- test/api/open_fga_api_test.py | 241 ++++++++++++++++++++++++++++++++- test/sync/open_fga_api_test.py | 237 +++++++++++++++++++++++++++++++- 6 files changed, 573 insertions(+), 14 deletions(-) diff --git a/openfga_sdk/api_client.py b/openfga_sdk/api_client.py index b68d2c54..2d5631fc 100644 --- a/openfga_sdk/api_client.py +++ b/openfga_sdk/api_client.py @@ -42,7 +42,7 @@ DEFAULT_USER_AGENT = "openfga-sdk python/0.9.3" -def random_time(loop_count, min_wait_in_ms): +def random_time(loop_count, min_wait_in_ms) -> float: """ Helper function to return the time (in s) to wait before retry """ @@ -253,11 +253,21 @@ async def __call_api( ) else 0 ) + max_wait_in_sec = ( + self.configuration.retry_params.max_wait_in_sec + if ( + self.configuration.retry_params is not None + and self.configuration.retry_params.max_wait_in_sec is not None + ) + else 120 + ) if _retry_params is not None: if _retry_params.max_retry is not None: max_retry = _retry_params.max_retry if _retry_params.min_wait_in_ms is not None: max_retry = _retry_params.min_wait_in_ms + if _retry_params.max_wait_in_sec is not None: + max_wait_in_sec = _retry_params.max_wait_in_sec _telemetry_attributes = TelemetryAttributes.fromRequest( user_agent=self.user_agent, @@ -300,7 +310,14 @@ async def __call_api( configuration=self.configuration.telemetry, ) - await asyncio.sleep(random_time(retry, min_wait_in_ms)) + try: + wait_time_in_sec = self._parse_retry_after_header(e.header) + except ValueError: + wait_time_in_sec = min( + random_time(retry, min_wait_in_ms), max_wait_in_sec + ) + + await asyncio.sleep(wait_time_in_sec) continue e.body = e.body.decode("utf-8") @@ -395,6 +412,25 @@ async def __call_api( else: return (return_data, response_data.status, response_data.headers) + def _parse_retry_after_header(self, headers) -> int: + retry_after_header = headers.get("retry-after") + if not retry_after_header: + raise ValueError("Retry-After header is not present") + + try: + parsed_http_date = self.__deserialize_datetime(retry_after_header).replace( + tzinfo=datetime.timezone.utc + ) + now = datetime.datetime.now(datetime.timezone.utc) + wait_time_in_sec = (parsed_http_date - now).total_seconds() + except ApiException: + wait_time_in_sec = int(retry_after_header) + + if wait_time_in_sec > 1800 or wait_time_in_sec < 1: + raise ValueError("Retry-After header is invalid") + + return math.ceil(wait_time_in_sec) + def sanitize_for_serialization(self, obj): """Builds a JSON POST object. @@ -825,7 +861,7 @@ def __deserialize_datetime(self, string): return parse(string) except ImportError: return string - except ValueError: + except (TypeError, ValueError): raise rest.ApiException( status=0, reason=(f"Failed to parse `{string}` as datetime object"), diff --git a/openfga_sdk/configuration.py b/openfga_sdk/configuration.py index e308dbac..7ccd9721 100644 --- a/openfga_sdk/configuration.py +++ b/openfga_sdk/configuration.py @@ -41,11 +41,13 @@ class RetryParams: :param max_retry: Maximum number of retry :param min_wait_in_ms: Minimum wait (in ms) between retry + :param max_wait_in_sec: Maximum wait (in seconds) between retry """ - def __init__(self, max_retry=3, min_wait_in_ms=100): + def __init__(self, max_retry=3, min_wait_in_ms=100, max_wait_in_sec=120): self._max_retry = max_retry self._min_wait_in_ms = min_wait_in_ms + self._max_wait_in_sec = max_wait_in_sec @property def max_retry(self): @@ -95,6 +97,25 @@ def min_wait_in_ms(self, value): self._min_wait_in_ms = value + @property + def max_wait_in_sec(self): + """ + Return the maximum allowed wait (in seconds) in between retry + """ + return self._max_wait_in_sec + + @max_wait_in_sec.setter + def max_wait_in_sec(self, value): + """ + Update the maximum allowed wait (in seconds) in between retry + """ + if not isinstance(value, int) or value < 0: + raise FgaValidationException( + "RetryParams.max_wait_in_sec must be an integer greater than or equal to 0" + ) + + self._max_wait_in_sec = value + class Configuration: """NOTE: This class is auto generated by OpenAPI Generator diff --git a/openfga_sdk/exceptions.py b/openfga_sdk/exceptions.py index 3b6c9373..a10a5542 100644 --- a/openfga_sdk/exceptions.py +++ b/openfga_sdk/exceptions.py @@ -10,13 +10,14 @@ NOTE: This file was auto generated by OpenAPI Generator (https://openapi-generator.tech). DO NOT EDIT. """ -# Specifc FGA header to be parsed +# Specific FGA header to be parsed X_RATELIMIT_LIMIT = "x-ratelimit-limit" X_RATELIMIT_REMAINING = "x_ratelimit_remaining" X_RATELIMIT_RESET = "x_ratelimit_reset" FGA_REQUEST_ID = "fga-request-id" FGA_QUERY_DURATION_MS = "fga-query-duration-ms" OPENFGA_AUTHORIZATION_MODEL_ID = "openfga_authorization_model_id" +RETRY_AFTER = "retry-after" RESPONSE_HEADERS_TO_KEEP = [ X_RATELIMIT_LIMIT, X_RATELIMIT_REMAINING, @@ -24,6 +25,7 @@ FGA_REQUEST_ID, FGA_QUERY_DURATION_MS, OPENFGA_AUTHORIZATION_MODEL_ID, + RETRY_AFTER, ] diff --git a/openfga_sdk/sync/api_client.py b/openfga_sdk/sync/api_client.py index 8333e950..c1fa8600 100644 --- a/openfga_sdk/sync/api_client.py +++ b/openfga_sdk/sync/api_client.py @@ -47,7 +47,6 @@ def random_time(loop_count, min_wait_in_ms) -> float: """ minimum = math.ceil(2**loop_count * min_wait_in_ms) maximum = math.ceil(2 ** (loop_count + 1) * min_wait_in_ms) - return random.randrange(minimum, maximum) / 1000 @@ -253,11 +252,21 @@ def __call_api( ) else 0 ) + max_wait_in_sec = ( + self.configuration.retry_params.max_wait_in_sec + if ( + self.configuration.retry_params is not None + and self.configuration.retry_params.max_wait_in_sec is not None + ) + else 120 + ) if _retry_params is not None: if _retry_params.max_retry is not None: max_retry = _retry_params.max_retry if _retry_params.min_wait_in_ms is not None: max_retry = _retry_params.min_wait_in_ms + if _retry_params.max_wait_in_sec is not None: + max_wait_in_sec = _retry_params.max_wait_in_sec _telemetry_attributes = TelemetryAttributes.fromRequest( user_agent=self.user_agent, @@ -300,8 +309,14 @@ def __call_api( configuration=self.configuration.telemetry, ) - time.sleep(random_time(retry, min_wait_in_ms)) + try: + wait_time_in_sec = self._parse_retry_after_header(e.header) + except ValueError: + wait_time_in_sec = min( + random_time(retry, min_wait_in_ms), max_wait_in_sec + ) + time.sleep(wait_time_in_sec) continue e.body = e.body.decode("utf-8") response_type = response_types_map.get(e.status, None) @@ -395,6 +410,25 @@ def __call_api( else: return (return_data, response_data.status, response_data.headers) + def _parse_retry_after_header(self, headers) -> int: + retry_after_header = headers.get("retry-after") + if not retry_after_header: + raise ValueError("Retry-After header is not present") + + try: + parsed_http_date = self.__deserialize_datetime(retry_after_header).replace( + tzinfo=datetime.timezone.utc + ) + now = datetime.datetime.now(datetime.timezone.utc) + wait_time_in_sec = (parsed_http_date - now).total_seconds() + except ApiException: + wait_time_in_sec = int(retry_after_header) + + if wait_time_in_sec > 1800 or wait_time_in_sec < 1: + raise ValueError("Retry-After header is invalid") + + return math.ceil(wait_time_in_sec) + def sanitize_for_serialization(self, obj): """Builds a JSON POST object. @@ -825,7 +859,7 @@ def __deserialize_datetime(self, string): return parse(string) except ImportError: return string - except ValueError: + except (TypeError, ValueError): raise rest.ApiException( status=0, reason=(f"Failed to parse `{string}` as datetime object"), diff --git a/test/api/open_fga_api_test.py b/test/api/open_fga_api_test.py index b7151963..2c83b530 100644 --- a/test/api/open_fga_api_test.py +++ b/test/api/open_fga_api_test.py @@ -12,7 +12,7 @@ import unittest -from datetime import datetime +from datetime import datetime, timedelta, timezone from unittest import IsolatedAsyncioTestCase from unittest.mock import ANY, patch @@ -98,10 +98,16 @@ # Helper function to construct mock response -def http_mock_response(body, status): - headers = urllib3.response.HTTPHeaderDict( +def http_mock_response(body, status, headers=None): + if headers is None: + headers = {} + + default_headers = urllib3.response.HTTPHeaderDict( {"content-type": "application/json", "Fga-Request-Id": request_id} ) + + headers = {**default_headers, **headers} + return urllib3.HTTPResponse( body.encode("utf-8"), headers, status, preload_content=False ) @@ -1384,6 +1390,235 @@ async def test_429_error_first_error(self, mock_request): mock_request.assert_called() self.assertEqual(mock_request.call_count, 2) + @patch("asyncio.sleep") + @patch.object(rest.RESTClientObject, "request") + async def test_429_error_retry_exponential_backoff(self, mock_request, mock_sleep): + """ + Test to ensure 429 errors are handled properly. + For this case, retry is configured but no Retry-After header is provided. + Should default to exponential backoff with an upper limit. + """ + response_body = '{"allowed": true, "resolution": "1234"}' + error_response_body = """ + { + "code": "rate_limit_exceeded", + "message": "Rate Limit exceeded" + } + """ + mock_request.side_effect = [ + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + mock_response(response_body, 200), + ] + max_wait_in_sec = 1 + retry = openfga_sdk.configuration.RetryParams( + max_retry=9, min_wait_in_ms=10, max_wait_in_sec=max_wait_in_sec + ) + configuration = self.configuration + configuration.store_id = store_id + configuration.retry_params = retry + + async with openfga_sdk.ApiClient(configuration) as api_client: + api_instance = open_fga_api.OpenFgaApi(api_client) + body = CheckRequest( + tuple_key=TupleKey( + object="document:2021-budget", + relation="reader", + user="user:81684243-9356-4421-8fbf-a4f8d36aa31b", + ), + ) + api_response = await api_instance.check( + body=body, + ) + self.assertIsInstance(api_response, CheckResponse) + self.assertTrue(api_response.allowed) + mock_request.assert_called() + self.assertEqual(mock_request.call_count, 9) + self.assertEqual(mock_sleep.call_args[0][0], max_wait_in_sec) + + @patch("asyncio.sleep") + @patch.object(rest.RESTClientObject, "request") + async def test_429_error_retry_configured_unparseable_retry_after( + self, mock_request, mock_sleep + ): + """ + Test to ensure 429 errors are handled properly. + For this case, retry is configured and the Retry-After header is provided as an HTTP date. + """ + response_body = '{"allowed": true, "resolution": "1234"}' + error_response_body = """ + { + "code": "rate_limit_exceeded", + "message": "Rate Limit exceeded" + } + """ + retry_after_in_sec = 5 + five_seconds_from_now = f"{retry_after_in_sec}s" + mock_http_response = http_mock_response( + body=error_response_body, + status=429, + headers={"Retry-After": five_seconds_from_now}, + ) + mock_request.side_effect = [ + RateLimitExceededError(http_resp=mock_http_response), + mock_response(response_body, 200), + ] + + retry = openfga_sdk.configuration.RetryParams( + max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1 + ) + configuration = self.configuration + configuration.store_id = store_id + configuration.retry_params = retry + async with openfga_sdk.ApiClient(configuration) as api_client: + api_instance = open_fga_api.OpenFgaApi(api_client) + body = CheckRequest( + tuple_key=TupleKey( + object="document:2021-budget", + relation="reader", + user="user:81684243-9356-4421-8fbf-a4f8d36aa31b", + ), + ) + api_response = await api_instance.check( + body=body, + ) + self.assertIsInstance(api_response, CheckResponse) + self.assertTrue(api_response.allowed) + mock_request.assert_called() + self.assertEqual(mock_request.call_count, 2) + self.assertNotEqual(mock_sleep.call_args[0][0], retry_after_in_sec) + + @patch("asyncio.sleep") + @patch.object(rest.RESTClientObject, "request") + async def test_429_error_retry_configured_with_http_date( + self, mock_request, mock_sleep + ): + """ + Test to ensure 429 errors are handled properly. + For this case, retry is configured and the Retry-After header is provided as an HTTP date. + """ + response_body = '{"allowed": true, "resolution": "1234"}' + error_response_body = """ + { + "code": "rate_limit_exceeded", + "message": "Rate Limit exceeded" + } + """ + retry_after_in_sec = 5 + five_seconds_from_now = ( + datetime.now(timezone.utc) + timedelta(seconds=retry_after_in_sec) + ).strftime("%a, %d %b %Y %H:%M:%S GMT") + mock_http_response = http_mock_response( + body=error_response_body, + status=429, + headers={"Retry-After": five_seconds_from_now}, + ) + mock_request.side_effect = [ + RateLimitExceededError(http_resp=mock_http_response), + mock_response(response_body, 200), + ] + + retry = openfga_sdk.configuration.RetryParams( + max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1 + ) + configuration = self.configuration + configuration.store_id = store_id + configuration.retry_params = retry + async with openfga_sdk.ApiClient(configuration) as api_client: + api_instance = open_fga_api.OpenFgaApi(api_client) + body = CheckRequest( + tuple_key=TupleKey( + object="document:2021-budget", + relation="reader", + user="user:81684243-9356-4421-8fbf-a4f8d36aa31b", + ), + ) + api_response = await api_instance.check( + body=body, + ) + self.assertIsInstance(api_response, CheckResponse) + self.assertTrue(api_response.allowed) + mock_request.assert_called() + self.assertEqual(mock_request.call_count, 2) + self.assertTrue( + retry_after_in_sec - 1 + <= mock_sleep.call_args[0][0] + <= retry_after_in_sec + ) + + @patch("asyncio.sleep") + @patch.object(rest.RESTClientObject, "request") + async def test_429_error_retry_configured_with_delay_seconds( + self, mock_request, mock_sleep + ): + """ + Test to ensure 429 errors are handled properly. + For this case, retry is configured and the Retry-After header is provided as an HTTP date. + """ + response_body = '{"allowed": true, "resolution": "1234"}' + error_response_body = """ + { + "code": "rate_limit_exceeded", + "message": "Rate Limit exceeded" + } + """ + retry_after_in_sec = 10 + mock_http_response = http_mock_response( + body=error_response_body, + status=429, + headers={"Retry-After": retry_after_in_sec}, + ) + mock_request.side_effect = [ + RateLimitExceededError(http_resp=mock_http_response), + mock_response(response_body, 200), + ] + + retry = openfga_sdk.configuration.RetryParams( + max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1 + ) + configuration = self.configuration + configuration.store_id = store_id + configuration.retry_params = retry + async with openfga_sdk.ApiClient(configuration) as api_client: + api_instance = open_fga_api.OpenFgaApi(api_client) + body = CheckRequest( + tuple_key=TupleKey( + object="document:2021-budget", + relation="reader", + user="user:81684243-9356-4421-8fbf-a4f8d36aa31b", + ), + ) + api_response = await api_instance.check( + body=body, + ) + self.assertIsInstance(api_response, CheckResponse) + self.assertTrue(api_response.allowed) + mock_request.assert_called() + self.assertEqual(mock_request.call_count, 2) + self.assertEqual(mock_sleep.call_args[0][0], retry_after_in_sec) + @patch.object(rest.RESTClientObject, "request") async def test_500_error(self, mock_request): """ diff --git a/test/sync/open_fga_api_test.py b/test/sync/open_fga_api_test.py index 6ad1fe48..9f7dc6e5 100644 --- a/test/sync/open_fga_api_test.py +++ b/test/sync/open_fga_api_test.py @@ -12,7 +12,7 @@ import unittest -from datetime import datetime +from datetime import datetime, timedelta, timezone from unittest import IsolatedAsyncioTestCase from unittest.mock import ANY, patch @@ -99,10 +99,16 @@ # Helper function to construct mock response -def http_mock_response(body, status): - headers = urllib3.response.HTTPHeaderDict( +def http_mock_response(body, status, headers=None): + if headers is None: + headers = {} + + default_headers = urllib3.response.HTTPHeaderDict( {"content-type": "application/json", "Fga-Request-Id": request_id} ) + + headers = {**default_headers, **headers} + return urllib3.HTTPResponse( body.encode("utf-8"), headers, status, preload_content=False ) @@ -1447,6 +1453,231 @@ def test_429_error_first_error(self, mock_request): mock_request.assert_called() self.assertEqual(mock_request.call_count, 2) + @patch("time.sleep") + @patch.object(rest.RESTClientObject, "request") + def test_429_error_retry_exponential_backoff(self, mock_request, mock_sleep): + """ + Test to ensure 429 errors are handled properly. + For this case, retry is configured but no Retry-After header is provided. + Should default to exponential backoff with an upper limit. + """ + response_body = '{"allowed": true, "resolution": "1234"}' + error_response_body = """ + { + "code": "rate_limit_exceeded", + "message": "Rate Limit exceeded" + } + """ + mock_request.side_effect = [ + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + RateLimitExceededError( + http_resp=http_mock_response(error_response_body, 429) + ), + mock_response(response_body, 200), + ] + max_wait_in_sec = 1 + retry = openfga_sdk.configuration.RetryParams( + max_retry=9, min_wait_in_ms=10, max_wait_in_sec=max_wait_in_sec + ) + configuration = self.configuration + configuration.store_id = store_id + configuration.retry_params = retry + + with ApiClient(configuration) as api_client: + api_instance = open_fga_api.OpenFgaApi(api_client) + body = CheckRequest( + tuple_key=TupleKey( + object="document:2021-budget", + relation="reader", + user="user:81684243-9356-4421-8fbf-a4f8d36aa31b", + ), + ) + api_response = api_instance.check( + body=body, + ) + self.assertIsInstance(api_response, CheckResponse) + self.assertTrue(api_response.allowed) + mock_request.assert_called() + self.assertEqual(mock_request.call_count, 9) + self.assertEqual(mock_sleep.call_args[0][0], max_wait_in_sec) + + @patch("time.sleep") + @patch.object(rest.RESTClientObject, "request") + async def test_429_error_retry_configured_unparseable_retry_after( + self, mock_request, mock_sleep + ): + """ + Test to ensure 429 errors are handled properly. + For this case, retry is configured and the Retry-After header is provided as an HTTP date. + """ + response_body = '{"allowed": true, "resolution": "1234"}' + error_response_body = """ + { + "code": "rate_limit_exceeded", + "message": "Rate Limit exceeded" + } + """ + retry_after_in_sec = 5 + five_seconds_from_now = f"{retry_after_in_sec}s" + mock_http_response = http_mock_response( + body=error_response_body, + status=429, + headers={"Retry-After": five_seconds_from_now}, + ) + mock_request.side_effect = [ + RateLimitExceededError(http_resp=mock_http_response), + mock_response(response_body, 200), + ] + + retry = openfga_sdk.configuration.RetryParams( + max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1 + ) + configuration = self.configuration + configuration.store_id = store_id + configuration.retry_params = retry + with ApiClient(configuration) as api_client: + api_instance = open_fga_api.OpenFgaApi(api_client) + body = CheckRequest( + tuple_key=TupleKey( + object="document:2021-budget", + relation="reader", + user="user:81684243-9356-4421-8fbf-a4f8d36aa31b", + ), + ) + api_response = api_instance.check( + body=body, + ) + self.assertIsInstance(api_response, CheckResponse) + self.assertTrue(api_response.allowed) + mock_request.assert_called() + self.assertEqual(mock_request.call_count, 2) + self.assertNotEqual(mock_sleep.call_args[0][0], retry_after_in_sec) + + @patch("time.sleep") + @patch.object(rest.RESTClientObject, "request") + async def test_429_error_retry_configured_with_http_date( + self, mock_request, mock_sleep + ): + """ + Test to ensure 429 errors are handled properly. + For this case, retry is configured and the Retry-After header is provided as an HTTP date. + """ + response_body = '{"allowed": true, "resolution": "1234"}' + error_response_body = """ + { + "code": "rate_limit_exceeded", + "message": "Rate Limit exceeded" + } + """ + retry_after_in_sec = 5 + five_seconds_from_now = ( + datetime.now(timezone.utc) + timedelta(seconds=retry_after_in_sec) + ).strftime("%a, %d %b %Y %H:%M:%S GMT") + mock_http_response = http_mock_response( + body=error_response_body, + status=429, + headers={"Retry-After": five_seconds_from_now}, + ) + mock_request.side_effect = [ + RateLimitExceededError(http_resp=mock_http_response), + mock_response(response_body, 200), + ] + + retry = openfga_sdk.configuration.RetryParams( + max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1 + ) + configuration = self.configuration + configuration.store_id = store_id + configuration.retry_params = retry + with ApiClient(configuration) as api_client: + api_instance = open_fga_api.OpenFgaApi(api_client) + body = CheckRequest( + tuple_key=TupleKey( + object="document:2021-budget", + relation="reader", + user="user:81684243-9356-4421-8fbf-a4f8d36aa31b", + ), + ) + api_response = api_instance.check( + body=body, + ) + self.assertIsInstance(api_response, CheckResponse) + self.assertTrue(api_response.allowed) + mock_request.assert_called() + self.assertEqual(mock_request.call_count, 2) + self.assertEqual(mock_sleep.call_args[0][0], retry_after_in_sec) + + @patch("time.sleep") + @patch.object(rest.RESTClientObject, "request") + async def test_429_error_retry_configured_with_delay_seconds( + self, mock_request, mock_sleep + ): + """ + Test to ensure 429 errors are handled properly. + For this case, retry is configured and the Retry-After header is provided as an HTTP date. + """ + response_body = '{"allowed": true, "resolution": "1234"}' + error_response_body = """ + { + "code": "rate_limit_exceeded", + "message": "Rate Limit exceeded" + } + """ + retry_after_in_sec = 10 + mock_http_response = http_mock_response( + body=error_response_body, + status=429, + headers={"Retry-After": retry_after_in_sec}, + ) + mock_request.side_effect = [ + RateLimitExceededError(http_resp=mock_http_response), + mock_response(response_body, 200), + ] + + retry = openfga_sdk.configuration.RetryParams( + max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1 + ) + configuration = self.configuration + configuration.store_id = store_id + configuration.retry_params = retry + with ApiClient(configuration) as api_client: + api_instance = open_fga_api.OpenFgaApi(api_client) + body = CheckRequest( + tuple_key=TupleKey( + object="document:2021-budget", + relation="reader", + user="user:81684243-9356-4421-8fbf-a4f8d36aa31b", + ), + ) + api_response = api_instance.check( + body=body, + ) + self.assertIsInstance(api_response, CheckResponse) + self.assertTrue(api_response.allowed) + mock_request.assert_called() + self.assertEqual(mock_request.call_count, 2) + self.assertEqual(mock_sleep.call_args[0][0], retry_after_in_sec) + @patch.object(rest.RESTClientObject, "request") def test_500_error(self, mock_request): """