From 753cd99cb3305ba32b013515c5bb50a4971e01b8 Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Tue, 1 Apr 2025 21:03:14 -0600 Subject: [PATCH 01/14] oof --- .github/workflows/run-pytest.yml | 4 +- demos/guessing_game.py | 8 +- demos/multi_guess_client.py | 2 +- demos/multi_guess_server.py | 10 +- {quest_test => history_test}/__init__.py | 0 .../custom_errors/custom_error.py | 0 {quest_test => history_test}/multiguess.py | 0 {quest_test => history_test}/pytest.ini | 0 {quest_test => history_test}/test_alias.py | 28 ++--- {quest_test => history_test}/test_basic.py | 14 +-- .../test_basic_tasks.py | 12 +-- .../test_configuration.py | 8 +- {quest_test => history_test}/test_context.py | 2 +- .../test_external_actions.py | 20 ++-- .../test_get_exception_class.py | 6 +- history_test/test_get_result.py | 100 ++++++++++++++++++ .../test_interruptions.py | 12 +-- {quest_test => history_test}/test_manager.py | 72 ++++++------- .../test_persistence.py | 42 ++++---- .../test_resource_stream.py | 30 +++--- .../test_serializer.py | 8 +- .../test_step_concurrency.py | 14 +-- .../test_step_error.py | 10 +- {quest_test => history_test}/test_suspend.py | 10 +- .../test_versioning.py | 16 +-- .../test_websockets.py | 12 +-- .../test_workflow_metrics.py | 38 +++---- {quest_test => history_test}/test_wrappers.py | 12 +-- history_test/utils.py | 46 ++++++++ main.py | 6 +- pyproject.toml | 4 +- quest_test/test_get_result.py | 100 ------------------ quest_test/utils.py | 46 -------- scratch/ainput.py | 10 +- scratch/inputs.py | 6 +- scratch/print_history.py | 4 +- scratch/scratch.py | 4 +- scratch/serialize_stuff.py | 6 +- scratch/sql_scratch.py | 4 +- src/{quest => history}/__init__.py | 26 ++--- src/{quest/history.py => history/book.py} | 4 +- src/{quest => history}/client.py | 0 src/{quest => history}/context.py | 0 src/{quest => history}/custom_exceptions.py | 0 src/{quest => history}/external.py | 16 +-- src/{quest => history}/extras/__init__.py | 0 src/{quest => history}/extras/aws.py | 22 ++-- src/{quest => history}/extras/sql.py | 12 +-- .../manager.py => history/historian.py} | 78 +++++++------- .../historian.py => history/history.py} | 92 ++++++++-------- .../history_types.py} | 0 src/{quest => history}/lifecycle.py | 8 +- src/{quest => history}/manager_wrappers.py | 6 +- src/{quest => history}/persistence.py | 6 +- src/{quest => history}/resources.py | 6 +- src/{quest => history}/serializer.py | 0 src/{quest => history}/server.py | 14 +-- src/{quest => history}/utils.py | 2 +- src/{quest => history}/versioning.py | 12 +-- src/{quest => history}/wrappers.py | 6 +- 60 files changed, 518 insertions(+), 518 deletions(-) rename {quest_test => history_test}/__init__.py (100%) rename {quest_test => history_test}/custom_errors/custom_error.py (100%) rename {quest_test => history_test}/multiguess.py (100%) rename {quest_test => history_test}/pytest.ini (100%) rename {quest_test => history_test}/test_alias.py (81%) rename {quest_test => history_test}/test_basic.py (93%) rename {quest_test => history_test}/test_basic_tasks.py (90%) rename {quest_test => history_test}/test_configuration.py (89%) rename {quest_test => history_test}/test_context.py (97%) rename {quest_test => history_test}/test_external_actions.py (94%) rename {quest_test => history_test}/test_get_exception_class.py (81%) create mode 100644 history_test/test_get_result.py rename {quest_test => history_test}/test_interruptions.py (71%) rename {quest_test => history_test}/test_manager.py (63%) rename {quest_test => history_test}/test_persistence.py (81%) rename {quest_test => history_test}/test_resource_stream.py (94%) rename {quest_test => history_test}/test_serializer.py (86%) rename {quest_test => history_test}/test_step_concurrency.py (90%) rename {quest_test => history_test}/test_step_error.py (93%) rename {quest_test => history_test}/test_suspend.py (87%) rename {quest_test => history_test}/test_versioning.py (92%) rename {quest_test => history_test}/test_websockets.py (83%) rename {quest_test => history_test}/test_workflow_metrics.py (60%) rename {quest_test => history_test}/test_wrappers.py (83%) create mode 100644 history_test/utils.py delete mode 100644 quest_test/test_get_result.py delete mode 100644 quest_test/utils.py rename src/{quest => history}/__init__.py (51%) rename src/{quest/history.py => history/book.py} (74%) rename src/{quest => history}/client.py (100%) rename src/{quest => history}/context.py (100%) rename src/{quest => history}/custom_exceptions.py (100%) rename src/{quest => history}/external.py (91%) rename src/{quest => history}/extras/__init__.py (100%) rename src/{quest => history}/extras/aws.py (90%) rename src/{quest => history}/extras/sql.py (87%) rename src/{quest/manager.py => history/historian.py} (83%) rename src/{quest/historian.py => history/history.py} (91%) rename src/{quest/quest_types.py => history/history_types.py} (100%) rename src/{quest => history}/lifecycle.py (90%) rename src/{quest => history}/manager_wrappers.py (80%) rename src/{quest => history}/persistence.py (96%) rename src/{quest => history}/resources.py (96%) rename src/{quest => history}/serializer.py (100%) rename src/{quest => history}/server.py (90%) rename src/{quest => history}/utils.py (95%) rename src/{quest => history}/versioning.py (80%) rename src/{quest => history}/wrappers.py (89%) diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index c2ce5694..40b6a112 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -47,7 +47,7 @@ jobs: env: PYTHONPATH: ${{ github.workspace }}/src run: | - pytest quest_test -o log_cli=true + pytest history_test -o log_cli=true - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v3 @@ -62,4 +62,4 @@ jobs: env: PYTHONPATH: ${{ github.workspace }}/src run: | - pytest quest_test -m "integration" -o log_cli=true \ No newline at end of file + pytest history_test -m "integration" -o log_cli=true \ No newline at end of file diff --git a/demos/guessing_game.py b/demos/guessing_game.py index f4c0af09..72954d71 100644 --- a/demos/guessing_game.py +++ b/demos/guessing_game.py @@ -4,7 +4,7 @@ import sys from pathlib import Path -from quest import step, create_filesystem_manager, ainput +from history import step, create_filesystem_historian, ainput @step async def pick_number(lower, upper): @@ -38,13 +38,13 @@ async def guessing_game(): async def main(): state = Path('state') # state.rmdir() - async with create_filesystem_manager( + async with create_filesystem_historian( state, 'guess_game_demo', lambda wid: guessing_game ) as manager: - if not manager.has_workflow('demo'): - manager.start_workflow( + if not manager.has('demo'): + manager.start_soon( '', f'demo' ) diff --git a/demos/multi_guess_client.py b/demos/multi_guess_client.py index de8e23b0..b3a73738 100644 --- a/demos/multi_guess_client.py +++ b/demos/multi_guess_client.py @@ -1,6 +1,6 @@ import asyncio -from quest import client, step +from history import client, step # TODO - write a websocket client diff --git a/demos/multi_guess_server.py b/demos/multi_guess_server.py index 9c504aad..decc7668 100644 --- a/demos/multi_guess_server.py +++ b/demos/multi_guess_server.py @@ -2,9 +2,9 @@ import random from pathlib import Path -from quest import (step, queue, state, identity_queue, - create_filesystem_manager, these) -from quest.server import Server +from history import (step, queue, state, identity_queue, + create_filesystem_historian, these) +from history.server import Server @step @@ -100,7 +100,7 @@ async def multi_guess(): async def main(): async with ( - create_filesystem_manager( + create_filesystem_historian( Path('state'), 'multi_guess', lambda wid: multi_guess @@ -113,7 +113,7 @@ async def main(): ): # TODO: Add ability to start workflows to server.py # Start the game - manager.start_workflow('', 'demo') + manager.start_soon('', 'demo') # Wait for it to finish await manager.get_workflow('demo') diff --git a/quest_test/__init__.py b/history_test/__init__.py similarity index 100% rename from quest_test/__init__.py rename to history_test/__init__.py diff --git a/quest_test/custom_errors/custom_error.py b/history_test/custom_errors/custom_error.py similarity index 100% rename from quest_test/custom_errors/custom_error.py rename to history_test/custom_errors/custom_error.py diff --git a/quest_test/multiguess.py b/history_test/multiguess.py similarity index 100% rename from quest_test/multiguess.py rename to history_test/multiguess.py diff --git a/quest_test/pytest.ini b/history_test/pytest.ini similarity index 100% rename from quest_test/pytest.ini rename to history_test/pytest.ini diff --git a/quest_test/test_alias.py b/history_test/test_alias.py similarity index 81% rename from quest_test/test_alias.py rename to history_test/test_alias.py index 29fbc6c5..58f7070d 100644 --- a/quest_test/test_alias.py +++ b/history_test/test_alias.py @@ -1,9 +1,9 @@ import asyncio import pytest -from quest.manager import DuplicateAliasException -from quest import queue, alias -from .utils import timeout, create_in_memory_workflow_manager +from history.historian import DuplicateAliasException +from history import queue, alias +from .utils import timeout, create_in_memory_historian @pytest.mark.asyncio @@ -27,8 +27,8 @@ async def workflow(): 'workflow': workflow } - async with create_in_memory_workflow_manager(workflows) as manager: - manager.start_workflow('workflow', 'wid') + async with create_in_memory_historian(workflows) as manager: + manager.start_soon('workflow', 'wid') await asyncio.sleep(0.1) await manager.send_event('wid', 'data', None, 'put', '1') @@ -83,11 +83,11 @@ async def workflow_b(): 'workflow_b': workflow_b, } - async with create_in_memory_workflow_manager(workflows) as manager: + async with create_in_memory_historian(workflows) as manager: # Gather resources - manager.start_workflow('workflow_a', 'wid_a') + manager.start_soon('workflow_a', 'wid_a') await asyncio.sleep(0.1) - manager.start_workflow('workflow_b', 'wid_b') + manager.start_soon('workflow_b', 'wid_b') await asyncio.sleep(0.1) first_pause.set() @@ -97,7 +97,7 @@ async def workflow_b(): await asyncio.sleep(0.1) # yield to the workflows # now both should be waiting on second gate and no one should be the foo - assert not manager.has_workflow('the_foo') + assert not manager.has('the_foo') assert 'data a 1' in data_a assert 'data foo 1' in data_a assert 'data b 1' in data_b @@ -138,13 +138,13 @@ async def workflow_b(): 'workflow_a': workflow_a, 'workflow_b': workflow_b, } - async with create_in_memory_workflow_manager(workflows) as manager: - manager.start_workflow('workflow_a', 'wid1', delete_on_finish=False) - manager.start_workflow('workflow_b', 'wid2', delete_on_finish=False) + async with create_in_memory_historian(workflows) as manager: + manager.start_soon('workflow_a', 'wid1', delete_on_finish=False) + manager.start_soon('workflow_b', 'wid2', delete_on_finish=False) await asyncio.sleep(0.1) pause.set() await asyncio.sleep(0.1) # Allow workflows to finish - result_wid1 = await manager.get_workflow_result('wid1', delete=True) - result_wid2 = await manager.get_workflow_result('wid2', delete=True) + result_wid1 = await manager.get_result('wid1', delete=True) + result_wid2 = await manager.get_result('wid2', delete=True) diff --git a/quest_test/test_basic.py b/history_test/test_basic.py similarity index 93% rename from quest_test/test_basic.py rename to history_test/test_basic.py index 6b1d23c2..4209f7b3 100644 --- a/quest_test/test_basic.py +++ b/history_test/test_basic.py @@ -3,9 +3,9 @@ import pytest from .utils import timeout -from quest import step -from quest.historian import Historian -from quest.serializer import NoopSerializer +from history import step +from history.history import History +from history.serializer import NoopSerializer # @@ -27,7 +27,7 @@ async def workflow(name): @timeout(3) async def test_basic_workflow(): history = [] - historian = Historian( + historian = History( 'test', workflow, history, @@ -76,7 +76,7 @@ async def longer_workflow(text): @timeout(3) async def test_resume(): history = [] - historian = Historian( + historian = History( 'test', longer_workflow, history, @@ -135,7 +135,7 @@ async def nested_workflow(text1, text2): @timeout(3) async def test_nested_steps_resume(): history = [] - historian = Historian( + historian = History( 'test', nested_workflow, history, @@ -170,7 +170,7 @@ async def dance(start): @pytest.mark.asyncio @timeout(3) async def test_resume_mid_step(): - historian = Historian( + historian = History( 'test', dance, [], diff --git a/quest_test/test_basic_tasks.py b/history_test/test_basic_tasks.py similarity index 90% rename from quest_test/test_basic_tasks.py rename to history_test/test_basic_tasks.py index 74eb45db..507d6431 100644 --- a/quest_test/test_basic_tasks.py +++ b/history_test/test_basic_tasks.py @@ -1,10 +1,10 @@ import asyncio import pytest -from quest import step -from quest.historian import Historian -from quest.wrappers import task -from quest.serializer import NoopSerializer +from history import step +from history.history import History +from history.wrappers import task +from history.serializer import NoopSerializer from .utils import timeout counters = {} @@ -44,7 +44,7 @@ async def test_basic_tasks(): pauses['basic_tasks'] = asyncio.Event() history = [] - historian = Historian( + historian = History( 'test', sub_task_workflow, history, @@ -68,7 +68,7 @@ async def test_basic_tasks_resume(): pauses['tasks_resume'] = asyncio.Event() history = [] - historian = Historian( + historian = History( 'test', sub_task_workflow, history, diff --git a/quest_test/test_configuration.py b/history_test/test_configuration.py similarity index 89% rename from quest_test/test_configuration.py rename to history_test/test_configuration.py index bdb0abf3..90147b9b 100644 --- a/quest_test/test_configuration.py +++ b/history_test/test_configuration.py @@ -3,8 +3,8 @@ import pytest -from quest import Historian, queue, step -from quest.serializer import NoopSerializer +from history import History, queue, step +from history.serializer import NoopSerializer @pytest.mark.asyncio @@ -51,7 +51,7 @@ async def configure_state(config): } history = [] - historian = Historian('test', application, history, serializer=NoopSerializer()) + historian = History('test', application, history, serializer=NoopSerializer()) historian.configure(configure_state, config1) historian.run() await asyncio.sleep(0.1) @@ -70,7 +70,7 @@ async def configure_state(config): 'foos': ['a'] } - historian = Historian('test', application, history, serializer=NoopSerializer()) + historian = History('test', application, history, serializer=NoopSerializer()) historian.configure(configure_state, config1) historian.configure(configure_state, config2) historian.run() diff --git a/quest_test/test_context.py b/history_test/test_context.py similarity index 97% rename from quest_test/test_context.py rename to history_test/test_context.py index c4aa5f5a..64f9e1aa 100644 --- a/quest_test/test_context.py +++ b/history_test/test_context.py @@ -1,6 +1,6 @@ """import pytest -from src.quest import these +from src.history import these class Context: diff --git a/quest_test/test_external_actions.py b/history_test/test_external_actions.py similarity index 94% rename from quest_test/test_external_actions.py rename to history_test/test_external_actions.py index 59247cd0..fa9e0678 100644 --- a/quest_test/test_external_actions.py +++ b/history_test/test_external_actions.py @@ -2,10 +2,10 @@ import pytest -from quest.external import state, queue, event, wrap_as_state, wrap_as_queue -from quest.historian import Historian -from quest.wrappers import task, step -from quest.serializer import NoopSerializer +from history.external import state, queue, event, wrap_as_state, wrap_as_queue +from history.history import History +from history.wrappers import task, step +from history.serializer import NoopSerializer from .utils import timeout @@ -49,7 +49,7 @@ async def state_workflow(identity): assert await name.get() == 'Barbaz' identity = 'foo_ident' - historian = Historian('test', state_workflow, [], serializer=NoopSerializer()) + historian = History('test', state_workflow, [], serializer=NoopSerializer()) workflow = historian.run(identity) await wait_for(historian) @@ -88,7 +88,7 @@ async def workflow_with_queue(identity): @timeout(3) async def test_external_queue(): identity = 'foo_ident' - historian = Historian( + historian = History( 'test', workflow_with_queue, [], @@ -139,7 +139,7 @@ async def queue_task_workflow(id1, id2): async def test_queue_tasks(): id_foo = 'FOO' id_bar = 'BAR' - historian = Historian( + historian = History( 'test', queue_task_workflow, [], @@ -190,7 +190,7 @@ async def workflow_nested_tasks(): @pytest.mark.asyncio @timeout(3) async def test_nested_tasks(): - historian = Historian( + historian = History( 'test', workflow_nested_tasks, [], @@ -220,7 +220,7 @@ async def test_queue_tasks_resume(): id_foo = 'FOO' id_bar = 'BAR' history = [] - historian = Historian( + historian = History( 'test', queue_task_workflow, history, @@ -287,7 +287,7 @@ async def test_step_specific_external(): then the external event on the now-obsolete resource must also be pruned. """ history = [] - historian = Historian('test', interactive_process_with_steps, history, serializer=NoopSerializer()) + historian = History('test', interactive_process_with_steps, history, serializer=NoopSerializer()) historian.run() await asyncio.sleep(0.1) resources = await historian.get_resources(None) diff --git a/quest_test/test_get_exception_class.py b/history_test/test_get_exception_class.py similarity index 81% rename from quest_test/test_get_exception_class.py rename to history_test/test_get_exception_class.py index 690c0f69..69af0230 100644 --- a/quest_test/test_get_exception_class.py +++ b/history_test/test_get_exception_class.py @@ -1,6 +1,6 @@ import pytest -from quest.utils import get_exception_class +from history.utils import get_exception_class def test_standard_exceptions(): @@ -15,14 +15,14 @@ def test_standard_exceptions(): def test_custom_exceptions(): - exception_class = get_exception_class('quest_test.custom_errors.custom_error.MyError') + exception_class = get_exception_class('history_test.custom_errors.custom_error.MyError') with pytest.raises(exception_class) as exc_info: raise exception_class("This is a custom error message") assert str(exc_info.value) == "This is a custom error message" def test_weird_custom_exception(): - exception_class = get_exception_class('quest_test.custom_errors.custom_error.WeirdCustomException') + exception_class = get_exception_class('history_test.custom_errors.custom_error.WeirdCustomException') with pytest.raises(exception_class) as exc_info: raise exception_class("This is a weird custom error", 404) assert str(exc_info.value) == "This is a weird custom error (Error Code: 404)" diff --git a/history_test/test_get_result.py b/history_test/test_get_result.py new file mode 100644 index 00000000..b85ab398 --- /dev/null +++ b/history_test/test_get_result.py @@ -0,0 +1,100 @@ +import asyncio + +import pytest + +from history.historian import WorkflowNotFound +from .utils import timeout, create_in_memory_historian + + +class OurException(Exception): + pass + + +@pytest.mark.asyncio +@timeout(3) +async def test_basic_store_result(): + async def workflow1(): + return "done" + + async with create_in_memory_historian({'w1': workflow1}) as manager: + manager.start_soon('w1', 'wid1', delete_on_finish=False) + await asyncio.sleep(0.1) + + assert await manager.get_result('wid1') == 'done' + assert manager.has('wid1') + assert await manager.get_result('wid1', delete=True) == 'done' + assert not manager.has('wid1') + + +@pytest.mark.asyncio +@timeout(3) +async def test_workflows_not_saved_have_no_results(): + async def workflow1(): + return "done" + + async with create_in_memory_historian({'w1': workflow1}) as manager: + manager.start_soon('w1', 'wid1') + await asyncio.sleep(0.1) + assert not manager.has('wid1') + + +@pytest.mark.asyncio +@timeout(3) +async def test_get_result_on_missing_workflow_raises(): + async def workflow1(): + return "done" + + async with create_in_memory_historian({'w1': workflow1}) as manager: + manager.start_soon('w1', 'wid1') + await asyncio.sleep(0.1) + assert not manager.has('wid1') + + with pytest.raises(WorkflowNotFound): + await manager.get_result('wid1') + + +@pytest.mark.asyncio +@timeout(6) +async def test_get_result_on_running_workflow(): + gate = asyncio.Event() + + async def sample_workflow(): + await gate.wait() + return "sample workflow result" + + workflows = { + "sample_workflow": sample_workflow + } + manager = create_in_memory_historian(workflows=workflows) + + async with manager: + manager.start_soon('sample_workflow', 'wid1') + await asyncio.sleep(0.1) + + get_result_task = asyncio.create_task(manager.get_result('wid1')) + + gate.set() + result = await get_result_task + assert result is not None + +@pytest.mark.asyncio +@timeout(3) +async def test_exception_store_result(): + async def workflow1(): + raise OurException('died') + + async with create_in_memory_historian({'w1': workflow1}) as manager: + manager.start_soon('w1', 'wid1', delete_on_finish=False) + await asyncio.sleep(0.1) + + assert manager.has('wid1') + + with pytest.raises(OurException): + await manager.get_result('wid1') + + assert manager.has('wid1') + + with pytest.raises(OurException): + await manager.get_result('wid1', delete=True) + + assert not manager.has('wid1') diff --git a/quest_test/test_interruptions.py b/history_test/test_interruptions.py similarity index 71% rename from quest_test/test_interruptions.py rename to history_test/test_interruptions.py index 4adc5819..03c94fa4 100644 --- a/quest_test/test_interruptions.py +++ b/history_test/test_interruptions.py @@ -4,7 +4,7 @@ from asyncio import CancelledError import pytest -from quest_test.utils import create_in_memory_workflow_manager +from history_test.utils import create_in_memory_historian gate_1 = asyncio.Event() gate_2 = asyncio.Event() @@ -33,23 +33,23 @@ async def workflow_2(counter_2): 'workflow_1': workflow_1, 'workflow_2': workflow_2, } - manager = create_in_memory_workflow_manager(workflows) + manager = create_in_memory_historian(workflows) counter_1 = [0] counter_2 = [0] async with manager: - manager.start_workflow('workflow_1', 'w1', counter_1, delete_on_finish=False) - manager.start_workflow('workflow_2', 'w2', counter_2, delete_on_finish=False) + manager.start_soon('workflow_1', 'w1', counter_1, delete_on_finish=False) + manager.start_soon('workflow_2', 'w2', counter_2, delete_on_finish=False) gate_1.set() await asyncio.sleep(0.1) with pytest.raises(CancelledError): - await manager.get_workflow_result("w1") + await manager.get_result("w1") with pytest.raises(CancelledError): - await manager.get_workflow_result("w2") + await manager.get_result("w2") assert counter_1[0] == 3 assert counter_2[0] == 3 diff --git a/quest_test/test_manager.py b/history_test/test_manager.py similarity index 63% rename from quest_test/test_manager.py rename to history_test/test_manager.py index 8baafdf5..20a44465 100644 --- a/quest_test/test_manager.py +++ b/history_test/test_manager.py @@ -1,11 +1,11 @@ import asyncio -from quest.utils import quest_logger +from history.utils import history_logger import pytest -from quest import PersistentHistory, queue, state, event -from quest.manager import WorkflowManager -from quest.persistence import InMemoryBlobStorage -from quest.serializer import NoopSerializer +from history import PersistentList, queue, state, event +from history.historian import Historian +from history.persistence import InMemoryBlobStorage +from history.serializer import NoopSerializer @pytest.mark.asyncio @@ -15,7 +15,7 @@ async def test_manager(): def create_history(wid: str): if wid not in histories: - histories[wid] = PersistentHistory(wid, InMemoryBlobStorage()) + histories[wid] = PersistentList(wid, InMemoryBlobStorage()) return histories[wid] pause = asyncio.Event() @@ -25,18 +25,18 @@ def create_history(wid: str): async def workflow(arg): nonlocal counter_a, counter_b - quest_logger.info('workflow started') + history_logger.info('workflow started') counter_a += 1 await pause.wait() - quest_logger.info('workflow passed pause') + history_logger.info('workflow passed pause') counter_b += 1 return 7 + arg - async with WorkflowManager('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: - manager.start_workflow('workflow', 'wid1', 4, delete_on_finish=False) + async with Historian('test-manager', storage, create_history, lambda w_type: workflow, + serializer=NoopSerializer()) as manager: + manager.start_soon('workflow', 'wid1', 4, delete_on_finish=False) await asyncio.sleep(0.1) # Now pause the manager and all workflows @@ -44,12 +44,12 @@ async def workflow(arg): assert counter_a == 1 assert counter_b == 0 - async with WorkflowManager('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: + async with Historian('test-manager', storage, create_history, lambda w_type: workflow, + serializer=NoopSerializer()) as manager: # At this point, all workflows should be resumed pause.set() await asyncio.sleep(0.1) - result = await manager.get_workflow_result('wid1') + result = await manager.get_result('wid1') assert result == 11 assert counter_a == 2 @@ -63,7 +63,7 @@ async def test_manager_events(): def create_history(wid: str): if wid not in histories: - histories[wid] = PersistentHistory(wid, InMemoryBlobStorage()) + histories[wid] = PersistentList(wid, InMemoryBlobStorage()) return histories[wid] counter_a = 0 @@ -73,13 +73,13 @@ async def workflow(arg: int): nonlocal counter_a, counter_b total = arg - quest_logger.info('workflow started') + history_logger.info('workflow started') counter_a += 1 async with queue('messages', None) as Q: while True: message = await Q.get() - quest_logger.info(f'message received: {message}') + history_logger.info(f'message received: {message}') counter_b += 1 if message == 0: @@ -87,9 +87,9 @@ async def workflow(arg: int): total += message - async with WorkflowManager('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: - manager.start_workflow('workflow', 'wid1', 1, delete_on_finish=False) + async with Historian('test-manager', storage, create_history, lambda w_type: workflow, + serializer=NoopSerializer()) as manager: + manager.start_soon('workflow', 'wid1', 1, delete_on_finish=False) await asyncio.sleep(0.1) await manager.send_event('wid1', 'messages', None, 'put', 2) await asyncio.sleep(0.1) @@ -99,13 +99,13 @@ async def workflow(arg: int): assert counter_a == 1 assert counter_b == 1 - async with WorkflowManager('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: + async with Historian('test-manager', storage, create_history, lambda w_type: workflow, + serializer=NoopSerializer()) as manager: # At this point, all workflows should be resumed await asyncio.sleep(0.1) await manager.send_event('wid1', 'messages', None, 'put', 3) await manager.send_event('wid1', 'messages', None, 'put', 0) # i.e. end the workflow - result = await manager.get_workflow_result('wid1') + result = await manager.get_result('wid1') assert result == 6 assert counter_a == 2 @@ -119,7 +119,7 @@ async def test_manager_background(): def create_history(wid: str): if wid not in histories: - histories[wid] = PersistentHistory(wid, InMemoryBlobStorage()) + histories[wid] = PersistentList(wid, InMemoryBlobStorage()) return histories[wid] counter_a = 0 @@ -130,13 +130,13 @@ async def workflow(arg: int): nonlocal counter_a, counter_b, total total = arg - quest_logger.info('workflow started') + history_logger.info('workflow started') counter_a += 1 async with queue('messages', None) as Q: while True: message = await Q.get() - quest_logger.info(f'message received: {message}') + history_logger.info(f'message received: {message}') counter_b += 1 if message == 0: @@ -144,9 +144,9 @@ async def workflow(arg: int): total += message - async with WorkflowManager('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: - manager.start_workflow('workflow', 'wid1', 1) + async with Historian('test-manager', storage, create_history, lambda w_type: workflow, + serializer=NoopSerializer()) as manager: + manager.start_soon('workflow', 'wid1', 1) await asyncio.sleep(0.1) await manager.send_event('wid1', 'messages', None, 'put', 2) await asyncio.sleep(0.1) @@ -156,14 +156,14 @@ async def workflow(arg: int): assert counter_a == 1 assert counter_b == 1 - async with WorkflowManager('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: + async with Historian('test-manager', storage, create_history, lambda w_type: workflow, + serializer=NoopSerializer()) as manager: # At this point, all workflows should be resumed await asyncio.sleep(0.1) await manager.send_event('wid1', 'messages', None, 'put', 3) await manager.send_event('wid1', 'messages', None, 'put', 0) # i.e. end the workflow await asyncio.sleep(0.1) # workflow now finishes and removes itself - assert not manager.has_workflow('wid1') + assert not manager.has('wid1') assert total == 6 assert counter_a == 2 @@ -184,11 +184,11 @@ async def workflow(): storage = InMemoryBlobStorage() def create_history(wid: str): - return PersistentHistory(wid, InMemoryBlobStorage()) + return PersistentList(wid, InMemoryBlobStorage()) - async with WorkflowManager('test', storage, create_history, lambda wid: workflow, - serializer=NoopSerializer()) as wm: - wm.start_workflow('workflow', 'wid', delete_on_finish=False) + async with Historian('test', storage, create_history, lambda wid: workflow, + serializer=NoopSerializer()) as wm: + wm.start_soon('workflow', 'wid', delete_on_finish=False) await asyncio.sleep(0.1) q = await wm.get_queue('wid', 'messages', None) result = await wm.get_state('wid', 'result', None) diff --git a/quest_test/test_persistence.py b/history_test/test_persistence.py similarity index 81% rename from quest_test/test_persistence.py rename to history_test/test_persistence.py index 96cac7c0..ac5c1e89 100644 --- a/quest_test/test_persistence.py +++ b/history_test/test_persistence.py @@ -5,10 +5,10 @@ import pytest -from quest import step -from quest.historian import Historian -from quest.serializer import NoopSerializer -from quest.persistence import PersistentHistory, LocalFileSystemBlobStorage +from history import step +from history.history import History +from history.serializer import NoopSerializer +from history.persistence import PersistentList, LocalFileSystemBlobStorage from .utils import timeout @@ -37,7 +37,7 @@ def __enter__(self): return storage def __exit__(self, *args): - return self.tmp_dir.__exit__(*args) + TemporaryDirectory().__exit__(*args) await persistence_basic(FileSystemStorageContext()) await resume_step_persistence(FileSystemStorageContext()) @@ -46,7 +46,7 @@ def __exit__(self, *args): @pytest.mark.asyncio @timeout(3) async def test_persistence_sql(): - from quest.extras.sql import SQLDatabase, SqlBlobStorage + from history.extras.sql import SQLDatabase, SqlBlobStorage class SqlStorageContext: def __enter__(self): @@ -66,7 +66,7 @@ def __exit__(self, *args): @timeout(6) @pytest.mark.integration async def test_persistence_aws(): - from quest.extras.aws import S3BlobStorage, S3Bucket, DynamoDB, DynamoDBBlobStorage + from history.extras.aws import S3BlobStorage, S3Bucket, DynamoDB, DynamoDBBlobStorage class DynamoDBStorageContext: def __enter__(self): @@ -95,8 +95,8 @@ def __exit__(self, *args): async def persistence_basic(storage_ctx): with storage_ctx as storage: - history = PersistentHistory('test', storage) - historian = Historian( + history = PersistentList('test', storage) + historian = History( 'test', simple_workflow, history, @@ -110,8 +110,8 @@ async def persistence_basic(storage_ctx): pause.set() with storage_ctx as storage: - history = PersistentHistory('test', storage) - historian = Historian( + history = PersistentList('test', storage) + historian = History( 'test', simple_workflow, history, @@ -136,8 +136,8 @@ async def resume_this_workflow(): async def resume_step_persistence(storage_ctx): with storage_ctx as storage: - history = PersistentHistory('test', storage) - historian = Historian( + history = PersistentList('test', storage) + historian = History( 'test', resume_this_workflow, history, @@ -151,8 +151,8 @@ async def resume_step_persistence(storage_ctx): event.set() with storage_ctx as storage: - history = PersistentHistory('test', storage) - historian = Historian( + history = PersistentList('test', storage) + historian = History( 'test', resume_this_workflow, history, @@ -165,8 +165,8 @@ async def resume_step_persistence(storage_ctx): @pytest.mark.asyncio async def test_workflow_cleanup_suspend(tmp_path): storage = LocalFileSystemBlobStorage(tmp_path) - history = PersistentHistory('test', storage) - historian = Historian( + history = PersistentList('test', storage) + historian = History( 'test', resume_this_workflow, history, @@ -180,8 +180,8 @@ async def test_workflow_cleanup_suspend(tmp_path): event.set() storage = LocalFileSystemBlobStorage(tmp_path) - history = PersistentHistory('test', storage) - historian = Historian( + history = PersistentList('test', storage) + historian = History( 'test', resume_this_workflow, history, @@ -196,8 +196,8 @@ async def test_workflow_cleanup_suspend(tmp_path): @pytest.mark.asyncio async def test_workflow_cleanup_basic(tmp_path): storage = LocalFileSystemBlobStorage(tmp_path) - history = PersistentHistory('test', storage) - historian = Historian( + history = PersistentList('test', storage) + historian = History( 'test', simple_workflow, history, diff --git a/quest_test/test_resource_stream.py b/history_test/test_resource_stream.py similarity index 94% rename from quest_test/test_resource_stream.py rename to history_test/test_resource_stream.py index 998dab27..dfdfe255 100644 --- a/quest_test/test_resource_stream.py +++ b/history_test/test_resource_stream.py @@ -1,10 +1,10 @@ import asyncio import pytest -from quest import Historian -from quest.external import state, queue, event, identity_queue, wrap_as_state, wrap_as_queue, wrap_as_identity_queue, \ +from history import History +from history.external import state, queue, event, identity_queue, wrap_as_state, wrap_as_queue, wrap_as_identity_queue, \ wrap_as_event -from .utils import timeout, create_test_historian +from .utils import timeout, create_test_history # A general-use workflow for these tests async def simple_workflow(phrase1_ident, phrase2_ident): @@ -33,7 +33,7 @@ class StreamListenerError(Exception): pass # A listener that fails streaming before the workflow completes -async def failing_listener(historian: Historian, identity): +async def failing_listener(historian: History, identity): try: with historian.get_resource_stream(identity) as resource_stream: i = 0 @@ -62,7 +62,7 @@ async def default_workflow(): async with event('gate', None) as gate: await gate.wait() - historian = create_test_historian( + historian = create_test_history( 'default', default_workflow ) @@ -138,7 +138,7 @@ async def default_workflow(): @pytest.mark.asyncio @timeout(3) async def test_typical(): - historian = create_test_historian( + historian = create_test_history( 'typical', lambda: simple_workflow(None, None) ) @@ -169,7 +169,7 @@ async def typical_listener(): @pytest.mark.asyncio @timeout(3) async def test_private_identity_streaming_public_resources(): - historian = create_test_historian( + historian = create_test_history( 'private_identity_streaming_public_resources', lambda: simple_workflow(None, None) ) @@ -183,7 +183,7 @@ async def test_private_identity_streaming_public_resources(): @pytest.mark.asyncio @timeout(3) async def test_public_streaming_private_resources(): - historian = create_test_historian( + historian = create_test_history( 'public_streaming_private_resources', lambda: simple_workflow('private_identity', 'private_identity') ) @@ -199,7 +199,7 @@ async def test_public_streaming_private_resources(): @pytest.mark.asyncio @timeout(3) async def test_exception(): - historian = create_test_historian( + historian = create_test_history( 'exception', lambda: simple_workflow(None, None) ) @@ -217,7 +217,7 @@ async def test_exception(): @pytest.mark.asyncio @timeout(3) async def test_concurrent_none_streams(): - historian = create_test_historian( + historian = create_test_history( 'concurrent_none_streams', lambda: simple_workflow(None, None) ) @@ -249,7 +249,7 @@ async def public_listener(): if phrase1_fail: assert False - historian = create_test_historian( + historian = create_test_history( 'mult_identity_workflow', lambda: simple_workflow(None, 'private_identity') ) @@ -293,7 +293,7 @@ async def ident2_listener(): if ident1_fail or ident2_fail: assert False - historian = create_test_historian( + historian = create_test_history( 'different_identity_streams', lambda: simple_workflow('ident1', 'ident2') ) @@ -309,7 +309,7 @@ async def ident2_listener(): @pytest.mark.asyncio @timeout(3) async def test_closing_different_identity_streams(): - historian = create_test_historian( + historian = create_test_history( 'different_identity_streams', lambda: simple_workflow(None, 'private_identity') ) @@ -327,7 +327,7 @@ async def test_closing_different_identity_streams(): @pytest.mark.asyncio @timeout(4) async def test_suspend_workflow(): - historian = create_test_historian( + historian = create_test_history( 'test_suspend_workflow', lambda: simple_workflow(None, None) ) @@ -355,7 +355,7 @@ async def test_suspend_workflow(): @pytest.mark.asyncio @timeout(4) async def test_suspend_resume_workflow(): - historian = create_test_historian( + historian = create_test_history( 'test_suspend_resume_workflow', lambda: simple_workflow(None, None) ) diff --git a/quest_test/test_serializer.py b/history_test/test_serializer.py similarity index 86% rename from quest_test/test_serializer.py rename to history_test/test_serializer.py index 5af60999..956d3e74 100644 --- a/quest_test/test_serializer.py +++ b/history_test/test_serializer.py @@ -3,9 +3,9 @@ import pytest from typing import TypeVar -from quest.historian import Historian -from quest.serializer import MasterSerializer, TypeSerializer -from quest.wrappers import step +from history.history import History +from history.serializer import MasterSerializer, TypeSerializer +from history.wrappers import step T = TypeVar('T') @@ -53,7 +53,7 @@ async def workflow(): async def test_master_serializer(): history = [] # Create historian with custom serializer - historian = Historian('test_workflow', workflow, history, serializer=serializer) + historian = History('test_workflow', workflow, history, serializer=serializer) workflow_task = historian.run() diff --git a/quest_test/test_step_concurrency.py b/history_test/test_step_concurrency.py similarity index 90% rename from quest_test/test_step_concurrency.py rename to history_test/test_step_concurrency.py index 17a3bc1d..e30f4770 100644 --- a/quest_test/test_step_concurrency.py +++ b/history_test/test_step_concurrency.py @@ -2,10 +2,10 @@ import pytest -from quest import step -from quest.historian import Historian -from quest.wrappers import task -from quest.serializer import NoopSerializer +from history import step +from history.history import History +from history.wrappers import task +from history.serializer import NoopSerializer from .utils import timeout @@ -30,7 +30,7 @@ async def fooflow(text1, text2): @pytest.mark.asyncio async def test_step_concurrency(): - historian = Historian( + historian = History( 'test', fooflow, [], @@ -55,7 +55,7 @@ async def doubleflow(text): @pytest.mark.asyncio @timeout(3) async def test_step_tasks(): - historian = Historian( + historian = History( 'test', doubleflow, [], @@ -124,7 +124,7 @@ async def long_fast_race(): @timeout(3) async def test_long_fast_race(): records = [] - history = Historian('test', long_fast_race, records, serializer=NoopSerializer()) + history = History('test', long_fast_race, records, serializer=NoopSerializer()) workflow = history.run() await asyncio.sleep(1) await history.suspend() diff --git a/quest_test/test_step_error.py b/history_test/test_step_error.py similarity index 93% rename from quest_test/test_step_error.py rename to history_test/test_step_error.py index e41b503d..0414ba11 100644 --- a/quest_test/test_step_error.py +++ b/history_test/test_step_error.py @@ -2,9 +2,9 @@ import pytest from .custom_errors.custom_error import MyError -from quest_test.utils import timeout -from quest import step, NoopSerializer -from quest.historian import Historian +from history_test.utils import timeout +from history import step, NoopSerializer +from history.history import History double_calls = 0 foo_calls = 0 @@ -44,7 +44,7 @@ async def longer_workflow(text): @timeout(10) async def test_custom_exception(): history = [] - historian = Historian( + historian = History( 'test', longer_workflow, history, @@ -106,7 +106,7 @@ async def longer_workflow2(text): @timeout(10) async def test_builtin(): history = [] - historian = Historian( + historian = History( 'test2', longer_workflow2, history, diff --git a/quest_test/test_suspend.py b/history_test/test_suspend.py similarity index 87% rename from quest_test/test_suspend.py rename to history_test/test_suspend.py index ed9f5176..2ebcce1a 100644 --- a/quest_test/test_suspend.py +++ b/history_test/test_suspend.py @@ -2,9 +2,9 @@ import pytest -from quest.historian import Historian -from quest.wrappers import task -from quest.serializer import NoopSerializer +from history.history import History +from history.wrappers import task +from history.serializer import NoopSerializer from .utils import timeout stop = asyncio.Event() @@ -20,7 +20,7 @@ async def workflow_will_stop(): @pytest.mark.asyncio @timeout(3) async def test_cancel(): - historian = Historian( + historian = History( 'test', workflow_will_stop, [], @@ -62,7 +62,7 @@ async def workflow_with_tasks(): @pytest.mark.asyncio @timeout(3) async def test_task_cancel(): - historian = Historian( + historian = History( 'test', workflow_with_tasks, [], diff --git a/quest_test/test_versioning.py b/history_test/test_versioning.py similarity index 92% rename from quest_test/test_versioning.py rename to history_test/test_versioning.py index 1a2101c9..53ba6225 100644 --- a/quest_test/test_versioning.py +++ b/history_test/test_versioning.py @@ -1,8 +1,8 @@ import asyncio import pytest -from quest import queue, Historian, version, get_version, task, step, state -from quest.serializer import NoopSerializer +from history import queue, History, version, get_version, task, step, state +from history.serializer import NoopSerializer V2 = '2023-08-25 append "2" to words' @@ -18,7 +18,7 @@ async def application(): return phrase history = [] - historian = Historian('test', application, history, serializer=NoopSerializer()) + historian = History('test', application, history, serializer=NoopSerializer()) historian.run() await asyncio.sleep(0.1) await historian.record_external_event('words', None, 'put', 'foo') @@ -41,7 +41,7 @@ async def application(): phrase.append(word + '2') return phrase - historian = Historian('test', application, history, serializer=NoopSerializer()) + historian = History('test', application, history, serializer=NoopSerializer()) result = historian.run() await asyncio.sleep(0.1) await historian.record_external_event('words', None, 'put', 'baz') @@ -63,7 +63,7 @@ async def application(): return phrase history = [] - historian = Historian('test', application, history, serializer=NoopSerializer()) + historian = History('test', application, history, serializer=NoopSerializer()) historian.run() await asyncio.sleep(0.1) await historian.record_external_event('words', None, 'put', 'foo') @@ -93,7 +93,7 @@ async def application(): phrase.append(await fix_case(word) + '2') return phrase - historian = Historian('test', application, history, serializer=NoopSerializer()) + historian = History('test', application, history, serializer=NoopSerializer()) result = historian.run() await asyncio.sleep(0.1) await historian.record_external_event('words', None, 'put', 'bar') @@ -125,7 +125,7 @@ async def application(): phrase.append(await fix_case(word) + '2') return phrase - historian = Historian('test', application, history, serializer=NoopSerializer()) + historian = History('test', application, history, serializer=NoopSerializer()) result = historian.run() await asyncio.sleep(0.1) await historian.record_external_event('words', None, 'put', 'baz') @@ -167,7 +167,7 @@ async def application(name1, name2, name3): name2 = 'bar' name3 = 'baz' - historian = Historian('test', application, history, serializer=NoopSerializer()) + historian = History('test', application, history, serializer=NoopSerializer()) result = historian.run(name1, name2, name3) await asyncio.sleep(0.1) diff --git a/quest_test/test_websockets.py b/history_test/test_websockets.py similarity index 83% rename from quest_test/test_websockets.py rename to history_test/test_websockets.py index 43c7ff73..6cc7ba09 100644 --- a/quest_test/test_websockets.py +++ b/history_test/test_websockets.py @@ -1,9 +1,9 @@ import asyncio import pytest -from quest import state, queue -from quest.client import Client -from quest.server import Server -from quest_test.utils import create_in_memory_workflow_manager +from history import state, queue +from history.client import Client +from history.server import Server +from history_test.utils import create_in_memory_historian def authorize(headers: dict[str, str]) -> bool: @@ -43,6 +43,6 @@ async def workflow(): messages = await messages.get() await phrase.set(messages) - manager = create_in_memory_workflow_manager({'workflow': workflow}) - manager.start_workflow('workflow', 'workflow1') + manager = create_in_memory_historian({'workflow': workflow}) + manager.start_soon('workflow', 'workflow1') await asyncio.gather(serve(manager), connect()) diff --git a/quest_test/test_workflow_metrics.py b/history_test/test_workflow_metrics.py similarity index 60% rename from quest_test/test_workflow_metrics.py rename to history_test/test_workflow_metrics.py index af1c4b02..f9257d81 100644 --- a/quest_test/test_workflow_metrics.py +++ b/history_test/test_workflow_metrics.py @@ -2,8 +2,8 @@ import pytest -from quest.manager import WorkflowNotFound -from .utils import timeout, create_in_memory_workflow_manager +from history.historian import WorkflowNotFound +from .utils import timeout, create_in_memory_historian @pytest.mark.asyncio @@ -21,12 +21,12 @@ async def workflow2(): await gate2.wait() return "done" - async with create_in_memory_workflow_manager({'w1': workflow1, 'w2': workflow2}) as manager: - manager.start_workflow('w1', 'wid1', delete_on_finish=False) - manager.start_workflow('w2', 'wid2') + async with create_in_memory_historian({'w1': workflow1, 'w2': workflow2}) as manager: + manager.start_soon('w1', 'wid1', delete_on_finish=False) + manager.start_soon('w2', 'wid2') # Scheduled workflows are counted as "running" - assert len(manager.get_workflow_metrics()) == 2 + assert len(manager.get_metrics()) == 2 # Yield to let the workflows start await asyncio.sleep(0.1) @@ -36,17 +36,17 @@ async def workflow2(): await asyncio.sleep(0.1) # Finished workflows with a stored result are still "running" - assert len(manager.get_workflow_metrics()) == 2 + assert len(manager.get_metrics()) == 2 # Allow wid1 to return result and clean up - await manager.get_workflow_result('wid1', delete=True) + await manager.get_result('wid1', delete=True) - assert len(manager.get_workflow_metrics()) == 1 + assert len(manager.get_metrics()) == 1 gate2.set() await asyncio.sleep(0.1) - assert len(manager.get_workflow_metrics()) == 0 + assert len(manager.get_metrics()) == 0 @pytest.mark.asyncio @@ -58,19 +58,19 @@ async def sample_workflow(): workflows = { "sample_workflow": sample_workflow } - manager = create_in_memory_workflow_manager(workflows=workflows) + manager = create_in_memory_historian(workflows=workflows) async with manager: # Start workflow but don't delete its result immediately - manager.start_workflow('sample_workflow', 'wid2', delete_on_finish=False) - future_wid2 = await manager.get_workflow_result('wid2') + manager.start_soon('sample_workflow', 'wid2', delete_on_finish=False) + future_wid2 = await manager.get_result('wid2') await asyncio.sleep(0.1) assert future_wid2 is not None await manager.delete_workflow('wid2') with pytest.raises(WorkflowNotFound): - await manager.get_workflow_result('wid2') + await manager.get_result('wid2') # Trying to delete a non-existing workflow raises WorkflowNotFound as well with pytest.raises(WorkflowNotFound): @@ -86,17 +86,17 @@ async def long_running_workflow(): await gate.wait() return "should not reach" - manager = create_in_memory_workflow_manager(workflows={"long_workflow": long_running_workflow}) + manager = create_in_memory_historian(workflows={"long_workflow": long_running_workflow}) async with manager: - manager.start_workflow('long_workflow', 'wid1') + manager.start_soon('long_workflow', 'wid1') await asyncio.sleep(0.1) # Cancel the running workflow - assert manager.has_workflow('wid1') + assert manager.has('wid1') await manager.delete_workflow('wid1') await asyncio.sleep(0.1) - assert not manager.has_workflow('wid1') + assert not manager.has('wid1') with pytest.raises(WorkflowNotFound): - await manager.get_workflow_result('wid1') + await manager.get_result('wid1') diff --git a/quest_test/test_wrappers.py b/history_test/test_wrappers.py similarity index 83% rename from quest_test/test_wrappers.py rename to history_test/test_wrappers.py index 4247721d..52448a83 100644 --- a/quest_test/test_wrappers.py +++ b/history_test/test_wrappers.py @@ -2,10 +2,10 @@ import pytest -from quest import Historian -from quest.external import event, wrap_as_event -from quest.wrappers import wrap_steps -from quest.serializer import NoopSerializer +from history import History +from history.external import event, wrap_as_event +from history.wrappers import wrap_steps +from history.serializer import NoopSerializer from .utils import timeout @@ -15,7 +15,7 @@ class CallMe: async def __call__(self): await asyncio.sleep(0.01) - historian = Historian('test', CallMe(), [], serializer=NoopSerializer()) + historian = History('test', CallMe(), [], serializer=NoopSerializer()) await historian.run() @@ -42,7 +42,7 @@ async def workflow(): await useful.foo() await useful.bar() - historian = Historian('test', workflow, [], serializer=NoopSerializer()) + historian = History('test', workflow, [], serializer=NoopSerializer()) historian.run() with historian.get_resource_stream(None) as resource_stream: diff --git a/history_test/utils.py b/history_test/utils.py new file mode 100644 index 00000000..37d8b27f --- /dev/null +++ b/history_test/utils.py @@ -0,0 +1,46 @@ +import asyncio +import sys +from functools import wraps + +from history import Historian, History +from history.persistence import InMemoryBlobStorage, PersistentList +from history.serializer import NoopSerializer + + +def timeout(delay): + if 'pydevd' in sys.modules: # i.e. debug mode + # Return a no-op decorator + return lambda func: func + + def decorator(func): + @wraps(func) + async def new_func(*args, **kwargs): + async with asyncio.timeout(delay): + return await func(*args, **kwargs) + + return new_func + + return decorator + + +def create_in_memory_historian(workflows: dict, serializer=None): + storage = InMemoryBlobStorage() + books = {} + + def create_book(wid: str): + if wid not in books: + books[wid] = PersistentList(wid, InMemoryBlobStorage()) + return books[wid] + + def create_workflow(wtype: str): + return workflows[wtype] + + if serializer is None: + serializer = NoopSerializer() + + return Historian('test', storage, create_book, create_workflow, serializer=serializer) + + +def create_test_history(workflow_name, workflow): + history = History(workflow_name, workflow, [], NoopSerializer()) + return history diff --git a/main.py b/main.py index f403a743..38fb2f25 100644 --- a/main.py +++ b/main.py @@ -5,8 +5,8 @@ import uuid from pathlib import Path -from src.quest import step, create_filesystem_historian -from src.quest.external import state, queue +from src.history import step, create_filesystem_history +from src.history.external import state, queue logging.basicConfig(level=logging.DEBUG) INPUT_EVENT_NAME = 'input' @@ -39,7 +39,7 @@ async def main(): workflow_id = str(uuid.uuid4()) - historian = create_filesystem_historian( + historian = create_filesystem_history( saved_state, 'demo', register_user ) diff --git a/pyproject.toml b/pyproject.toml index 3688eb29..d4029434 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [tool.poetry] -name = "quest.py" +name = "history" version = "0.2.9" description = "Framework for coordinated, long-running processes" authors = ["Gordon Bean "] -packages = [{ include = "quest", from = "src" }] +packages = [{ include = "history", from = "src" }] [tool.poetry.dependencies] python = "^3.11" diff --git a/quest_test/test_get_result.py b/quest_test/test_get_result.py deleted file mode 100644 index 7bc33405..00000000 --- a/quest_test/test_get_result.py +++ /dev/null @@ -1,100 +0,0 @@ -import asyncio - -import pytest - -from quest.manager import WorkflowNotFound -from .utils import timeout, create_in_memory_workflow_manager - - -class OurException(Exception): - pass - - -@pytest.mark.asyncio -@timeout(3) -async def test_basic_store_result(): - async def workflow1(): - return "done" - - async with create_in_memory_workflow_manager({'w1': workflow1}) as manager: - manager.start_workflow('w1', 'wid1', delete_on_finish=False) - await asyncio.sleep(0.1) - - assert await manager.get_workflow_result('wid1') == 'done' - assert manager.has_workflow('wid1') - assert await manager.get_workflow_result('wid1', delete=True) == 'done' - assert not manager.has_workflow('wid1') - - -@pytest.mark.asyncio -@timeout(3) -async def test_workflows_not_saved_have_no_results(): - async def workflow1(): - return "done" - - async with create_in_memory_workflow_manager({'w1': workflow1}) as manager: - manager.start_workflow('w1', 'wid1') - await asyncio.sleep(0.1) - assert not manager.has_workflow('wid1') - - -@pytest.mark.asyncio -@timeout(3) -async def test_get_result_on_missing_workflow_raises(): - async def workflow1(): - return "done" - - async with create_in_memory_workflow_manager({'w1': workflow1}) as manager: - manager.start_workflow('w1', 'wid1') - await asyncio.sleep(0.1) - assert not manager.has_workflow('wid1') - - with pytest.raises(WorkflowNotFound): - await manager.get_workflow_result('wid1') - - -@pytest.mark.asyncio -@timeout(6) -async def test_get_result_on_running_workflow(): - gate = asyncio.Event() - - async def sample_workflow(): - await gate.wait() - return "sample workflow result" - - workflows = { - "sample_workflow": sample_workflow - } - manager = create_in_memory_workflow_manager(workflows=workflows) - - async with manager: - manager.start_workflow('sample_workflow', 'wid1') - await asyncio.sleep(0.1) - - get_result_task = asyncio.create_task(manager.get_workflow_result('wid1')) - - gate.set() - result = await get_result_task - assert result is not None - -@pytest.mark.asyncio -@timeout(3) -async def test_exception_store_result(): - async def workflow1(): - raise OurException('died') - - async with create_in_memory_workflow_manager({'w1': workflow1}) as manager: - manager.start_workflow('w1', 'wid1', delete_on_finish=False) - await asyncio.sleep(0.1) - - assert manager.has_workflow('wid1') - - with pytest.raises(OurException): - await manager.get_workflow_result('wid1') - - assert manager.has_workflow('wid1') - - with pytest.raises(OurException): - await manager.get_workflow_result('wid1', delete=True) - - assert not manager.has_workflow('wid1') diff --git a/quest_test/utils.py b/quest_test/utils.py deleted file mode 100644 index 7457a616..00000000 --- a/quest_test/utils.py +++ /dev/null @@ -1,46 +0,0 @@ -import asyncio -import sys -from functools import wraps - -from quest import WorkflowManager, Historian -from quest.persistence import InMemoryBlobStorage, PersistentHistory -from quest.serializer import NoopSerializer - - -def timeout(delay): - if 'pydevd' in sys.modules: # i.e. debug mode - # Return a no-op decorator - return lambda func: func - - def decorator(func): - @wraps(func) - async def new_func(*args, **kwargs): - async with asyncio.timeout(delay): - return await func(*args, **kwargs) - - return new_func - - return decorator - - -def create_in_memory_workflow_manager(workflows: dict, serializer=None): - storage = InMemoryBlobStorage() - histories = {} - - def create_history(wid: str): - if wid not in histories: - histories[wid] = PersistentHistory(wid, InMemoryBlobStorage()) - return histories[wid] - - def create_workflow(wtype: str): - return workflows[wtype] - - if serializer is None: - serializer = NoopSerializer() - - return WorkflowManager('test', storage, create_history, create_workflow, serializer=serializer) - - -def create_test_historian(workflow_name, workflow): - historian = Historian(workflow_name, workflow, [], NoopSerializer()) - return historian diff --git a/scratch/ainput.py b/scratch/ainput.py index f5e4a57a..1f054c4e 100644 --- a/scratch/ainput.py +++ b/scratch/ainput.py @@ -1,8 +1,8 @@ import asyncio import signal -from quest.utils import ainput -from quest import step, create_filesystem_manager +from history.utils import ainput +from history import step, create_filesystem_historian from pathlib import Path def handle(*args): @@ -55,13 +55,13 @@ async def the_workflow(): async def sleep_workflow(): print('Sleep workflow running') - async with create_filesystem_manager( + async with create_filesystem_historian( Path('ainput_state'), 'sleep', lambda wid: the_workflow ) as manager: - if not manager.has_workflow('sleep_workflow'): - manager.start_workflow('sleep_workflow', 'sleep_workflow') + if not manager.has('sleep_workflow'): + manager.start_soon('sleep_workflow', 'sleep_workflow') await manager.get_workflow('sleep_workflow') diff --git a/scratch/inputs.py b/scratch/inputs.py index a70595eb..915809b9 100644 --- a/scratch/inputs.py +++ b/scratch/inputs.py @@ -3,7 +3,7 @@ import uuid from pathlib import Path -from quest import task, create_filesystem_manager +from history import task, create_filesystem_historian @task @@ -30,12 +30,12 @@ async def do_stuff(): async def main(): wid = 'input-' + str(uuid.uuid1()) - async with create_filesystem_manager( + async with create_filesystem_historian( Path('state'), 'guess_game_demo', lambda wid: do_stuff ) as manager: - manager.start_workflow( + manager.start_soon( '', wid ) diff --git a/scratch/print_history.py b/scratch/print_history.py index 371f5370..9204883e 100644 --- a/scratch/print_history.py +++ b/scratch/print_history.py @@ -1,6 +1,6 @@ from pathlib import Path -from quest import LocalFileSystemBlobStorage, PersistentHistory +from history import LocalFileSystemBlobStorage, PersistentList def print_record(record): @@ -9,7 +9,7 @@ def print_record(record): def main(wid: str, namespace_folder: Path): - history = PersistentHistory(wid, LocalFileSystemBlobStorage(namespace_folder / wid)) + history = PersistentList(wid, LocalFileSystemBlobStorage(namespace_folder / wid)) for record in history: print_record(record) diff --git a/scratch/scratch.py b/scratch/scratch.py index 7583a172..80ad109d 100644 --- a/scratch/scratch.py +++ b/scratch/scratch.py @@ -1,5 +1,5 @@ -from src.quest.workflow import signal, promised_signal, any_promise -from src.quest import step +from src.history.workflow import signal, promised_signal, any_promise +from src.history import step @promised_signal diff --git a/scratch/serialize_stuff.py b/scratch/serialize_stuff.py index 2875a399..2be2cd38 100644 --- a/scratch/serialize_stuff.py +++ b/scratch/serialize_stuff.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from typing import Callable, Protocol, TypeVar, Any, TypedDict -from quest import step, Historian +from history import step, History T = TypeVar('T') @@ -75,7 +75,7 @@ async def workflow(): async def main(): - historian = Historian('demo', workflow, []) + historian = History('demo', workflow, []) wtask = historian.run() await asyncio.sleep(0.1) await historian.suspend() @@ -93,7 +93,7 @@ def create_historian(wid: str, func: Callable, type_serializers: dict=None): else: step_serializer = MasterSerializer(type_serializers) - return Historian( + return History( wid, func, history, diff --git a/scratch/sql_scratch.py b/scratch/sql_scratch.py index 7f26f21b..b2a3826e 100644 --- a/scratch/sql_scratch.py +++ b/scratch/sql_scratch.py @@ -2,7 +2,7 @@ import random from pathlib import Path -from quest import step, create_filesystem_manager, create_sql_manager, ainput +from history import step, create_filesystem_historian, create_sql_manager, ainput @step @@ -40,7 +40,7 @@ async def main(): 'guess_game_demo', lambda wid: guessing_game ) as manager: - manager.start_workflow( + manager.start_soon( '', 'demo' ) diff --git a/src/quest/__init__.py b/src/history/__init__.py similarity index 51% rename from src/quest/__init__.py rename to src/history/__init__.py index 5c0e6bb3..18573a34 100644 --- a/src/quest/__init__.py +++ b/src/history/__init__.py @@ -3,39 +3,39 @@ from .context import these from .external import state, queue, identity_queue, event -from .historian import Historian from .history import History -from .manager import WorkflowManager, WorkflowFactory +from .book import Book +from .historian import Historian, WorkflowFactory from .manager_wrappers import alias -from .persistence import LocalFileSystemBlobStorage, PersistentHistory, BlobStorage, Blob +from .persistence import LocalFileSystemBlobStorage, PersistentList, BlobStorage, Blob from .serializer import StepSerializer, MasterSerializer, NoopSerializer from .utils import ainput from .versioning import version, get_version from .wrappers import step, task, wrap_steps -def create_filesystem_historian(save_folder: Path, historian_id: str, function: Callable, - serializer: StepSerializer = None) -> Historian: +def create_filesystem_history(save_folder: Path, history_id: str, function: Callable, + serializer: StepSerializer = None) -> History: storage = LocalFileSystemBlobStorage(save_folder) - history = PersistentHistory(historian_id, storage) + history = PersistentList(history_id, storage) serializer = serializer or NoopSerializer() - return Historian( - historian_id, + return History( + history_id, function, history, serializer=serializer ) -def create_filesystem_manager( +def create_filesystem_historian( save_folder: Path, namespace: str, factory: WorkflowFactory, serializer: StepSerializer = NoopSerializer() -) -> WorkflowManager: - def create_history(wid: str) -> History: - return PersistentHistory(wid, LocalFileSystemBlobStorage(save_folder / namespace / wid)) +) -> Historian: + def create_book(wid: str) -> Book: + return PersistentList(wid, LocalFileSystemBlobStorage(save_folder / namespace / wid)) workflow_manager_storage = LocalFileSystemBlobStorage(save_folder / namespace) - return WorkflowManager(namespace, workflow_manager_storage, create_history, factory, serializer=serializer) + return Historian(namespace, workflow_manager_storage, create_book, factory, serializer=serializer) diff --git a/src/quest/history.py b/src/history/book.py similarity index 74% rename from src/quest/history.py rename to src/history/book.py index 61bf3418..6fd1e8e5 100644 --- a/src/quest/history.py +++ b/src/history/book.py @@ -1,8 +1,8 @@ from typing import Protocol, Reversible -from .quest_types import EventRecord +from .history_types import EventRecord -class History(Protocol, Reversible): +class Book(Protocol, Reversible): def append(self, item: EventRecord): ... def remove(self, item: EventRecord): ... diff --git a/src/quest/client.py b/src/history/client.py similarity index 100% rename from src/quest/client.py rename to src/history/client.py diff --git a/src/quest/context.py b/src/history/context.py similarity index 100% rename from src/quest/context.py rename to src/history/context.py diff --git a/src/quest/custom_exceptions.py b/src/history/custom_exceptions.py similarity index 100% rename from src/quest/custom_exceptions.py rename to src/history/custom_exceptions.py diff --git a/src/quest/external.py b/src/history/external.py similarity index 91% rename from src/quest/external.py rename to src/history/external.py index 9e4ea736..ba72e396 100644 --- a/src/quest/external.py +++ b/src/history/external.py @@ -2,7 +2,7 @@ import uuid from typing import TypeVar, Generic -from .historian import Historian, find_historian, SUSPENDED, wrap_methods_as_historian_events +from .history import History, find_history, SUSPENDED, wrap_methods_as_history_events # @@ -84,11 +84,11 @@ def __init__(self, name, identity, resource: T): self._name = name self._identity = identity self._resource: T = resource - self._historian = find_historian() + self._historian = find_history() async def __aenter__(self) -> T: await self._historian.register_resource(self._name, self._identity, self._resource) - return wrap_methods_as_historian_events(self._resource, self._name, self._identity, self._historian) + return wrap_methods_as_history_events(self._resource, self._name, self._identity, self._historian) async def __aexit__(self, exc_type, exc_val, exc_tb): suspending = (exc_type == asyncio.CancelledError and exc_val.args and exc_val.args[0] == SUSPENDED) @@ -111,7 +111,7 @@ def identity_queue(name): return InternalResource(name, None, IdentityQueue()) class _ResourceWrapper: - def __init__(self, name: str, identity: str | None, historian: 'Historian', resource_class): + def __init__(self, name: str, identity: str | None, historian: 'History', resource_class): self._name = name self._identity = identity self._historian = historian @@ -129,14 +129,14 @@ async def wrapper(*args, _name=self._name, _identity=self._identity, **kwargs): return wrapper -def wrap_as_queue(name: str, identity: str | None, historian: Historian) -> Queue: +def wrap_as_queue(name: str, identity: str | None, historian: History) -> Queue: return _ResourceWrapper(name, identity, historian, Queue) -def wrap_as_event(name: str, identity: str | None, historian: Historian) -> Event: +def wrap_as_event(name: str, identity: str | None, historian: History) -> Event: return _ResourceWrapper(name, identity, historian, Event) -def wrap_as_state(name: str, identity: str | None, historian: Historian) -> State: +def wrap_as_state(name: str, identity: str | None, historian: History) -> State: return _ResourceWrapper(name, identity, historian, State) -def wrap_as_identity_queue(name: str, identity: str | None, historian: Historian) -> IdentityQueue: +def wrap_as_identity_queue(name: str, identity: str | None, historian: History) -> IdentityQueue: return _ResourceWrapper(name, identity, historian, IdentityQueue) \ No newline at end of file diff --git a/src/quest/extras/__init__.py b/src/history/extras/__init__.py similarity index 100% rename from src/quest/extras/__init__.py rename to src/history/extras/__init__.py diff --git a/src/quest/extras/aws.py b/src/history/extras/aws.py similarity index 90% rename from src/quest/extras/aws.py rename to src/history/extras/aws.py index b30358a5..3d71e56f 100644 --- a/src/quest/extras/aws.py +++ b/src/history/extras/aws.py @@ -1,13 +1,13 @@ import os import json -from .. import BlobStorage, Blob, WorkflowManager, WorkflowFactory, PersistentHistory, History +from .. import BlobStorage, Blob, Historian, WorkflowFactory, PersistentList, Book try: import boto3 from botocore.exceptions import ClientError except ImportError: - raise ImportError("The 'aws' extra is required to use this module. Run 'pip install quest-py[aws]'.") + raise ImportError("The 'aws' extra is required to use this module. Run 'pip install history-py[aws]'.") class S3Bucket: @@ -20,7 +20,7 @@ def __init__(self): region_name=self._region ) - self._bucket_name = 'quest-records' + self._bucket_name = 'history-records' self._s3_client = self.session.client('s3') self._prepare_bucket() @@ -87,15 +87,15 @@ def delete_blob(self, key: str): def create_s3_manager( namespace: str, factory: WorkflowFactory, -) -> WorkflowManager: +) -> Historian: s3 = S3Bucket() storage = S3BlobStorage(namespace, s3.get_s3_client(), s3.get_bucket_name()) - def create_history(wid: str) -> History: - return PersistentHistory(wid, S3BlobStorage(wid, s3.get_s3_client(), s3.get_bucket_name())) + def create_history(wid: str) -> Book: + return PersistentList(wid, S3BlobStorage(wid, s3.get_s3_client(), s3.get_bucket_name())) - return WorkflowManager(namespace, storage, create_history, factory) + return Historian(namespace, storage, create_history, factory) class DynamoDB: @@ -202,12 +202,12 @@ def delete_blob(self, key: str): def create_dynamodb_manager( namespace: str, factory: WorkflowFactory, -) -> WorkflowManager: +) -> Historian: dynamodb = DynamoDB() storage = DynamoDBBlobStorage(namespace, dynamodb.get_table()) - def create_history(wid: str) -> History: - return PersistentHistory(wid, DynamoDBBlobStorage(wid, dynamodb.get_table())) + def create_history(wid: str) -> Book: + return PersistentList(wid, DynamoDBBlobStorage(wid, dynamodb.get_table())) - return WorkflowManager(namespace, storage, create_history, factory) + return Historian(namespace, storage, create_history, factory) diff --git a/src/quest/extras/sql.py b/src/history/extras/sql.py similarity index 87% rename from src/quest/extras/sql.py rename to src/history/extras/sql.py index faa5c803..ccfbace7 100644 --- a/src/quest/extras/sql.py +++ b/src/history/extras/sql.py @@ -1,10 +1,10 @@ -from .. import WorkflowFactory, WorkflowManager, PersistentHistory, History, BlobStorage, Blob +from .. import WorkflowFactory, Historian, PersistentList, Book, BlobStorage, Blob try: from sqlalchemy import create_engine, Column, Integer, String, JSON, Engine from sqlalchemy.orm import sessionmaker, Session, declarative_base except ImportError: - raise ImportError("The 'sql' extra is required to use this module. Run 'pip install quest-py[sql]'.") + raise ImportError("The 'sql' extra is required to use this module. Run 'pip install history-py[sql]'.") Base = declarative_base() @@ -77,12 +77,12 @@ def create_sql_manager( db_url: str, namespace: str, factory: WorkflowFactory -) -> WorkflowManager: +) -> Historian: database = SQLDatabase(db_url) storage = SqlBlobStorage(namespace, database.get_session()) - def create_history(wid: str) -> History: - return PersistentHistory(wid, SqlBlobStorage(wid, database.get_session())) + def create_history(wid: str) -> Book: + return PersistentList(wid, SqlBlobStorage(wid, database.get_session())) - return WorkflowManager(namespace, storage, create_history, factory) + return Historian(namespace, storage, create_history, factory) diff --git a/src/quest/manager.py b/src/history/historian.py similarity index 83% rename from src/quest/manager.py rename to src/history/historian.py index cd946191..bfca7ec9 100644 --- a/src/quest/manager.py +++ b/src/history/historian.py @@ -6,19 +6,19 @@ from typing import Protocol, Callable, TypeVar, Any, TypedDict from .external import State, IdentityQueue, Queue, Event -from .historian import Historian, _Wrapper, SUSPENDED -from .history import History +from .history import History, _Wrapper, SUSPENDED +from .book import Book from .persistence import BlobStorage from .serializer import StepSerializer -from .utils import quest_logger, serialize_exception, deserialize_exception +from .utils import history_logger, serialize_exception, deserialize_exception class WorkflowNotFound(Exception): pass -class HistoryFactory(Protocol): - def __call__(self, workflow_id: str) -> History: ... +class BookFactory(Protocol): + def __call__(self, workflow_id: str) -> Book: ... class WorkflowFactory(Protocol): @@ -27,7 +27,7 @@ def __call__(self, workflow_type: str) -> Callable: ... T = TypeVar('T') -workflow_manager = ContextVar('workflow_manager') +historian = ContextVar('historian') class DuplicateAliasException(Exception): @@ -53,34 +53,34 @@ class WorkflowData(TypedDict): start_time: str -class WorkflowManager: +class Historian: """ Runs workflow tasks It remembers which tasks are still active and resumes them on replay """ - def __init__(self, namespace: str, storage: BlobStorage, create_history: HistoryFactory, + def __init__(self, namespace: str, storage: BlobStorage, create_book: BookFactory, create_workflow: WorkflowFactory, serializer: StepSerializer): self._namespace = namespace self._storage = storage - self._create_history = create_history + self.create_book = create_book self._create_workflow = create_workflow self._workflow_data: dict[str, WorkflowData] = {} # Tracks all workflows - self._workflows: dict[str, Historian] = {} + self._workflows: dict[str, History] = {} self._workflow_tasks: dict[str, asyncio.Task] = {} self._alias_dictionary = {} self._serializer: StepSerializer = serializer self._results: dict[str, WorkflowResult] = {} - async def __aenter__(self) -> 'WorkflowManager': + async def __aenter__(self) -> 'Historian': """Load the workflows and get them running again""" - def our_handler(sig, frame): - self._quest_signal_handler(sig, frame) + def handler(sig, frame): + self._signal_handler(sig, frame) raise asyncio.CancelledError(SUSPENDED) # TODO - add cancel api - get cancel from historian api - signal.signal(signal.SIGINT, our_handler) + signal.signal(signal.SIGINT, handler) if self._storage.has_blob(self._namespace): self._workflow_data = self._storage.read_blob(self._namespace) @@ -95,14 +95,14 @@ def our_handler(sig, frame): args = data["workflow_args"] kwargs = data["workflow_kwargs"] delete_on_finish = data["delete_on_finish"] - self._start_workflow(wtype, wid, args, kwargs, delete_on_finish=delete_on_finish) + self._start_soon(wtype, wid, args, kwargs, delete_on_finish=delete_on_finish) return self async def __aexit__(self, exc_type, exc_val, exc_tb): """Save whatever state is necessary before exiting""" - for wid, historian in self._workflows.items(): - await historian.suspend() + for wid, history in self._workflows.items(): + await history.suspend() self._storage.write_blob(self._namespace, self._workflow_data) self._storage.write_blob(f'{self._namespace}_results', self._results) @@ -110,28 +110,28 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): self._workflows.clear() self._workflow_tasks.clear() - def _quest_signal_handler(self, sig, frame): - quest_logger.debug(f'Caught KeyboardInterrupt: {sig}') - for wid, historian in self._workflows.items(): - historian.signal_suspend() + def _signal_handler(self, sig, frame): + history_logger.debug(f'Caught KeyboardInterrupt: {sig}') + for wid, history in self._workflows.items(): + history.signal_suspend() def _get_workflow(self, workflow_id: str): workflow_id = self._alias_dictionary.get(workflow_id, workflow_id) return self._workflows[workflow_id] - def _start_workflow(self, - workflow_type: str, workflow_id: str, workflow_args, workflow_kwargs, - delete_on_finish: bool = True): + def _start_soon(self, + workflow_type: str, workflow_id: str, workflow_args, workflow_kwargs, + delete_on_finish: bool = True): workflow_function = self._create_workflow(workflow_type) - workflow_manager.set(self) + historian.set(self) - history = self._create_history(workflow_id) - historian: Historian = Historian(workflow_id, workflow_function, history, serializer=self._serializer) - self._workflows[workflow_id] = historian + book = self.create_book(workflow_id) + history: History = History(workflow_id, workflow_function, book, serializer=self._serializer) + self._workflows[workflow_id] = history - self._workflow_tasks[workflow_id] = (task := historian.run(*workflow_args, **workflow_kwargs)) + self._workflow_tasks[workflow_id] = (task := history.run(*workflow_args, **workflow_kwargs)) # run _store_result asynchronously in the background task.add_done_callback(lambda t: asyncio.create_task(self._store_result(workflow_id, t, delete_on_finish))) @@ -178,8 +178,8 @@ async def _store_result(self, workflow_id: str, task: asyncio.Task, delete_on_fi del self._workflow_tasks[workflow_id] del self._workflow_data[workflow_id] - def start_workflow(self, workflow_type: str, workflow_id: str, *workflow_args, delete_on_finish: bool = True, - **workflow_kwargs): + def start_soon(self, workflow_type: str, workflow_id: str, *workflow_args, delete_on_finish: bool = True, + **workflow_kwargs): """Start the workflow, but do not restart previously canceled ones""" start_time = datetime.utcnow().isoformat() @@ -193,10 +193,10 @@ def start_workflow(self, workflow_type: str, workflow_id: str, *workflow_args, d delete_on_finish=delete_on_finish, start_time=start_time ) - self._start_workflow(workflow_type, workflow_id, workflow_args, workflow_kwargs, - delete_on_finish=delete_on_finish) + self._start_soon(workflow_type, workflow_id, workflow_args, workflow_kwargs, + delete_on_finish=delete_on_finish) - def has_workflow(self, workflow_id: str) -> bool: + def has(self, workflow_id: str) -> bool: workflow_id = self._alias_dictionary.get(workflow_id, workflow_id) return workflow_id in self._workflows or workflow_id in self._results @@ -274,7 +274,7 @@ async def _deregister_alias(self, alias: str): if alias in self._alias_dictionary: del self._alias_dictionary[alias] - def get_workflow_metrics(self): + def get_metrics(self): """Return metrics for active workflows""" metrics = [] for wid, data in self._results.items(): @@ -291,7 +291,7 @@ def get_workflow_metrics(self): }) return metrics - async def get_workflow_result(self, workflow_id: str, delete: bool = False): + async def get_result(self, workflow_id: str, delete: bool = False): if workflow_id in self._workflow_tasks: # The workflow is still running, so return the running return await self._workflow_tasks[workflow_id] @@ -325,6 +325,6 @@ async def get_workflow_result(self, workflow_id: str, delete: bool = False): raise WorkflowNotFound(f"Workflow '{workflow_id}' does not exist.") -def find_workflow_manager() -> WorkflowManager: - if (manager := workflow_manager.get()) is not None: - return manager +def find_historian() -> Historian: + if (h := historian.get()) is not None: + return h diff --git a/src/quest/historian.py b/src/history/history.py similarity index 91% rename from src/quest/historian.py rename to src/history/history.py index 7854b6ca..e9e02ff7 100644 --- a/src/quest/historian.py +++ b/src/history/history.py @@ -7,19 +7,19 @@ from functools import wraps from typing import Callable, TypeVar -from .history import History -from .quest_types import ConfigurationRecord, VersionRecord, StepStartRecord, StepEndRecord, \ +from .book import Book +from .history_types import ConfigurationRecord, VersionRecord, StepStartRecord, StepEndRecord, \ ExceptionDetails, ResourceAccessEvent, ResourceEntry, ResourceLifecycleEvent, TaskEvent from .resources import ResourceStreamManager from .serializer import StepSerializer -from .utils import quest_logger, task_name_getter +from .utils import history_logger, task_name_getter from .utils import ( serialize_exception, deserialize_exception ) -QUEST_VERSIONS = "_quest_versions" +HISTORY_VERSIONS = "_history_versions" GLOBAL_VERSION = "_global_version" # external replay: @@ -87,8 +87,8 @@ class _Wrapper: pass -def wrap_methods_as_historian_events(resource: T, name: str, identity: str | None, historian: 'Historian', - internal=True) -> T: +def wrap_methods_as_history_events(resource: T, name: str, identity: str | None, historian: 'History', + internal=True) -> T: wrapper = _Wrapper() historian_action = historian.handle_internal_event if internal else historian.record_external_event @@ -122,7 +122,7 @@ def _get_id(item): return item -def _prune(step_id: str, history: "History"): +def _prune(step_id: str, history: "Book"): """ Remove substep work Records whose step_ids are prefixed by the step_id of the step are substep work @@ -198,8 +198,8 @@ def _get_exception_class(exception_type: str): return exception_class -class Historian: - def __init__(self, workflow_id: str, workflow: Callable, history: History, serializer: StepSerializer): +class History: + def __init__(self, workflow_id: str, workflow: Callable, history: Book, serializer: StepSerializer): # TODO - change nomenclature (away from workflow)? Maybe just use workflow.__name__? self.workflow_id = workflow_id self.workflow = workflow @@ -210,7 +210,7 @@ def __init__(self, workflow_id: str, workflow: Callable, history: History, seria self._workflow_completed = False # These things need to be serialized - self._history: History = history + self._history: Book = history self._serializer: StepSerializer = serializer @@ -288,7 +288,7 @@ def __init__(self, workflow_id: str, workflow: Callable, history: History, seria self._last_record_gate: asyncio.Future = None def _reset_replay(self): - quest_logger.debug('Resetting replay') + history_logger.debug('Resetting replay') self._configuration_pos = 0 @@ -332,7 +332,7 @@ async def _replay_complete(self): if self._last_record_gate is not None: await self._last_record_gate - quest_logger.debug(f'{self.workflow_id} -- Replay Complete --') + history_logger.debug(f'{self.workflow_id} -- Replay Complete --') # TODO - log this only once? self._process_discovered_versions() @@ -387,11 +387,11 @@ async def _task_replay_records(self, task_id): if record['task_id'] != task_id: if (gate := self._record_gates[_get_id(record)]).done(): if gate.exception() is not None: - quest_logger.debug(f'{task_id} found {record} errored: {gate.exception()}') + history_logger.debug(f'{task_id} found {record} errored: {gate.exception()}') else: - quest_logger.debug(f'{task_id} found {record} completed') + history_logger.debug(f'{task_id} found {record} completed') else: - quest_logger.debug(f'{task_id} waiting on {record}') + history_logger.debug(f'{task_id} waiting on {record}') # We await either way so if the gate has an error we see it await gate @@ -399,27 +399,27 @@ async def _task_replay_records(self, task_id): def complete(r, exc_type, exc_val, exc_tb): if exc_type is not None: exc_info = "".join(traceback.format_exception(exc_type, exc_val, exc_tb)) - quest_logger.debug(f'Noting that record {r} raised: \n{exc_info}') + history_logger.debug(f'Noting that record {r} raised: \n{exc_info}') # Note: # While Futures, the record gates are only used as gates # The return values are never used # Thus, even if there was an error when the task completed # we simply want to indicate the gate is finished # The relevant error will be raised in handle_step - quest_logger.debug(f'{task_id} completing {r}') + history_logger.debug(f'{task_id} completing {r}') self._record_gates[_get_id(r)].set_result(None) # noinspection PyUnboundLocalVariable - quest_logger.debug(f'{self._get_task_name()} replaying {record}') + history_logger.debug(f'{self._get_task_name()} replaying {record}') yield self._NextRecord(record, complete) - quest_logger.debug(f'Replay for {self._get_task_name()} complete') + history_logger.debug(f'Replay for {self._get_task_name()} complete') task_replay.set() await self._replay_complete() async def _external_handler(self): try: - quest_logger.debug(f'External event handler {self._get_task_name()} starting') + history_logger.debug(f'External event handler {self._get_task_name()} starting') async for next_record in self._task_replay_records(self._get_external_task_name()): with next_record as record: if record['type'] == 'external': @@ -431,9 +431,9 @@ async def _external_handler(self): elif record['type'] == 'configuration': await self._run_configuration(record) - quest_logger.debug(f'External event handler {self._get_task_name()} completed') + history_logger.debug(f'External event handler {self._get_task_name()} completed') except Exception: - quest_logger.exception('Error in _external_handler') + history_logger.exception('Error in _external_handler') raise async def _next_record(self): @@ -449,7 +449,7 @@ async def _next_record(self): async def _run_configuration(self, config_record: ConfigurationRecord): config_function, args, kwargs = self._configurations[self._configuration_pos] - quest_logger.debug(f'Running configuration: {get_function_name(config_function)}(*{args}, **{kwargs})') + history_logger.debug(f'Running configuration: {get_function_name(config_function)}(*{args}, **{kwargs})') assert config_record['function_name'] == get_function_name(config_function), str(config_record) assert config_record['args'] == args, str(config_record) @@ -460,7 +460,7 @@ async def _run_configuration(self, config_record: ConfigurationRecord): def get_version(self, module_name, function_name, version_name=GLOBAL_VERSION): version = self._versions.get(_get_qualified_version(module_name, function_name, version_name), None) - quest_logger.debug( + history_logger.debug( f'{self._get_task_name()} get_version({module_name}, {function_name}, {version_name} returned "{version}"') return version @@ -484,7 +484,7 @@ def _record_version_event(self, version_name, version): if self._versions.get(version_name, None) == version: return # Version not changed - quest_logger.debug(f'Version record: {version_name} = {version}') + history_logger.debug(f'Version record: {version_name} = {version}') self._versions[version_name] = version self._history.append(VersionRecord( @@ -496,13 +496,13 @@ def _record_version_event(self, version_name, version): )) def _replay_version(self, record: VersionRecord): - quest_logger.debug(f'{self._get_task_name()} setting version {record["step_id"]} = "{record["version"]}"') + history_logger.debug(f'{self._get_task_name()} setting version {record["step_id"]} = "{record["version"]}"') self._versions[record['step_id']] = record['version'] # TODO - keep or discard? async def _after_version(self, module_name, func_name, version_name, version): version_name = _get_qualified_version(module_name, func_name, version_name) - quest_logger.debug(f'{self._get_task_name()} is waiting for version {version_name}=={version}') + history_logger.debug(f'{self._get_task_name()} is waiting for version {version_name}=={version}') found = False for record in self._existing_history: @@ -513,7 +513,7 @@ async def _after_version(self, module_name, func_name, version_name, version): await self._record_gates[_get_id(record)] if not found: - quest_logger.error(f'{self._get_task_name()} did not find version {version_name}=={version}') + history_logger.error(f'{self._get_task_name()} did not find version {version_name}=={version}') raise Exception(f'{self._get_task_name()} did not find version {version_name}=={version}') if (next_record := await self._next_record()) is not None: @@ -552,7 +552,7 @@ async def handle_step(self, func_name, func: Callable, *args, **kwargs): assert record['type'] == 'start' if next_record is None: - quest_logger.debug(f'{self._get_task_name()} starting step {func_name} with {args} and {kwargs}') + history_logger.debug(f'{self._get_task_name()} starting step {func_name} with {args} and {kwargs}') self._history.append(StepStartRecord( type='start', timestamp=_get_current_timestamp(), @@ -567,7 +567,7 @@ async def handle_step(self, func_name, func: Callable, *args, **kwargs): result = func(*args, **kwargs) if hasattr(result, '__await__'): result = await result - quest_logger.debug(f'{self._get_task_name()} completing step {func_name} with {result}') + history_logger.debug(f'{self._get_task_name()} completing step {func_name} with {result}') serialized_result = await self._serializer.serialize(result) @@ -587,7 +587,7 @@ async def handle_step(self, func_name, func: Callable, *args, **kwargs): prune_on_exit = False raise asyncio.CancelledError(SUSPENDED) from cancel else: - quest_logger.exception(f'{step_id} canceled') + history_logger.exception(f'{step_id} canceled') serialized_exception = serialize_exception(cancel) self._history.append(StepEndRecord( type='end', @@ -600,7 +600,7 @@ async def handle_step(self, func_name, func: Callable, *args, **kwargs): raise except Exception as ex: - quest_logger.exception(f'Error in {step_id}') + history_logger.exception(f'Error in {step_id}') serialized_exception = serialize_exception(ex) self._history.append(StepEndRecord( type='end', @@ -624,7 +624,7 @@ async def record_external_event(self, name, identity, action, *args, **kwargs): resource_id = _create_resource_id(name, identity) step_id = self._get_unique_id(resource_id + '.' + action) - quest_logger.debug(f'External event {step_id} with {args} and {kwargs}') + history_logger.debug(f'External event {step_id} with {args} and {kwargs}') resource = self._resources[resource_id]['resource'] @@ -696,7 +696,7 @@ async def handle_internal_event(self, name, identity, action, *args, **kwargs): assert list(args) == list(record['args']), str(record) assert kwargs == record['kwargs'], str(record) - quest_logger.debug(f'Calling {step_id} with {args} and {kwargs}') + history_logger.debug(f'Calling {step_id} with {args} and {kwargs}') if inspect.iscoroutinefunction(function): result = await function(*args, **kwargs) else: @@ -736,7 +736,7 @@ async def register_resource(self, name, identity, resource): # TODO - custom exception step_id = self._get_unique_id(resource_id + '.' + '__init__') - quest_logger.debug(f'Creating {resource_id}') + history_logger.debug(f'Creating {resource_id}') self._resources[resource_id] = ResourceEntry( name=name, @@ -770,7 +770,7 @@ async def delete_resource(self, name, identity, suspending=False): # TODO - custom exception step_id = self._get_unique_id(resource_id + '.' + '__del__') - quest_logger.debug(f'Removing {resource_id}') + history_logger.debug(f'Removing {resource_id}') resource_entry = self._resources.pop(resource_id) if not suspending: @@ -793,11 +793,11 @@ async def delete_resource(self, name, identity, suspending=False): def start_task(self, func, *args, name=None, task_factory=asyncio.create_task, **kwargs): historian_context.set(self) task_id = name or self._get_unique_id(get_function_name(func)) - quest_logger.debug(f'Requested {task_id} start') + history_logger.debug(f'Requested {task_id} start') @wraps(func) async def _func(*a, **kw): - quest_logger.debug(f'Starting task {task_id}') + history_logger.debug(f'Starting task {task_id}') if (next_record := await self._next_record()) is None: self._history.append(TaskEvent( @@ -826,7 +826,7 @@ async def _func(*a, **kw): assert record['type'] == 'complete_task' assert record['task_id'] == task_id - quest_logger.debug(f'Completing task {task_id}') + history_logger.debug(f'Completing task {task_id}') return result @@ -861,7 +861,7 @@ async def _run_with_exception_handling(self, *args, **kwargs): async def _run(self, *args, **kwargs): historian_context.set(self) task_name_getter.set(self._get_task_name) - quest_logger.debug(f'Running workflow {self.workflow_id}') + history_logger.debug(f'Running workflow {self.workflow_id}') self._add_new_configurations() self._reset_replay() @@ -929,7 +929,7 @@ def _add_new_configurations(self): # Add new configuration records for config_function, args, kwargs in self._configurations[len(config_records):]: - quest_logger.debug(f'Adding new configuration: {get_function_name(config_function)}(*{args}, **{kwargs}') + history_logger.debug(f'Adding new configuration: {get_function_name(config_function)}(*{args}, **{kwargs}') self._history.append(ConfigurationRecord( type='configuration', @@ -942,7 +942,7 @@ def _add_new_configurations(self): )) def signal_suspend(self): - quest_logger.debug(f'-- Suspending {self.workflow_id} --') + history_logger.debug(f'-- Suspending {self.workflow_id} --') self._resource_stream_manager.notify_of_workflow_stop() @@ -952,7 +952,7 @@ def signal_suspend(self): # so we cancel the children before the parents. for task in list(reversed(self._open_tasks)): if not task.done() or task.cancelled() or task.cancelling(): - quest_logger.debug(f'Suspending task {task.get_name()}') + history_logger.debug(f'Suspending task {task.get_name()}') task.cancel(SUSPENDED) async def suspend(self): @@ -965,7 +965,7 @@ async def suspend(self): try: await task except asyncio.CancelledError: - quest_logger.debug(f'Task {task.get_name()} was cancelled') + history_logger.debug(f'Task {task.get_name()} was cancelled') pass async def get_resources(self, identity): @@ -1000,7 +1000,7 @@ class HistorianNotFoundException(Exception): pass -def find_historian() -> Historian: +def find_history() -> History: if (workflow := historian_context.get()) is not None: return workflow @@ -1010,5 +1010,5 @@ def find_historian() -> Historian: outer_frame = outer_frame.f_back if outer_frame is None: raise HistorianNotFoundException("Historian object not found in event stack") - is_workflow = isinstance(outer_frame.f_locals.get('self'), Historian) + is_workflow = isinstance(outer_frame.f_locals.get('self'), History) return outer_frame.f_locals.get('self') diff --git a/src/quest/quest_types.py b/src/history/history_types.py similarity index 100% rename from src/quest/quest_types.py rename to src/history/history_types.py diff --git a/src/quest/lifecycle.py b/src/history/lifecycle.py similarity index 90% rename from src/quest/lifecycle.py rename to src/history/lifecycle.py index 28f91c8a..220c0e35 100644 --- a/src/quest/lifecycle.py +++ b/src/history/lifecycle.py @@ -1,7 +1,7 @@ from typing import Protocol, TypeVar -from .historian import Historian from .history import History +from .book import Book WT = TypeVar('WT') @@ -29,7 +29,7 @@ def save_workflow(self, workflow_id: str, workflow_function: WT): class HistoryFactory(Protocol): - def __call__(self, workflow_id) -> History: ... + def __call__(self, workflow_id) -> Book: ... class WorkflowLifecycleManager: @@ -40,11 +40,11 @@ def __init__(self, self._workflow_factory = workflow_factory self._history_factory = history_factory - self._historians: dict[str, Historian] = {} + self._historians: dict[str, History] = {} async def run_workflow(self, workflow_id: str, *args, **kwargs): if workflow_id not in self._historians: - self._historians[workflow_id] = Historian( + self._historians[workflow_id] = History( workflow_id, self._workflow_factory.create_new_workflow(), self._history_factory(workflow_id) diff --git a/src/quest/manager_wrappers.py b/src/history/manager_wrappers.py similarity index 80% rename from src/quest/manager_wrappers.py rename to src/history/manager_wrappers.py index 697a7992..49b22586 100644 --- a/src/quest/manager_wrappers.py +++ b/src/history/manager_wrappers.py @@ -1,5 +1,5 @@ -from .manager import find_workflow_manager from .historian import find_historian +from .history import find_history class Alias: @@ -16,6 +16,6 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): def alias(alias: str) -> Alias: - manager = find_workflow_manager() - workflow_id = find_historian().workflow_id + manager = find_historian() + workflow_id = find_history().workflow_id return Alias(alias, manager, workflow_id) diff --git a/src/quest/persistence.py b/src/history/persistence.py similarity index 96% rename from src/quest/persistence.py rename to src/history/persistence.py index d944a763..0fa647a8 100644 --- a/src/quest/persistence.py +++ b/src/history/persistence.py @@ -5,8 +5,8 @@ from pathlib import Path from typing import Protocol, Union -from .history import History -from .quest_types import EventRecord +from .book import Book +from .history_types import EventRecord Blob = Union[dict, list, str, int, bool, float] @@ -21,7 +21,7 @@ def has_blob(self, key: str) -> bool: ... def delete_blob(self, key: str): ... -class PersistentHistory(History): +class PersistentList: def __init__(self, namespace: str, storage: BlobStorage): self._namespace = namespace self._storage = storage diff --git a/src/quest/resources.py b/src/history/resources.py similarity index 96% rename from src/quest/resources.py rename to src/history/resources.py index 841431b0..92f85d49 100644 --- a/src/quest/resources.py +++ b/src/history/resources.py @@ -1,6 +1,6 @@ import asyncio from typing import Callable, Coroutine -from .utils import quest_logger +from .utils import history_logger # noinspection PyProtectedMember @@ -26,13 +26,13 @@ def __init__(self, def __enter__(self): self._is_entered = True self._on_open(self) - quest_logger.debug(f'Resource stream opened for {id(self)}') + history_logger.debug(f'Resource stream opened for {id(self)}') return self def __exit__(self, exc_type, exc_value, traceback): self._is_entered = False self._on_close(self) - quest_logger.debug(f'Resource stream closed for {id(self)}') + history_logger.debug(f'Resource stream closed for {id(self)}') async def __aiter__(self): """ diff --git a/src/quest/serializer.py b/src/history/serializer.py similarity index 100% rename from src/quest/serializer.py rename to src/history/serializer.py diff --git a/src/quest/server.py b/src/history/server.py similarity index 90% rename from src/quest/server.py rename to src/history/server.py index 185f0e32..81d7d2ce 100644 --- a/src/quest/server.py +++ b/src/history/server.py @@ -3,11 +3,11 @@ import json import traceback from typing import Callable -from quest.utils import quest_logger +from history.utils import history_logger # TODO: Update websockets to use latest version import websockets from websockets import WebSocketServerProtocol -from quest import WorkflowManager +from history import Historian class MethodNotFoundException(Exception): @@ -24,7 +24,7 @@ async def serialize_resources(resources): class Server: - def __init__(self, manager: WorkflowManager, host: str, port: int, authorizer: Callable[[dict[str, str]], bool]): + def __init__(self, manager: Historian, host: str, port: int, authorizer: Callable[[dict[str, str]], bool]): """ Initialize the server. @@ -32,7 +32,7 @@ def __init__(self, manager: WorkflowManager, host: str, port: int, authorizer: C :param host: Host address for the server. :param port: Port for the server. """ - self._manager: WorkflowManager = manager + self._manager: Historian = manager self._host = host self._port = port self._authorizer = authorizer @@ -43,7 +43,7 @@ async def __aenter__(self): Start the server in an async with context. """ self._server = await websockets.serve(self.handler, self._host, self._port) - quest_logger.info(f'Server started at ws://{self._host}:{self._port}') + history_logger.info(f'Server started at ws://{self._host}:{self._port}') return self async def __aexit__(self, exc_type, exc_val, exc_tb): @@ -52,7 +52,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): """ self._server.close() await self._server.wait_closed() - quest_logger.info(f'Server at ws://{self._host}:{self._port} stopped') + history_logger.info(f'Server at ws://{self._host}:{self._port} stopped') async def handler(self, ws: WebSocketServerProtocol, path: str): """ @@ -65,7 +65,7 @@ async def handler(self, ws: WebSocketServerProtocol, path: str): await ws.close(reason="Unauthorized") return - quest_logger.info(f'New connection: {path}') + history_logger.info(f'New connection: {path}') if path == '/call': await self.handle_call(ws) elif path == '/stream': diff --git a/src/quest/utils.py b/src/history/utils.py similarity index 95% rename from src/quest/utils.py rename to src/history/utils.py index 889bbdfb..1d95ead8 100644 --- a/src/quest/utils.py +++ b/src/history/utils.py @@ -28,7 +28,7 @@ def __init__(self, name): logging.setLoggerClass(TaskFieldLogger) logging.getLogger().addFilter(TaskFieldFilter()) # Add filter on root logger -quest_logger = logging.getLogger('quest') # Create custom quest logger +history_logger = logging.getLogger('history') # Create custom history logger # Add filter on any existing loggers for logger_name in logging.root.manager.loggerDict.keys(): diff --git a/src/quest/versioning.py b/src/history/versioning.py similarity index 80% rename from src/quest/versioning.py rename to src/history/versioning.py index f37f4bbb..6afeabea 100644 --- a/src/quest/versioning.py +++ b/src/history/versioning.py @@ -1,7 +1,7 @@ import inspect from functools import wraps -from .historian import GLOBAL_VERSION, QUEST_VERSIONS, find_historian +from .history import GLOBAL_VERSION, HISTORY_VERSIONS, find_history DEFAULT_VERSION = '' @@ -21,11 +21,11 @@ async def _quest_versioned_function(*args, **kwargs): nonlocal versions_discovered nonlocal _quest_versions # save quest_versions in the local vars to lookup later if not versions_discovered: - find_historian()._discover_versions(func, _quest_versions) + find_history()._discover_versions(func, _quest_versions) versions_discovered = True return await func(*args, **kwargs) - setattr(_quest_versioned_function, QUEST_VERSIONS, _quest_versions) + setattr(_quest_versioned_function, HISTORY_VERSIONS, _quest_versions) return _quest_versioned_function return decorator @@ -35,7 +35,7 @@ def _get_qualified_version(version_name): outer_frame = inspect.currentframe() while outer_frame is not None: if outer_frame.f_code.co_name == '_quest_versioned_function' and \ - version_name in outer_frame.f_locals.get(QUEST_VERSIONS): + version_name in outer_frame.f_locals.get(HISTORY_VERSIONS): module_name = outer_frame.f_locals.get('func').__module__ func_name = outer_frame.f_locals.get('func').__qualname__ return module_name, func_name, version_name @@ -49,7 +49,7 @@ def _get_qualified_version(version_name): def get_version(version_name=GLOBAL_VERSION): module_name, func_name, version_name = _get_qualified_version(version_name) if version_name is not None: - return find_historian().get_version(module_name, func_name, version_name) or DEFAULT_VERSION + return find_history().get_version(module_name, func_name, version_name) or DEFAULT_VERSION # If I haven't found a version in the stack, then that version is not defined # The default version is "", which sorts BEFORE all other strings @@ -65,4 +65,4 @@ async def after_version(version=None, **versions): for version_name, version in versions.items(): module_name, func_name, version_name = _get_qualified_version(version_name) - await find_historian()._after_version(module_name, func_name, version_name, version) + await find_history()._after_version(module_name, func_name, version_name, version) diff --git a/src/quest/wrappers.py b/src/history/wrappers.py similarity index 89% rename from src/quest/wrappers.py rename to src/history/wrappers.py index 62851d94..06156fd4 100644 --- a/src/quest/wrappers.py +++ b/src/history/wrappers.py @@ -3,7 +3,7 @@ from functools import wraps from typing import Callable, Coroutine, TypeVar -from .historian import find_historian +from .history import find_history def _get_func_name(func) -> str: @@ -27,7 +27,7 @@ def step(func): @wraps(func) async def new_func(*args, **kwargs): - return await find_historian().handle_step(func_name, func, *args, **kwargs) + return await find_history().handle_step(func_name, func, *args, **kwargs) new_func._is_quest_step = True @@ -40,7 +40,7 @@ def task(func: Callable[..., Coroutine]) -> Callable[..., Task]: @wraps(func) def new_func(*args, **kwargs): - return find_historian().start_task(func, *args, **kwargs) + return find_history().start_task(func, *args, **kwargs) return new_func From 726fc91b607eb9a6f7c90e360dc03026a1227115 Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Wed, 2 Apr 2025 10:26:03 -0600 Subject: [PATCH 02/14] versioning fix --- history_test/pytest.ini | 4 +- history_test/test_step_concurrency.py | 2 +- history_test/test_versioning.py | 18 +++---- src/history/history.py | 73 +++++++++++++-------------- src/history/versioning.py | 6 +-- 5 files changed, 51 insertions(+), 52 deletions(-) diff --git a/history_test/pytest.ini b/history_test/pytest.ini index 4907d77b..5f5318e9 100644 --- a/history_test/pytest.ini +++ b/history_test/pytest.ini @@ -6,5 +6,5 @@ addopts = -m "not integration" log_format = %(asctime)s %(levelname)1s %(task)s %(message)s log_date_format = %Y-%m-%d %H:%M:%S log_level = INFO -;log_cli = true -;log_cli_level = DEBUG \ No newline at end of file +log_cli = true +log_cli_level = DEBUG \ No newline at end of file diff --git a/history_test/test_step_concurrency.py b/history_test/test_step_concurrency.py index e30f4770..c444a39c 100644 --- a/history_test/test_step_concurrency.py +++ b/history_test/test_step_concurrency.py @@ -129,7 +129,7 @@ async def test_long_fast_race(): await asyncio.sleep(1) await history.suspend() print('original', records) - print('_history', history._history) + print('_history', history._book) workflow = history.run() workflow_pause.set() diff --git a/history_test/test_versioning.py b/history_test/test_versioning.py index 53ba6225..c9aac569 100644 --- a/history_test/test_versioning.py +++ b/history_test/test_versioning.py @@ -17,16 +17,16 @@ async def application(): phrase.append(word) return phrase - history = [] - historian = History('test', application, history, serializer=NoopSerializer()) - historian.run() + book = [] + history = History('test', application, book, serializer=NoopSerializer()) + history.run() await asyncio.sleep(0.1) - await historian.record_external_event('words', None, 'put', 'foo') - await historian.record_external_event('words', None, 'put', 'bar') + await history.record_external_event('words', None, 'put', 'foo') + await history.record_external_event('words', None, 'put', 'bar') await asyncio.sleep(0.01) # Application shuts down - await historian.suspend() + await history.suspend() # New version of application is deployed @version(V2) @@ -41,10 +41,10 @@ async def application(): phrase.append(word + '2') return phrase - historian = History('test', application, history, serializer=NoopSerializer()) - result = historian.run() + history = History('test', application, book, serializer=NoopSerializer()) + result = history.run() await asyncio.sleep(0.1) - await historian.record_external_event('words', None, 'put', 'baz') + await history.record_external_event('words', None, 'put', 'baz') assert (await result) == ['foo', 'bar', 'baz2'] diff --git a/src/history/history.py b/src/history/history.py index e9e02ff7..a2907d65 100644 --- a/src/history/history.py +++ b/src/history/history.py @@ -19,7 +19,6 @@ deserialize_exception ) -HISTORY_VERSIONS = "_history_versions" GLOBAL_VERSION = "_global_version" # external replay: @@ -87,11 +86,11 @@ class _Wrapper: pass -def wrap_methods_as_history_events(resource: T, name: str, identity: str | None, historian: 'History', +def wrap_methods_as_history_events(resource: T, name: str, identity: str | None, history: 'History', internal=True) -> T: wrapper = _Wrapper() - historian_action = historian.handle_internal_event if internal else historian.record_external_event + history_action = history.handle_internal_event if internal else history.record_external_event for field in dir(resource): if field.startswith('_'): @@ -101,7 +100,7 @@ def wrap_methods_as_history_events(resource: T, name: str, identity: str | None, # Use default-value kwargs to force value binding instead of late binding @wraps(method) async def record(*args, _name=name, _identity=identity, _field=field, **kwargs): - return await historian_action(_name, _identity, _field, *args, **kwargs) + return await history_action(_name, _identity, _field, *args, **kwargs) setattr(wrapper, field, record) @@ -181,7 +180,7 @@ def _create_resource_id(name: str, identity: str | None) -> str: return f'{name}|{identity}' if identity is not None else name -historian_context = ContextVar('historian') +history_context = ContextVar('history') def get_function_name(func): @@ -199,7 +198,7 @@ def _get_exception_class(exception_type: str): class History: - def __init__(self, workflow_id: str, workflow: Callable, history: Book, serializer: StepSerializer): + def __init__(self, workflow_id: str, workflow: Callable, book: Book, serializer: StepSerializer): # TODO - change nomenclature (away from workflow)? Maybe just use workflow.__name__? self.workflow_id = workflow_id self.workflow = workflow @@ -210,7 +209,7 @@ def __init__(self, workflow_id: str, workflow: Callable, history: Book, serializ self._workflow_completed = False # These things need to be serialized - self._history: Book = history + self._book: Book = book self._serializer: StepSerializer = serializer @@ -242,7 +241,7 @@ def __init__(self, workflow_id: str, workflow: Callable, history: Book, serializ # See also external.py self._resources: dict[str, ResourceEntry] = {} - # This is the resource stream manager that handles calls to stream the historian's resources + # This is the resource stream manager that handles calls to stream the history's resources self._resource_stream_manager = ResourceStreamManager() # We keep track of all open tasks so we can properly suspend them @@ -264,9 +263,9 @@ def __init__(self, workflow_id: str, workflow: Callable, history: Book, serializ # Then we wait until the replay has completed (see _replay_complete) self._replay_started = asyncio.Event() - # The existing history is a copy of the initial history + # The existing book is a copy of the initial book # This ensures that we only replay the pre-existing records - self._existing_history = [] + self._existing_book = [] # Each task needs to replay its separate event records, # but the records are all interleaved. @@ -294,7 +293,7 @@ def _reset_replay(self): self._versions = {} - self._existing_history = list(self._history) + self._existing_book = list(self._book) self._resources = {} # The workflow ID is used as the task name for the root task @@ -309,7 +308,7 @@ def _reset_replay(self): # We add the workflow ID and the external task name # so that the root task (see run()) and the external event handler - # are both recognized as "belonging" to this historian + # are both recognized as "belonging" to this history # See also _get_task_name() self._replay_records = { self.workflow_id: None, @@ -318,7 +317,7 @@ def _reset_replay(self): self._record_gates = { _get_id(record): asyncio.Future() - for record in self._existing_history + for record in self._existing_book } for self._last_record_gate in self._record_gates.values(): @@ -380,7 +379,7 @@ async def _task_replay_records(self, task_id): self._task_replays[task_id] = task_replay """Yield the tasks for this task ID""" - for record in self._existing_history: + for record in self._existing_book: # If the record belongs to another task, we need to wait # for that other task to finish with the record # before we move on @@ -487,7 +486,7 @@ def _record_version_event(self, version_name, version): history_logger.debug(f'Version record: {version_name} = {version}') self._versions[version_name] = version - self._history.append(VersionRecord( + self._book.append(VersionRecord( type='set_version', timestamp=_get_current_timestamp(), step_id=version_name, @@ -505,7 +504,7 @@ async def _after_version(self, module_name, func_name, version_name, version): history_logger.debug(f'{self._get_task_name()} is waiting for version {version_name}=={version}') found = False - for record in self._existing_history: + for record in self._existing_book: if record['type'] == 'version' \ and record['version_name'] == version_name \ and record['version'] == version: @@ -523,7 +522,7 @@ async def _after_version(self, module_name, func_name, version_name, version): assert record['version'] == version, str(record) else: - self._history.append(VersionRecord( + self._book.append(VersionRecord( type='after_version', timestamp=_get_current_timestamp(), step_id='version', @@ -553,7 +552,7 @@ async def handle_step(self, func_name, func: Callable, *args, **kwargs): if next_record is None: history_logger.debug(f'{self._get_task_name()} starting step {func_name} with {args} and {kwargs}') - self._history.append(StepStartRecord( + self._book.append(StepStartRecord( type='start', timestamp=_get_current_timestamp(), task_id=self._get_task_name(), @@ -571,7 +570,7 @@ async def handle_step(self, func_name, func: Callable, *args, **kwargs): serialized_result = await self._serializer.serialize(result) - self._history.append(StepEndRecord( + self._book.append(StepEndRecord( type='end', timestamp=_get_current_timestamp(), task_id=self._get_task_name(), @@ -589,7 +588,7 @@ async def handle_step(self, func_name, func: Callable, *args, **kwargs): else: history_logger.exception(f'{step_id} canceled') serialized_exception = serialize_exception(cancel) - self._history.append(StepEndRecord( + self._book.append(StepEndRecord( type='end', timestamp=_get_current_timestamp(), task_id=self._get_task_name(), @@ -602,7 +601,7 @@ async def handle_step(self, func_name, func: Callable, *args, **kwargs): except Exception as ex: history_logger.exception(f'Error in {step_id}') serialized_exception = serialize_exception(ex) - self._history.append(StepEndRecord( + self._book.append(StepEndRecord( type='end', timestamp=_get_current_timestamp(), step_id=step_id, @@ -614,7 +613,7 @@ async def handle_step(self, func_name, func: Callable, *args, **kwargs): finally: if prune_on_exit: - _prune(step_id, self._history) + _prune(step_id, self._book) self._prefix[self._get_task_name()].pop(-1) async def record_external_event(self, name, identity, action, *args, **kwargs): @@ -634,7 +633,7 @@ async def record_external_event(self, name, identity, action, *args, **kwargs): else: result = function(*args, **kwargs) - self._history.append(ResourceAccessEvent( + self._book.append(ResourceAccessEvent( type='external', timestamp=_get_current_timestamp(), step_id=step_id, @@ -677,7 +676,7 @@ async def handle_internal_event(self, name, identity, action, *args, **kwargs): function = getattr(resource, action) if (next_record := await self._next_record()) is None: - self._history.append(ResourceAccessEvent( + self._book.append(ResourceAccessEvent( type='internal_start', timestamp=_get_current_timestamp(), step_id=step_id, @@ -703,7 +702,7 @@ async def handle_internal_event(self, name, identity, action, *args, **kwargs): result = function(*args, **kwargs) if (next_record := await self._next_record()) is None: - self._history.append(ResourceAccessEvent( + self._book.append(ResourceAccessEvent( type='internal_end', timestamp=_get_current_timestamp(), step_id=step_id, @@ -746,7 +745,7 @@ async def register_resource(self, name, identity, resource): ) if (next_record := await self._next_record()) is None: - self._history.append(ResourceLifecycleEvent( + self._book.append(ResourceLifecycleEvent( type='create_resource', timestamp=_get_current_timestamp(), step_id=step_id, @@ -775,7 +774,7 @@ async def delete_resource(self, name, identity, suspending=False): if not suspending: if (next_record := await self._next_record()) is None: - self._history.append(ResourceLifecycleEvent( + self._book.append(ResourceLifecycleEvent( type='delete_resource', timestamp=_get_current_timestamp(), step_id=step_id, @@ -791,7 +790,7 @@ async def delete_resource(self, name, identity, suspending=False): assert record['resource_id'] == resource_id def start_task(self, func, *args, name=None, task_factory=asyncio.create_task, **kwargs): - historian_context.set(self) + history_context.set(self) task_id = name or self._get_unique_id(get_function_name(func)) history_logger.debug(f'Requested {task_id} start') @@ -800,7 +799,7 @@ async def _func(*a, **kw): history_logger.debug(f'Starting task {task_id}') if (next_record := await self._next_record()) is None: - self._history.append(TaskEvent( + self._book.append(TaskEvent( type='start_task', timestamp=_get_current_timestamp(), step_id=task_id + '.start', @@ -814,7 +813,7 @@ async def _func(*a, **kw): result = await func(*a, **kw) if (next_record := await self._next_record()) is None: - self._history.append(TaskEvent( + self._book.append(TaskEvent( type='complete_task', timestamp=_get_current_timestamp(), step_id=task_id + '.complete', @@ -859,7 +858,7 @@ async def _run_with_exception_handling(self, *args, **kwargs): raise async def _run(self, *args, **kwargs): - historian_context.set(self) + history_context.set(self) task_name_getter.set(self._get_task_name) history_logger.debug(f'Running workflow {self.workflow_id}') self._add_new_configurations() @@ -893,7 +892,7 @@ async def _run(self, *args, **kwargs): def _clear_history(self, task): if self._workflow_completed: - self._history.clear() + self._book.clear() def run(self, *args, **kwargs): self._replay_started.clear() @@ -915,7 +914,7 @@ def configure(self, config_function, *args, **kwargs): def _add_new_configurations(self): config_records = [ record - for record in self._history + for record in self._book if record['type'] == 'configuration' ] @@ -931,7 +930,7 @@ def _add_new_configurations(self): for config_function, args, kwargs in self._configurations[len(config_records):]: history_logger.debug(f'Adding new configuration: {get_function_name(config_function)}(*{args}, **{kwargs}') - self._history.append(ConfigurationRecord( + self._book.append(ConfigurationRecord( type='configuration', timestamp=_get_current_timestamp(), step_id='configuration', @@ -996,12 +995,12 @@ async def _update_resource_stream(self, identity): await self._resource_stream_manager.update(identity) -class HistorianNotFoundException(Exception): +class HistoryNotFoundException(Exception): pass def find_history() -> History: - if (workflow := historian_context.get()) is not None: + if (workflow := history_context.get()) is not None: return workflow outer_frame = inspect.currentframe() @@ -1009,6 +1008,6 @@ def find_history() -> History: while not is_workflow: outer_frame = outer_frame.f_back if outer_frame is None: - raise HistorianNotFoundException("Historian object not found in event stack") + raise HistoryNotFoundException("History object not found in event stack") is_workflow = isinstance(outer_frame.f_locals.get('self'), History) return outer_frame.f_locals.get('self') diff --git a/src/history/versioning.py b/src/history/versioning.py index 6afeabea..a36923bc 100644 --- a/src/history/versioning.py +++ b/src/history/versioning.py @@ -1,7 +1,7 @@ import inspect from functools import wraps -from .history import GLOBAL_VERSION, HISTORY_VERSIONS, find_history +from .history import GLOBAL_VERSION, find_history DEFAULT_VERSION = '' @@ -25,7 +25,7 @@ async def _quest_versioned_function(*args, **kwargs): versions_discovered = True return await func(*args, **kwargs) - setattr(_quest_versioned_function, HISTORY_VERSIONS, _quest_versions) + setattr(_quest_versioned_function, '_quest_versions', _quest_versions) return _quest_versioned_function return decorator @@ -35,7 +35,7 @@ def _get_qualified_version(version_name): outer_frame = inspect.currentframe() while outer_frame is not None: if outer_frame.f_code.co_name == '_quest_versioned_function' and \ - version_name in outer_frame.f_locals.get(HISTORY_VERSIONS): + version_name in outer_frame.f_locals.get('_quest_versions'): module_name = outer_frame.f_locals.get('func').__module__ func_name = outer_frame.f_locals.get('func').__qualname__ return module_name, func_name, version_name From 5ba700156157bc35bc1672e8610a677b708aafd6 Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Thu, 3 Apr 2025 20:27:51 -0600 Subject: [PATCH 03/14] trying no location constraint --- src/history/extras/aws.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/history/extras/aws.py b/src/history/extras/aws.py index 3d71e56f..807fd042 100644 --- a/src/history/extras/aws.py +++ b/src/history/extras/aws.py @@ -38,7 +38,7 @@ def _prepare_bucket(self): if self._region: self._s3_client.create_bucket( Bucket=self._bucket_name, - CreateBucketConfiguration={'LocationConstraint': self._region} + # CreateBucketConfiguration={'LocationConstraint': self._region} ) else: self._s3_client.create_bucket(Bucket=self._bucket_name) @@ -107,7 +107,7 @@ def __init__(self): os.environ['AWS_REGION'] ) - self._table_name = 'quest_records' + self._table_name = 'history_records' self._dynamodb = self.session.resource('dynamodb') self._table = self._prepare_table() From a858d665a1f03156962f4a791136a5f6428ace7c Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Thu, 3 Apr 2025 20:34:45 -0600 Subject: [PATCH 04/14] trying no location constraint --- history_test/pytest.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/history_test/pytest.ini b/history_test/pytest.ini index 5f5318e9..4907d77b 100644 --- a/history_test/pytest.ini +++ b/history_test/pytest.ini @@ -6,5 +6,5 @@ addopts = -m "not integration" log_format = %(asctime)s %(levelname)1s %(task)s %(message)s log_date_format = %Y-%m-%d %H:%M:%S log_level = INFO -log_cli = true -log_cli_level = DEBUG \ No newline at end of file +;log_cli = true +;log_cli_level = DEBUG \ No newline at end of file From 79755fa4f211a1d79f612619661abe1d04acd8b3 Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Thu, 3 Apr 2025 20:38:50 -0600 Subject: [PATCH 05/14] try this? --- .github/workflows/run-pytest.yml | 2 ++ src/history/extras/aws.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index 40b6a112..a61d6b95 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -62,4 +62,6 @@ jobs: env: PYTHONPATH: ${{ github.workspace }}/src run: | + echo "############################ Full AWS environment variables:" + env | grep AWS pytest history_test -m "integration" -o log_cli=true \ No newline at end of file diff --git a/src/history/extras/aws.py b/src/history/extras/aws.py index 807fd042..5a612bef 100644 --- a/src/history/extras/aws.py +++ b/src/history/extras/aws.py @@ -38,7 +38,7 @@ def _prepare_bucket(self): if self._region: self._s3_client.create_bucket( Bucket=self._bucket_name, - # CreateBucketConfiguration={'LocationConstraint': self._region} + CreateBucketConfiguration={'LocationConstraint': self._region} ) else: self._s3_client.create_bucket(Bucket=self._bucket_name) From af253b41b26f567cd2bc5322d274a64a0d0845aa Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Thu, 3 Apr 2025 20:56:10 -0600 Subject: [PATCH 06/14] try this --- src/history/extras/aws.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/history/extras/aws.py b/src/history/extras/aws.py index 5a612bef..2adeb1ff 100644 --- a/src/history/extras/aws.py +++ b/src/history/extras/aws.py @@ -12,7 +12,7 @@ class S3Bucket: def __init__(self): - self._region = os.environ['AWS_REGION'] + self._region = os.environ.get('AWS_REGION', 'us-east-1') self.session = boto3.session.Session( aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], @@ -21,7 +21,7 @@ def __init__(self): ) self._bucket_name = 'history-records' - self._s3_client = self.session.client('s3') + self._s3_client = self.session.client('s3', region_name=self._region) self._prepare_bucket() @@ -35,14 +35,13 @@ def _prepare_bucket(self): try: self._s3_client.head_bucket(Bucket=self._bucket_name) except ClientError: - if self._region: + if self._region == 'us-east-1': + self._s3_client.create_bucket(Bucket=self._bucket_name) + else: self._s3_client.create_bucket( Bucket=self._bucket_name, CreateBucketConfiguration={'LocationConstraint': self._region} ) - else: - self._s3_client.create_bucket(Bucket=self._bucket_name) - class S3BlobStorage(BlobStorage): def __init__(self, name, s3_client, bucket_name): From b9e839990d48f1e9fcc22d819a916310466f899f Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Thu, 3 Apr 2025 21:01:44 -0600 Subject: [PATCH 07/14] change bucketname --- src/history/extras/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/history/extras/aws.py b/src/history/extras/aws.py index 2adeb1ff..38166d9d 100644 --- a/src/history/extras/aws.py +++ b/src/history/extras/aws.py @@ -20,7 +20,7 @@ def __init__(self): region_name=self._region ) - self._bucket_name = 'history-records' + self._bucket_name = 'history-records-testing-that-this-is-the-issue' self._s3_client = self.session.client('s3', region_name=self._region) self._prepare_bucket() From d15cf42f00bc5d56297fdba7f47b78fff090895a Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Fri, 4 Apr 2025 11:27:00 -0600 Subject: [PATCH 08/14] update with bucketname parameter --- history_test/test_persistence.py | 2 +- src/history/extras/aws.py | 50 +++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/history_test/test_persistence.py b/history_test/test_persistence.py index ac5c1e89..ebaf75da 100644 --- a/history_test/test_persistence.py +++ b/history_test/test_persistence.py @@ -79,7 +79,7 @@ def __exit__(self, *args): class S3StorageContext: def __enter__(self): - s3 = S3Bucket() + s3 = S3Bucket(bucket_name='quest-records') storage = S3BlobStorage('test', s3.get_s3_client(), s3.get_bucket_name()) return storage diff --git a/src/history/extras/aws.py b/src/history/extras/aws.py index 38166d9d..4c78ff05 100644 --- a/src/history/extras/aws.py +++ b/src/history/extras/aws.py @@ -2,6 +2,7 @@ import json from .. import BlobStorage, Blob, Historian, WorkflowFactory, PersistentList, Book +from history.utils import history_logger try: import boto3 @@ -11,7 +12,7 @@ class S3Bucket: - def __init__(self): + def __init__(self, bucket_name): self._region = os.environ.get('AWS_REGION', 'us-east-1') self.session = boto3.session.Session( aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], @@ -20,7 +21,7 @@ def __init__(self): region_name=self._region ) - self._bucket_name = 'history-records-testing-that-this-is-the-issue' + self._bucket_name = bucket_name # TODO: What should we do here? Also, fix this self._s3_client = self.session.client('s3', region_name=self._region) self._prepare_bucket() @@ -31,17 +32,45 @@ def get_s3_client(self): def get_bucket_name(self): return self._bucket_name + # def _prepare_bucket(self): + # try: + # self._s3_client.head_bucket(Bucket=self._bucket_name) + # except ClientError: + # if self._region == 'us-east-1': + # self._s3_client.create_bucket(Bucket=self._bucket_name) + # else: + # self._s3_client.create_bucket( + # Bucket=self._bucket_name, + # CreateBucketConfiguration={'LocationConstraint': self._region} + # ) + def _prepare_bucket(self): try: self._s3_client.head_bucket(Bucket=self._bucket_name) - except ClientError: - if self._region == 'us-east-1': - self._s3_client.create_bucket(Bucket=self._bucket_name) + history_logger.debug(f"Bucket '{self._bucket_name}' exists and is accessible.") + except self._s3_client.exceptions.NoSuchBucket: + history_logger.debug(f"Bucket '{self._bucket_name}' does not exist. Creating it...") + self._create_bucket() + except self._s3_client.exceptions.ClientError as e: + error_code = e.response["Error"]["Code"] + if error_code == "403": + raise PermissionError(f"Access denied to bucket '{self._bucket_name}'. Check IAM policies.") + elif error_code == "404": + history_logger.debug(f"Bucket '{self._bucket_name}' does not exist. Creating it...") + self._create_bucket() else: - self._s3_client.create_bucket( - Bucket=self._bucket_name, - CreateBucketConfiguration={'LocationConstraint': self._region} - ) + raise RuntimeError(f"Unexpected error checking bucket: {e}") + + def _create_bucket(self): + """ Creates the S3 bucket with proper region handling """ + if self._region == "us-east-1": + self._s3_client.create_bucket(Bucket=self._bucket_name) + else: + self._s3_client.create_bucket( + Bucket=self._bucket_name, + CreateBucketConfiguration={"LocationConstraint": self._region} + ) + history_logger.debug(f"Bucket '{self._bucket_name}' successfully created in {self._region}.") class S3BlobStorage(BlobStorage): def __init__(self, name, s3_client, bucket_name): @@ -86,8 +115,9 @@ def delete_blob(self, key: str): def create_s3_manager( namespace: str, factory: WorkflowFactory, + bucket_name ) -> Historian: - s3 = S3Bucket() + s3 = S3Bucket(bucket_name=bucket_name) storage = S3BlobStorage(namespace, s3.get_s3_client(), s3.get_bucket_name()) From 168ae09d416870159229144baf8d670368b6c976 Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Fri, 4 Apr 2025 17:09:31 -0600 Subject: [PATCH 09/14] fix src import --- history_test/test_basic.py | 8 ++++---- history_test/test_context.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/history_test/test_basic.py b/history_test/test_basic.py index 4209f7b3..d05b3073 100644 --- a/history_test/test_basic.py +++ b/history_test/test_basic.py @@ -26,15 +26,15 @@ async def workflow(name): @pytest.mark.asyncio @timeout(3) async def test_basic_workflow(): - history = [] - historian = History( + in_memory_list = [] + history = History( 'test', workflow, - history, + in_memory_list, serializer=NoopSerializer() ) - result = await historian.run('world') + result = await history.run('world') assert result == 'Hello world' diff --git a/history_test/test_context.py b/history_test/test_context.py index 64f9e1aa..b921f4de 100644 --- a/history_test/test_context.py +++ b/history_test/test_context.py @@ -1,6 +1,6 @@ """import pytest -from src.history import these +from history import these class Context: From a8b5f5ffcaaa64ed260898703d5ea6232f9b08d5 Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Mon, 7 Apr 2025 10:36:39 -0600 Subject: [PATCH 10/14] error handling --- history_test/test_persistence.py | 2 +- src/history/extras/aws.py | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/history_test/test_persistence.py b/history_test/test_persistence.py index ebaf75da..874e08ac 100644 --- a/history_test/test_persistence.py +++ b/history_test/test_persistence.py @@ -79,7 +79,7 @@ def __exit__(self, *args): class S3StorageContext: def __enter__(self): - s3 = S3Bucket(bucket_name='quest-records') + s3 = S3Bucket(bucket_name='history-records') storage = S3BlobStorage('test', s3.get_s3_client(), s3.get_bucket_name()) return storage diff --git a/src/history/extras/aws.py b/src/history/extras/aws.py index 4c78ff05..ef3bcfd7 100644 --- a/src/history/extras/aws.py +++ b/src/history/extras/aws.py @@ -52,14 +52,9 @@ def _prepare_bucket(self): history_logger.debug(f"Bucket '{self._bucket_name}' does not exist. Creating it...") self._create_bucket() except self._s3_client.exceptions.ClientError as e: - error_code = e.response["Error"]["Code"] - if error_code == "403": - raise PermissionError(f"Access denied to bucket '{self._bucket_name}'. Check IAM policies.") - elif error_code == "404": - history_logger.debug(f"Bucket '{self._bucket_name}' does not exist. Creating it...") - self._create_bucket() - else: - raise RuntimeError(f"Unexpected error checking bucket: {e}") + raise PermissionError(f"Access denied to bucket or bucket does not exist '{self._bucket_name}'. Check IAM policies or region.") + except Exception as e: + raise RuntimeError(f"An error occurred while accessing the bucket '{self._bucket_name}': {e}") def _create_bucket(self): """ Creates the S3 bucket with proper region handling """ From 8d59a22ace739adf953bfca4c31f8c50ea02548c Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Mon, 7 Apr 2025 10:37:59 -0600 Subject: [PATCH 11/14] it works --- history_test/test_persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/history_test/test_persistence.py b/history_test/test_persistence.py index 874e08ac..ebaf75da 100644 --- a/history_test/test_persistence.py +++ b/history_test/test_persistence.py @@ -79,7 +79,7 @@ def __exit__(self, *args): class S3StorageContext: def __enter__(self): - s3 = S3Bucket(bucket_name='history-records') + s3 = S3Bucket(bucket_name='quest-records') storage = S3BlobStorage('test', s3.get_s3_client(), s3.get_bucket_name()) return storage From e8df2f58512d604426f3ae952aa318a25fae96c0 Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Mon, 7 Apr 2025 10:39:30 -0600 Subject: [PATCH 12/14] clean up wf --- .github/workflows/run-pytest.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index a61d6b95..40b6a112 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -62,6 +62,4 @@ jobs: env: PYTHONPATH: ${{ github.workspace }}/src run: | - echo "############################ Full AWS environment variables:" - env | grep AWS pytest history_test -m "integration" -o log_cli=true \ No newline at end of file From fd0d05c67edc361dba093546752191e3a81ce3ca Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Wed, 9 Apr 2025 10:43:47 -0600 Subject: [PATCH 13/14] test changes --- history_test/test_alias.py | 40 +++--- history_test/test_basic.py | 34 ++--- history_test/test_basic_tasks.py | 20 +-- history_test/test_configuration.py | 32 ++--- history_test/test_external_actions.py | 122 +++++++++--------- history_test/test_get_result.py | 48 +++---- .../{test_manager.py => test_historian.py} | 108 ++++++++-------- history_test/test_interruptions.py | 12 +- history_test/test_persistence.py | 44 +++---- history_test/test_resource_stream.py | 104 +++++++-------- history_test/test_serializer.py | 10 +- history_test/test_step_concurrency.py | 8 +- history_test/test_step_error.py | 28 ++-- history_test/test_suspend.py | 12 +- history_test/test_versioning.py | 36 +++--- history_test/test_websockets.py | 10 +- history_test/test_workflow_metrics.py | 44 +++---- history_test/test_wrappers.py | 18 +-- 18 files changed, 365 insertions(+), 365 deletions(-) rename history_test/{test_manager.py => test_historian.py} (54%) diff --git a/history_test/test_alias.py b/history_test/test_alias.py index 58f7070d..0dee054f 100644 --- a/history_test/test_alias.py +++ b/history_test/test_alias.py @@ -27,22 +27,22 @@ async def workflow(): 'workflow': workflow } - async with create_in_memory_historian(workflows) as manager: - manager.start_soon('workflow', 'wid') + async with create_in_memory_historian(workflows) as historian: + historian.start_soon('workflow', 'wid') await asyncio.sleep(0.1) - await manager.send_event('wid', 'data', None, 'put', '1') + await historian.send_event('wid', 'data', None, 'put', '1') await asyncio.sleep(0.1) assert '1' in data - await manager.send_event('the_foo', 'data', None, 'put', 'foo') + await historian.send_event('the_foo', 'data', None, 'put', 'foo') first_pause.set() await asyncio.sleep(0.1) assert 'foo' in data - await manager.send_event('wid', 'data', None, 'put', '2') + await historian.send_event('wid', 'data', None, 'put', '2') second_pause.set() await asyncio.sleep(0.1) @@ -83,21 +83,21 @@ async def workflow_b(): 'workflow_b': workflow_b, } - async with create_in_memory_historian(workflows) as manager: + async with create_in_memory_historian(workflows) as historian: # Gather resources - manager.start_soon('workflow_a', 'wid_a') + historian.start_soon('workflow_a', 'wid_a') await asyncio.sleep(0.1) - manager.start_soon('workflow_b', 'wid_b') + historian.start_soon('workflow_b', 'wid_b') await asyncio.sleep(0.1) first_pause.set() - await manager.send_event('wid_a', 'data', None, 'put', 'data a 1') - await manager.send_event('wid_b', 'data', None, 'put', 'data b 1') - await manager.send_event('the_foo', 'data', None, 'put', 'data foo 1') + await historian.send_event('wid_a', 'data', None, 'put', 'data a 1') + await historian.send_event('wid_b', 'data', None, 'put', 'data b 1') + await historian.send_event('the_foo', 'data', None, 'put', 'data foo 1') await asyncio.sleep(0.1) # yield to the workflows # now both should be waiting on second gate and no one should be the foo - assert not manager.has('the_foo') + assert not historian.has('the_foo') assert 'data a 1' in data_a assert 'data foo 1' in data_a assert 'data b 1' in data_b @@ -106,9 +106,9 @@ async def workflow_b(): await asyncio.sleep(0.1) # yield # now workflow b should be the foo - await manager.send_event('wid_a', 'data', None, 'put', 'data a 2') - await manager.send_event('wid_b', 'data', None, 'put', 'data b 2') - await manager.send_event('the_foo', 'data', None, 'put', 'data foo 2') + await historian.send_event('wid_a', 'data', None, 'put', 'data a 2') + await historian.send_event('wid_b', 'data', None, 'put', 'data b 2') + await historian.send_event('the_foo', 'data', None, 'put', 'data foo 2') third_pause.set() await asyncio.sleep(0.1) # yield to the workflows @@ -138,13 +138,13 @@ async def workflow_b(): 'workflow_a': workflow_a, 'workflow_b': workflow_b, } - async with create_in_memory_historian(workflows) as manager: - manager.start_soon('workflow_a', 'wid1', delete_on_finish=False) - manager.start_soon('workflow_b', 'wid2', delete_on_finish=False) + async with create_in_memory_historian(workflows) as historian: + historian.start_soon('workflow_a', 'wid1', delete_on_finish=False) + historian.start_soon('workflow_b', 'wid2', delete_on_finish=False) await asyncio.sleep(0.1) pause.set() await asyncio.sleep(0.1) # Allow workflows to finish - result_wid1 = await manager.get_result('wid1', delete=True) - result_wid2 = await manager.get_result('wid2', delete=True) + result_wid1 = await historian.get_result('wid1', delete=True) + result_wid2 = await historian.get_result('wid2', delete=True) diff --git a/history_test/test_basic.py b/history_test/test_basic.py index d05b3073..313c3277 100644 --- a/history_test/test_basic.py +++ b/history_test/test_basic.py @@ -75,25 +75,25 @@ async def longer_workflow(text): @pytest.mark.asyncio @timeout(3) async def test_resume(): - history = [] - historian = History( + book = [] + history = History( 'test', longer_workflow, - history, + book, serializer=NoopSerializer() ) - workflow = historian.run('abc') + workflow = history.run('abc') await asyncio.sleep(0.01) - await historian.suspend() + await history.suspend() - assert history # should not be empty + assert book # should not be empty # Allow workflow to proceed block_workflow.set() # Start the workflow again - result = await historian.run('abc') + result = await history.run('abc') assert result == 'abcabcfooabcabcfoo' assert double_calls == 2 @@ -134,20 +134,20 @@ async def nested_workflow(text1, text2): @pytest.mark.asyncio @timeout(3) async def test_nested_steps_resume(): - history = [] - historian = History( + book = [] + history = History( 'test', nested_workflow, - history, + book, serializer=NoopSerializer() ) - workflow = historian.run('abc', 'xyz') + workflow = history.run('abc', 'xyz') await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() pause.set() - result = await historian.run() + result = await history.run() assert result == 'foofooabcbarfooxyzbar' @@ -170,19 +170,19 @@ async def dance(start): @pytest.mark.asyncio @timeout(3) async def test_resume_mid_step(): - historian = History( + history = History( 'test', dance, [], serializer=NoopSerializer() ) - wtask = historian.run(1) + wtask = history.run(1) await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() stop.set() - wtask = historian.run(1) + wtask = history.run(1) await asyncio.sleep(0.1) assert await wtask == 3 diff --git a/history_test/test_basic_tasks.py b/history_test/test_basic_tasks.py index 507d6431..197cb976 100644 --- a/history_test/test_basic_tasks.py +++ b/history_test/test_basic_tasks.py @@ -43,18 +43,18 @@ async def test_basic_tasks(): counters['basic_tasks'] = 0 pauses['basic_tasks'] = asyncio.Event() - history = [] - historian = History( + book = [] + history = History( 'test', sub_task_workflow, - history, + book, serializer=NoopSerializer() ) # Don't pause pauses['basic_tasks'].set() - result = await historian.run('abc', 'xyz', 'basic_tasks') + result = await history.run('abc', 'xyz', 'basic_tasks') assert counters['basic_tasks'] == 4 assert result == 'foofooabcbarbarfoofooxyzbarbar' @@ -67,25 +67,25 @@ async def test_basic_tasks_resume(): counters['tasks_resume'] = 0 pauses['tasks_resume'] = asyncio.Event() - history = [] - historian = History( + book = [] + history = History( 'test', sub_task_workflow, - history, + book, serializer=NoopSerializer() ) # Will run and block on the event - workflow = historian.run('abc', 'xyz', 'tasks_resume') + workflow = history.run('abc', 'xyz', 'tasks_resume') await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() # Both subtasks should have run the first foobar assert counters['tasks_resume'] == 2 # Don't pause this time pauses['tasks_resume'].set() - result = await historian.run() + result = await history.run() assert counters['tasks_resume'] == 4 assert result == 'foofooabcbarbarfoofooxyzbarbar' diff --git a/history_test/test_configuration.py b/history_test/test_configuration.py index 90147b9b..0c9a490a 100644 --- a/history_test/test_configuration.py +++ b/history_test/test_configuration.py @@ -50,19 +50,19 @@ async def configure_state(config): 'foos': ['a', 'b'] } - history = [] - historian = History('test', application, history, serializer=NoopSerializer()) - historian.configure(configure_state, config1) - historian.run() + book = [] + history = History('test', application, book, serializer=NoopSerializer()) + history.configure(configure_state, config1) + history.run() await asyncio.sleep(0.1) - await historian.record_external_event('foobar', None, 'put', 'a') - await historian.record_external_event('foobar', None, 'put', 'b') - await historian.record_external_event('foobar', None, 'put', 'c') + await history.record_external_event('foobar', None, 'put', 'a') + await history.record_external_event('foobar', None, 'put', 'b') + await history.record_external_event('foobar', None, 'put', 'c') await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() # Application reboots with new config # 'b' is no longer a foo, and should be moved to bar @@ -70,19 +70,19 @@ async def configure_state(config): 'foos': ['a'] } - historian = History('test', application, history, serializer=NoopSerializer()) - historian.configure(configure_state, config1) - historian.configure(configure_state, config2) - historian.run() + history = History('test', application, book, serializer=NoopSerializer()) + history.configure(configure_state, config1) + history.configure(configure_state, config2) + history.run() await asyncio.sleep(0.1) - await historian.record_external_event('foobar', None, 'put', 'a') - await historian.record_external_event('foobar', None, 'put', 'b') - await historian.record_external_event('foobar', None, 'put', 'c') + await history.record_external_event('foobar', None, 'put', 'a') + await history.record_external_event('foobar', None, 'put', 'b') + await history.record_external_event('foobar', None, 'put', 'c') await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() assert messages == [ 'a is foo', diff --git a/history_test/test_external_actions.py b/history_test/test_external_actions.py index fa9e0678..e8114057 100644 --- a/history_test/test_external_actions.py +++ b/history_test/test_external_actions.py @@ -49,23 +49,23 @@ async def state_workflow(identity): assert await name.get() == 'Barbaz' identity = 'foo_ident' - historian = History('test', state_workflow, [], serializer=NoopSerializer()) - workflow = historian.run(identity) - await wait_for(historian) + history = History('test', state_workflow, [], serializer=NoopSerializer()) + workflow = history.run(identity) + await wait_for(history) # Observe state - resources = await historian.get_resources(None) # i.e. public resources + resources = await history.get_resources(None) # i.e. public resources assert not resources # should be empty - resources = await historian.get_resources(identity) + resources = await history.get_resources(identity) assert ('name', 'foo_ident') in resources - name = wrap_as_state('name', 'foo_ident', historian) + name = wrap_as_state('name', 'foo_ident', history) assert await name.value() == 'Foobar' # Set state await name.set('Barbaz') - resources = await historian.get_resources(identity) + resources = await history.get_resources(identity) assert ('name', 'foo_ident') in resources assert await name.value() == 'Barbaz' @@ -88,20 +88,20 @@ async def workflow_with_queue(identity): @timeout(3) async def test_external_queue(): identity = 'foo_ident' - historian = History( + history = History( 'test', workflow_with_queue, [], serializer=NoopSerializer() ) - workflow = historian.run(identity) - await wait_for(historian) + workflow = history.run(identity) + await wait_for(history) - resources = await historian.get_resources(None) - items = wrap_as_queue('items', 'foo_ident', historian) + resources = await history.get_resources(None) + items = wrap_as_queue('items', 'foo_ident', history) assert not resources - resources = await historian.get_resources(identity) + resources = await history.get_resources(identity) assert ('items', 'foo_ident') in resources await items.put(7) @@ -139,31 +139,31 @@ async def queue_task_workflow(id1, id2): async def test_queue_tasks(): id_foo = 'FOO' id_bar = 'BAR' - historian = History( + history = History( 'test', queue_task_workflow, [], serializer=NoopSerializer() ) - workflow = historian.run(id_foo, id_bar) - await wait_for(historian) + workflow = history.run(id_foo, id_bar) + await wait_for(history) - resources = await historian.get_resources(id_foo) + resources = await history.get_resources(id_foo) assert ('foo', 'FOO') in resources assert ('foo_done', 'FOO') in resources - resources = await historian.get_resources(id_bar) + resources = await history.get_resources(id_bar) assert ('foo', 'BAR') in resources assert ('foo_done', 'BAR') in resources - await historian.record_external_event('foo', id_bar, 'put', 4) - await historian.record_external_event('foo', id_foo, 'put', 1) - await historian.record_external_event('foo', id_foo, 'put', 2) - await historian.record_external_event('foo', id_bar, 'put', 5) - await historian.record_external_event('foo_done', id_bar, 'set') - await historian.record_external_event('foo', id_foo, 'put', 3) - await historian.record_external_event('foo_done', id_foo, 'set') + await history.record_external_event('foo', id_bar, 'put', 4) + await history.record_external_event('foo', id_foo, 'put', 1) + await history.record_external_event('foo', id_foo, 'put', 2) + await history.record_external_event('foo', id_bar, 'put', 5) + await history.record_external_event('foo_done', id_bar, 'set') + await history.record_external_event('foo', id_foo, 'put', 3) + await history.record_external_event('foo_done', id_foo, 'set') assert await workflow == [1, 2, 3, 4, 5] @@ -190,22 +190,22 @@ async def workflow_nested_tasks(): @pytest.mark.asyncio @timeout(3) async def test_nested_tasks(): - historian = History( + history = History( 'test', workflow_nested_tasks, [], serializer=NoopSerializer() ) - workflow = historian.run() - await wait_for(historian) + workflow = history.run() + await wait_for(history) - await historian.record_external_event('the_queue', None, 'put', 1) - await historian.suspend() + await history.record_external_event('the_queue', None, 'put', 1) + await history.suspend() - new_workflow = historian.run() + new_workflow = history.run() await asyncio.sleep(1) - await historian.record_external_event('the_queue', None, 'put', 2) + await history.record_external_event('the_queue', None, 'put', 2) assert await new_workflow == 3 @@ -219,48 +219,48 @@ async def test_nested_tasks(): async def test_queue_tasks_resume(): id_foo = 'FOO' id_bar = 'BAR' - history = [] - historian = History( + book = [] + history = History( 'test', queue_task_workflow, - history, + book, serializer=NoopSerializer() ) - workflow = historian.run(id_foo, id_bar) - await wait_for(historian) + workflow = history.run(id_foo, id_bar) + await wait_for(history) - resources = await historian.get_resources(id_foo) + resources = await history.get_resources(id_foo) assert ('foo', 'FOO') in resources assert ('foo_done', 'FOO') in resources - resources = await historian.get_resources(id_bar) + resources = await history.get_resources(id_bar) assert ('foo', 'BAR') in resources assert ('foo_done', 'BAR') in resources - await historian.record_external_event('foo', id_bar, 'put', 4) - await historian.record_external_event('foo', id_foo, 'put', 1) - await historian.record_external_event('foo', id_foo, 'put', 2) + await history.record_external_event('foo', id_bar, 'put', 4) + await history.record_external_event('foo', id_foo, 'put', 1) + await history.record_external_event('foo', id_foo, 'put', 2) - await historian.suspend() + await history.suspend() # Start it over - workflow = historian.run(id_foo, id_bar) - await wait_for(historian) + workflow = history.run(id_foo, id_bar) + await wait_for(history) await asyncio.sleep(1) - resources = await historian.get_resources(id_foo) + resources = await history.get_resources(id_foo) assert ('foo', 'FOO') in resources assert ('foo_done', 'FOO') in resources - resources = await historian.get_resources(id_bar) + resources = await history.get_resources(id_bar) assert ('foo', 'BAR') in resources assert ('foo_done', 'BAR') in resources - await historian.record_external_event('foo', id_bar, 'put', 5) - await historian.record_external_event('foo_done', id_bar, 'set') - await historian.record_external_event('foo', id_foo, 'put', 3) - await historian.record_external_event('foo_done', id_foo, 'set') + await history.record_external_event('foo', id_bar, 'put', 5) + await history.record_external_event('foo_done', id_bar, 'set') + await history.record_external_event('foo', id_foo, 'put', 3) + await history.record_external_event('foo_done', id_foo, 'set') assert await workflow == [1, 2, 3, 4, 5] @@ -283,24 +283,24 @@ async def interactive_process_with_steps(): async def test_step_specific_external(): """ When an external event occurs on a resources that is specific to the step, - and the step history is pruned after the step completes, + and the step book is pruned after the step completes, then the external event on the now-obsolete resource must also be pruned. """ - history = [] - historian = History('test', interactive_process_with_steps, history, serializer=NoopSerializer()) - historian.run() + book = [] + history = History('test', interactive_process_with_steps, book, serializer=NoopSerializer()) + history.run() await asyncio.sleep(0.1) - resources = await historian.get_resources(None) + resources = await history.get_resources(None) assert ('the-queue', None) in resources - await historian.record_external_event('the-queue', None, 'put', 1) + await history.record_external_event('the-queue', None, 'put', 1) await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() - workflow = historian.run() + workflow = history.run() await asyncio.sleep(0.1) - resources = await historian.get_resources(None) + resources = await history.get_resources(None) assert ('the-queue', None) in resources - await historian.record_external_event('the-queue', None, 'put', 2) + await history.record_external_event('the-queue', None, 'put', 2) assert (await workflow) == 3 diff --git a/history_test/test_get_result.py b/history_test/test_get_result.py index b85ab398..028bab34 100644 --- a/history_test/test_get_result.py +++ b/history_test/test_get_result.py @@ -16,14 +16,14 @@ async def test_basic_store_result(): async def workflow1(): return "done" - async with create_in_memory_historian({'w1': workflow1}) as manager: - manager.start_soon('w1', 'wid1', delete_on_finish=False) + async with create_in_memory_historian({'w1': workflow1}) as historian: + historian.start_soon('w1', 'wid1', delete_on_finish=False) await asyncio.sleep(0.1) - assert await manager.get_result('wid1') == 'done' - assert manager.has('wid1') - assert await manager.get_result('wid1', delete=True) == 'done' - assert not manager.has('wid1') + assert await historian.get_result('wid1') == 'done' + assert historian.has('wid1') + assert await historian.get_result('wid1', delete=True) == 'done' + assert not historian.has('wid1') @pytest.mark.asyncio @@ -32,10 +32,10 @@ async def test_workflows_not_saved_have_no_results(): async def workflow1(): return "done" - async with create_in_memory_historian({'w1': workflow1}) as manager: - manager.start_soon('w1', 'wid1') + async with create_in_memory_historian({'w1': workflow1}) as historian: + historian.start_soon('w1', 'wid1') await asyncio.sleep(0.1) - assert not manager.has('wid1') + assert not historian.has('wid1') @pytest.mark.asyncio @@ -44,13 +44,13 @@ async def test_get_result_on_missing_workflow_raises(): async def workflow1(): return "done" - async with create_in_memory_historian({'w1': workflow1}) as manager: - manager.start_soon('w1', 'wid1') + async with create_in_memory_historian({'w1': workflow1}) as historian: + historian.start_soon('w1', 'wid1') await asyncio.sleep(0.1) - assert not manager.has('wid1') + assert not historian.has('wid1') with pytest.raises(WorkflowNotFound): - await manager.get_result('wid1') + await historian.get_result('wid1') @pytest.mark.asyncio @@ -65,13 +65,13 @@ async def sample_workflow(): workflows = { "sample_workflow": sample_workflow } - manager = create_in_memory_historian(workflows=workflows) + historian = create_in_memory_historian(workflows=workflows) - async with manager: - manager.start_soon('sample_workflow', 'wid1') + async with historian: + historian.start_soon('sample_workflow', 'wid1') await asyncio.sleep(0.1) - get_result_task = asyncio.create_task(manager.get_result('wid1')) + get_result_task = asyncio.create_task(historian.get_result('wid1')) gate.set() result = await get_result_task @@ -83,18 +83,18 @@ async def test_exception_store_result(): async def workflow1(): raise OurException('died') - async with create_in_memory_historian({'w1': workflow1}) as manager: - manager.start_soon('w1', 'wid1', delete_on_finish=False) + async with create_in_memory_historian({'w1': workflow1}) as historian: + historian.start_soon('w1', 'wid1', delete_on_finish=False) await asyncio.sleep(0.1) - assert manager.has('wid1') + assert historian.has('wid1') with pytest.raises(OurException): - await manager.get_result('wid1') + await historian.get_result('wid1') - assert manager.has('wid1') + assert historian.has('wid1') with pytest.raises(OurException): - await manager.get_result('wid1', delete=True) + await historian.get_result('wid1', delete=True) - assert not manager.has('wid1') + assert not historian.has('wid1') diff --git a/history_test/test_manager.py b/history_test/test_historian.py similarity index 54% rename from history_test/test_manager.py rename to history_test/test_historian.py index 20a44465..e6df0795 100644 --- a/history_test/test_manager.py +++ b/history_test/test_historian.py @@ -9,14 +9,14 @@ @pytest.mark.asyncio -async def test_manager(): +async def test_historian(): storage = InMemoryBlobStorage() - histories = {} + books = {} - def create_history(wid: str): - if wid not in histories: - histories[wid] = PersistentList(wid, InMemoryBlobStorage()) - return histories[wid] + def create_book(wid: str): + if wid not in books: + books[wid] = PersistentList(wid, InMemoryBlobStorage()) + return books[wid] pause = asyncio.Event() counter_a = 0 @@ -34,22 +34,22 @@ async def workflow(arg): return 7 + arg - async with Historian('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: - manager.start_soon('workflow', 'wid1', 4, delete_on_finish=False) + async with Historian('test-manager', storage, create_book, lambda w_type: workflow, + serializer=NoopSerializer()) as historian: + historian.start_soon('workflow', 'wid1', 4, delete_on_finish=False) await asyncio.sleep(0.1) - # Now pause the manager and all workflows + # Now pause the historian and all workflows - assert 'wid1' in histories + assert 'wid1' in books assert counter_a == 1 assert counter_b == 0 - async with Historian('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: + async with Historian('test-manager', storage, create_book, lambda w_type: workflow, + serializer=NoopSerializer()) as historian: # At this point, all workflows should be resumed pause.set() await asyncio.sleep(0.1) - result = await manager.get_result('wid1') + result = await historian.get_result('wid1') assert result == 11 assert counter_a == 2 @@ -57,14 +57,14 @@ async def workflow(arg): @pytest.mark.asyncio -async def test_manager_events(): +async def test_historian_events(): storage = InMemoryBlobStorage() - histories = {} + books = {} - def create_history(wid: str): - if wid not in histories: - histories[wid] = PersistentList(wid, InMemoryBlobStorage()) - return histories[wid] + def create_book(wid: str): + if wid not in books: + books[wid] = PersistentList(wid, InMemoryBlobStorage()) + return books[wid] counter_a = 0 counter_b = 0 @@ -87,25 +87,25 @@ async def workflow(arg: int): total += message - async with Historian('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: - manager.start_soon('workflow', 'wid1', 1, delete_on_finish=False) + async with Historian('test-manager', storage, create_book, lambda w_type: workflow, + serializer=NoopSerializer()) as historian: + historian.start_soon('workflow', 'wid1', 1, delete_on_finish=False) await asyncio.sleep(0.1) - await manager.send_event('wid1', 'messages', None, 'put', 2) + await historian.send_event('wid1', 'messages', None, 'put', 2) await asyncio.sleep(0.1) # Now pause the manager and all workflows - assert 'wid1' in histories + assert 'wid1' in books assert counter_a == 1 assert counter_b == 1 - async with Historian('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: + async with Historian('test-manager', storage, create_book, lambda w_type: workflow, + serializer=NoopSerializer()) as historian: # At this point, all workflows should be resumed await asyncio.sleep(0.1) - await manager.send_event('wid1', 'messages', None, 'put', 3) - await manager.send_event('wid1', 'messages', None, 'put', 0) # i.e. end the workflow - result = await manager.get_result('wid1') + await historian.send_event('wid1', 'messages', None, 'put', 3) + await historian.send_event('wid1', 'messages', None, 'put', 0) # i.e. end the workflow + result = await historian.get_result('wid1') assert result == 6 assert counter_a == 2 @@ -113,14 +113,14 @@ async def workflow(arg: int): @pytest.mark.asyncio -async def test_manager_background(): +async def test_historian_background(): storage = InMemoryBlobStorage() - histories = {} + books = {} - def create_history(wid: str): - if wid not in histories: - histories[wid] = PersistentList(wid, InMemoryBlobStorage()) - return histories[wid] + def create_book(wid: str): + if wid not in books: + books[wid] = PersistentList(wid, InMemoryBlobStorage()) + return books[wid] counter_a = 0 counter_b = 0 @@ -144,26 +144,26 @@ async def workflow(arg: int): total += message - async with Historian('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: - manager.start_soon('workflow', 'wid1', 1) + async with Historian('test-manager', storage, create_book, lambda w_type: workflow, + serializer=NoopSerializer()) as historian: + historian.start_soon('workflow', 'wid1', 1) await asyncio.sleep(0.1) - await manager.send_event('wid1', 'messages', None, 'put', 2) + await historian.send_event('wid1', 'messages', None, 'put', 2) await asyncio.sleep(0.1) - # Now pause the manager and all workflows + # Now pause the historian and all workflows - assert 'wid1' in histories + assert 'wid1' in books assert counter_a == 1 assert counter_b == 1 - async with Historian('test-manager', storage, create_history, lambda w_type: workflow, - serializer=NoopSerializer()) as manager: + async with Historian('test-manager', storage, create_book, lambda w_type: workflow, + serializer=NoopSerializer()) as historian: # At this point, all workflows should be resumed await asyncio.sleep(0.1) - await manager.send_event('wid1', 'messages', None, 'put', 3) - await manager.send_event('wid1', 'messages', None, 'put', 0) # i.e. end the workflow + await historian.send_event('wid1', 'messages', None, 'put', 3) + await historian.send_event('wid1', 'messages', None, 'put', 0) # i.e. end the workflow await asyncio.sleep(0.1) # workflow now finishes and removes itself - assert not manager.has('wid1') + assert not historian.has('wid1') assert total == 6 assert counter_a == 2 @@ -183,16 +183,16 @@ async def workflow(): storage = InMemoryBlobStorage() - def create_history(wid: str): + def create_book(wid: str): return PersistentList(wid, InMemoryBlobStorage()) - async with Historian('test', storage, create_history, lambda wid: workflow, - serializer=NoopSerializer()) as wm: - wm.start_soon('workflow', 'wid', delete_on_finish=False) + async with Historian('test', storage, create_book, lambda wid: workflow, + serializer=NoopSerializer()) as historian: + historian.start_soon('workflow', 'wid', delete_on_finish=False) await asyncio.sleep(0.1) - q = await wm.get_queue('wid', 'messages', None) - result = await wm.get_state('wid', 'result', None) - finish = await wm.get_event('wid', 'finish', None) + q = await historian.get_queue('wid', 'messages', None) + result = await historian.get_state('wid', 'result', None) + finish = await historian.get_event('wid', 'finish', None) assert await result.get() is None await q.put(3) diff --git a/history_test/test_interruptions.py b/history_test/test_interruptions.py index 03c94fa4..3971d6cd 100644 --- a/history_test/test_interruptions.py +++ b/history_test/test_interruptions.py @@ -33,23 +33,23 @@ async def workflow_2(counter_2): 'workflow_1': workflow_1, 'workflow_2': workflow_2, } - manager = create_in_memory_historian(workflows) + historian = create_in_memory_historian(workflows) counter_1 = [0] counter_2 = [0] - async with manager: - manager.start_soon('workflow_1', 'w1', counter_1, delete_on_finish=False) - manager.start_soon('workflow_2', 'w2', counter_2, delete_on_finish=False) + async with historian: + historian.start_soon('workflow_1', 'w1', counter_1, delete_on_finish=False) + historian.start_soon('workflow_2', 'w2', counter_2, delete_on_finish=False) gate_1.set() await asyncio.sleep(0.1) with pytest.raises(CancelledError): - await manager.get_result("w1") + await historian.get_result("w1") with pytest.raises(CancelledError): - await manager.get_result("w2") + await historian.get_result("w2") assert counter_1[0] == 3 assert counter_2[0] == 3 diff --git a/history_test/test_persistence.py b/history_test/test_persistence.py index ebaf75da..8dd1c277 100644 --- a/history_test/test_persistence.py +++ b/history_test/test_persistence.py @@ -95,29 +95,29 @@ def __exit__(self, *args): async def persistence_basic(storage_ctx): with storage_ctx as storage: - history = PersistentList('test', storage) - historian = History( + book = PersistentList('test', storage) + history = History( 'test', simple_workflow, - history, + book, serializer=NoopSerializer() ) - workflow = historian.run() + workflow = history.run() await asyncio.sleep(0.01) - await historian.suspend() + await history.suspend() pause.set() with storage_ctx as storage: - history = PersistentList('test', storage) - historian = History( + book = PersistentList('test', storage) + history = History( 'test', simple_workflow, - history, + book, serializer=NoopSerializer() ) - result = await historian.run() + result = await history.run() assert result == 14 @@ -165,30 +165,30 @@ async def resume_step_persistence(storage_ctx): @pytest.mark.asyncio async def test_workflow_cleanup_suspend(tmp_path): storage = LocalFileSystemBlobStorage(tmp_path) - history = PersistentList('test', storage) - historian = History( + book = PersistentList('test', storage) + history = History( 'test', resume_this_workflow, - history, + book, serializer=NoopSerializer() ) - workflow = historian.run() + workflow = history.run() await asyncio.sleep(0.01) - await historian.suspend() + await history.suspend() event.set() storage = LocalFileSystemBlobStorage(tmp_path) - history = PersistentList('test', storage) - historian = History( + book = PersistentList('test', storage) + history = History( 'test', resume_this_workflow, - history, + book, serializer=NoopSerializer() ) - await historian.run() + await history.run() assert not any(tmp_path.iterdir()) @@ -196,15 +196,15 @@ async def test_workflow_cleanup_suspend(tmp_path): @pytest.mark.asyncio async def test_workflow_cleanup_basic(tmp_path): storage = LocalFileSystemBlobStorage(tmp_path) - history = PersistentList('test', storage) - historian = History( + book = PersistentList('test', storage) + history = History( 'test', simple_workflow, - history, + book, serializer=NoopSerializer() ) pause.set() - await historian.run() + await history.run() assert not os.listdir(tmp_path) diff --git a/history_test/test_resource_stream.py b/history_test/test_resource_stream.py index dfdfe255..f8abe659 100644 --- a/history_test/test_resource_stream.py +++ b/history_test/test_resource_stream.py @@ -16,10 +16,10 @@ async def simple_workflow(phrase1_ident, phrase2_ident): # A general-use listener that asserts that both resources are seen while streaming -async def simple_listener(historian, stream_ident=None, phrase1_ident=None, phrase2_ident=None): +async def simple_listener(history, stream_ident=None, phrase1_ident=None, phrase2_ident=None): saw_phrase1 = False saw_phrase2 = False - with historian.get_resource_stream(stream_ident) as resource_stream: + with history.get_resource_stream(stream_ident) as resource_stream: async for resources in resource_stream: if ('phrase1', phrase1_ident) in resources: saw_phrase1 = True @@ -33,9 +33,9 @@ class StreamListenerError(Exception): pass # A listener that fails streaming before the workflow completes -async def failing_listener(historian: History, identity): +async def failing_listener(history: History, identity): try: - with historian.get_resource_stream(identity) as resource_stream: + with history.get_resource_stream(identity) as resource_stream: i = 0 async for resources in resource_stream: if i == 5: @@ -62,18 +62,18 @@ async def default_workflow(): async with event('gate', None) as gate: await gate.wait() - historian = create_test_history( + history = create_test_history( 'default', default_workflow ) - w_task = historian.run() + w_task = history.run() - with historian.get_resource_stream(None) as resource_stream: - phrase = wrap_as_state('phrase', None, historian) - messages = wrap_as_queue('messages', None, historian) - ident_messages = wrap_as_identity_queue('ident_messages', None, historian) - gate = wrap_as_event('gate', None, historian) + with history.get_resource_stream(None) as resource_stream: + phrase = wrap_as_state('phrase', None, history) + messages = wrap_as_queue('messages', None, history) + ident_messages = wrap_as_identity_queue('ident_messages', None, history) + gate = wrap_as_event('gate', None, history) updates = aiter(resource_stream) resources = await anext(updates) @@ -138,20 +138,20 @@ async def default_workflow(): @pytest.mark.asyncio @timeout(3) async def test_typical(): - historian = create_test_history( + history = create_test_history( 'typical', lambda: simple_workflow(None, None) ) async def run_workflow(): - w_task = historian.run() + w_task = history.run() await w_task # Demonstrates what a typical listener would look like async def typical_listener(): reported_resources = [] - with historian.get_resource_stream(None) as resource_stream: + with history.get_resource_stream(None) as resource_stream: async for resources in resource_stream: reported_resources.append(resources) @@ -169,13 +169,13 @@ async def typical_listener(): @pytest.mark.asyncio @timeout(3) async def test_private_identity_streaming_public_resources(): - historian = create_test_history( + history = create_test_history( 'private_identity_streaming_public_resources', lambda: simple_workflow(None, None) ) - w_task = historian.run() - await simple_listener(historian, None, None, None) + w_task = history.run() + await simple_listener(history, None, None, None) await w_task @@ -183,13 +183,13 @@ async def test_private_identity_streaming_public_resources(): @pytest.mark.asyncio @timeout(3) async def test_public_streaming_private_resources(): - historian = create_test_history( + history = create_test_history( 'public_streaming_private_resources', lambda: simple_workflow('private_identity', 'private_identity') ) - w_task = historian.run() - with historian.get_resource_stream(None) as resource_stream: + w_task = history.run() + with history.get_resource_stream(None) as resource_stream: async for resources in resource_stream: assert not resources await w_task @@ -199,16 +199,16 @@ async def test_public_streaming_private_resources(): @pytest.mark.asyncio @timeout(3) async def test_exception(): - historian = create_test_history( + history = create_test_history( 'exception', lambda: simple_workflow(None, None) ) - wtask = historian.run() + wtask = history.run() - await failing_listener(historian, None) + await failing_listener(history, None) - assert historian._resource_stream_manager._resource_streams == {} + assert history._resource_stream_manager._resource_streams == {} await wtask @@ -217,17 +217,17 @@ async def test_exception(): @pytest.mark.asyncio @timeout(3) async def test_concurrent_none_streams(): - historian = create_test_history( + history = create_test_history( 'concurrent_none_streams', lambda: simple_workflow(None, None) ) - wtask = historian.run() + wtask = history.run() # Run multiple streams concurrently - await asyncio.gather(simple_listener(historian, None, None, None), - simple_listener(historian, None, None, None), - simple_listener(historian, None, None, None)) + await asyncio.gather(simple_listener(history, None, None, None), + simple_listener(history, None, None, None), + simple_listener(history, None, None, None)) await wtask @@ -239,7 +239,7 @@ async def test_concurrent_none_streams(): @timeout(3) async def test_mult_identity_workflow(): async def public_listener(): - with historian.get_resource_stream(None) as resource_stream: + with history.get_resource_stream(None) as resource_stream: phrase1_fail = True async for resources in resource_stream: if ('phrase1', None) in resources: @@ -249,14 +249,14 @@ async def public_listener(): if phrase1_fail: assert False - historian = create_test_history( + history = create_test_history( 'mult_identity_workflow', lambda: simple_workflow(None, 'private_identity') ) - w_task = historian.run() + w_task = history.run() await asyncio.gather(public_listener(), simple_listener( - historian, + history, 'private_identity', None, 'private_identity' @@ -270,7 +270,7 @@ async def public_listener(): @timeout(3) async def test_multiple_private_identity_streams(): async def ident1_listener(): - with historian.get_resource_stream('ident1') as resource_stream: + with history.get_resource_stream('ident1') as resource_stream: ident1_fail = True ident2_fail = False async for resources in resource_stream: @@ -282,7 +282,7 @@ async def ident1_listener(): assert False async def ident2_listener(): - with historian.get_resource_stream('ident2') as resource_stream: + with history.get_resource_stream('ident2') as resource_stream: ident1_fail = False ident2_fail = True async for resources in resource_stream: @@ -293,12 +293,12 @@ async def ident2_listener(): if ident1_fail or ident2_fail: assert False - historian = create_test_history( + history = create_test_history( 'different_identity_streams', lambda: simple_workflow('ident1', 'ident2') ) - wtask = historian.run() + wtask = history.run() await asyncio.gather(ident1_listener(), ident2_listener()) await wtask @@ -309,15 +309,15 @@ async def ident2_listener(): @pytest.mark.asyncio @timeout(3) async def test_closing_different_identity_streams(): - historian = create_test_history( + history = create_test_history( 'different_identity_streams', lambda: simple_workflow(None, 'private_identity') ) - w_task = historian.run() + w_task = history.run() await asyncio.gather( - failing_listener(historian, None), - simple_listener(historian, 'private_identity', None, 'private_identity') + failing_listener(history, None), + simple_listener(history, 'private_identity', None, 'private_identity') ) await w_task @@ -327,19 +327,19 @@ async def test_closing_different_identity_streams(): @pytest.mark.asyncio @timeout(4) async def test_suspend_workflow(): - historian = create_test_history( + history = create_test_history( 'test_suspend_workflow', lambda: simple_workflow(None, None) ) - historian.run() + history.run() - with historian.get_resource_stream(None) as resource_stream: + with history.get_resource_stream(None) as resource_stream: updates = aiter(resource_stream) for i in range(4): await anext(updates) - await historian.suspend() + await history.suspend() try: await anext(updates) @@ -355,22 +355,22 @@ async def test_suspend_workflow(): @pytest.mark.asyncio @timeout(4) async def test_suspend_resume_workflow(): - historian = create_test_history( + history = create_test_history( 'test_suspend_resume_workflow', lambda: simple_workflow(None, None) ) - historian.run() - with historian.get_resource_stream(None) as resource_stream: + history.run() + with history.get_resource_stream(None) as resource_stream: updates = aiter(resource_stream) for i in range(3): # Call anext up to just before creation of phrase2 await anext(updates) - await historian.suspend() - w_task = historian.run() + await history.suspend() + w_task = history.run() - with historian.get_resource_stream(None) as resource_stream: - phrase2 = wrap_as_state('phrase2', None, historian) + with history.get_resource_stream(None) as resource_stream: + phrase2 = wrap_as_state('phrase2', None, history) updates = aiter(resource_stream) await anext(updates) # Get initial snapshot of resources diff --git a/history_test/test_serializer.py b/history_test/test_serializer.py index 956d3e74..8e024a33 100644 --- a/history_test/test_serializer.py +++ b/history_test/test_serializer.py @@ -51,17 +51,17 @@ async def workflow(): @pytest.mark.asyncio async def test_master_serializer(): - history = [] + book = [] # Create historian with custom serializer - historian = History('test_workflow', workflow, history, serializer=serializer) + history = History('test_workflow', workflow, book, serializer=serializer) - workflow_task = historian.run() + workflow_task = history.run() await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() pause_event.set() - workflow_task = historian.run() + workflow_task = history.run() result = await workflow_task diff --git a/history_test/test_step_concurrency.py b/history_test/test_step_concurrency.py index c444a39c..9b6c49bf 100644 --- a/history_test/test_step_concurrency.py +++ b/history_test/test_step_concurrency.py @@ -30,14 +30,14 @@ async def fooflow(text1, text2): @pytest.mark.asyncio async def test_step_concurrency(): - historian = History( + history = History( 'test', fooflow, [], serializer=NoopSerializer() ) - assert await historian.run('abc', 'xyz') == ('abcfoofoofoo', 'xyzfoofoofoo') + assert await history.run('abc', 'xyz') == ('abcfoofoofoo', 'xyzfoofoofoo') @task @@ -55,14 +55,14 @@ async def doubleflow(text): @pytest.mark.asyncio @timeout(3) async def test_step_tasks(): - historian = History( + history = History( 'test', doubleflow, [], serializer=NoopSerializer() ) - assert await historian.run('abcxyz') == 'abcxyzabcxyzabcabc' + assert await history.run('abcxyz') == 'abcxyzabcxyzabcabc' # Two tasks running concurrently diff --git a/history_test/test_step_error.py b/history_test/test_step_error.py index 0414ba11..5c8fee73 100644 --- a/history_test/test_step_error.py +++ b/history_test/test_step_error.py @@ -43,25 +43,25 @@ async def longer_workflow(text): @pytest.mark.asyncio @timeout(10) async def test_custom_exception(): - history = [] - historian = History( + book = [] + history = History( 'test', longer_workflow, - history, + book, NoopSerializer() ) - workflow = historian.run('abc') + workflow = history.run('abc') await asyncio.sleep(0.01) - await historian.suspend() + await history.suspend() - assert history # should not be empty + assert book # should not be empty # Allow workflow to proceed block_workflow.set() # Start the workflow again - result = await historian.run('abc') + result = await history.run('abc') assert result == 'abcabcfooabcabcfoo' assert double_calls == 2 @@ -105,25 +105,25 @@ async def longer_workflow2(text): @pytest.mark.asyncio @timeout(10) async def test_builtin(): - history = [] - historian = History( + book = [] + history = History( 'test2', longer_workflow2, - history, + book, NoopSerializer() ) - workflow2 = historian.run('abc') + workflow2 = history.run('abc') await asyncio.sleep(0.01) - await historian.suspend() + await history.suspend() - assert history # should not be empty + assert book # should not be empty # Allow workflow to proceed block_workflow2.set() # Start the workflow again - result2 = await historian.run('abc') + result2 = await history.run('abc') assert result2 == 'abcabcabcabc' assert double_calls2 == 2 diff --git a/history_test/test_suspend.py b/history_test/test_suspend.py index 2ebcce1a..5efdabeb 100644 --- a/history_test/test_suspend.py +++ b/history_test/test_suspend.py @@ -20,16 +20,16 @@ async def workflow_will_stop(): @pytest.mark.asyncio @timeout(3) async def test_cancel(): - historian = History( + history = History( 'test', workflow_will_stop, [], serializer=NoopSerializer() ) - workflow = historian.run() + workflow = history.run() await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() stop.set() await asyncio.sleep(0.1) @@ -62,16 +62,16 @@ async def workflow_with_tasks(): @pytest.mark.asyncio @timeout(3) async def test_task_cancel(): - historian = History( + history = History( 'test', workflow_with_tasks, [], serializer=NoopSerializer() ) - workflow = historian.run() + workflow = history.run() await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() block.set() await asyncio.sleep(0.1) diff --git a/history_test/test_versioning.py b/history_test/test_versioning.py index c9aac569..f2beaeef 100644 --- a/history_test/test_versioning.py +++ b/history_test/test_versioning.py @@ -161,20 +161,20 @@ async def application(name1, name2, name3): t3 = get_numbers(name3) return (await t1) + (await t2) + (await t3) - history = [] + book = [] name1 = 'foo' name2 = 'bar' name3 = 'baz' - historian = History('test', application, history, serializer=NoopSerializer()) - result = historian.run(name1, name2, name3) + history = History('test', application, book, serializer=NoopSerializer()) + result = history.run(name1, name2, name3) await asyncio.sleep(0.1) - await historian.record_external_event('numbers', name1, 'put', 1) + await history.record_external_event('numbers', name1, 'put', 1) await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() # New version! @task @@ -194,14 +194,14 @@ async def get_numbers(name): return result # historian = Historian('test', application, history) - result = historian.run(name1, name2, name3) + result = history.run(name1, name2, name3) await asyncio.sleep(0.1) - await historian.record_external_event('numbers', name1, 'put', 1) # 2nd number - await historian.record_external_event('numbers', name2, 'put', 1) # 1st number + await history.record_external_event('numbers', name1, 'put', 1) # 2nd number + await history.record_external_event('numbers', name2, 'put', 1) # 1st number await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() # New Version! @task @@ -224,15 +224,15 @@ async def get_numbers(name): return result - result = historian.run(name1, name2, name3) + result = history.run(name1, name2, name3) await asyncio.sleep(0.1) - await historian.record_external_event('numbers', name1, 'put', 1) # 3rd number - await historian.record_external_event('numbers', name2, 'put', 1) # 2nd number - await historian.record_external_event('numbers', name3, 'put', 1) # 1st number + await history.record_external_event('numbers', name1, 'put', 1) # 3rd number + await history.record_external_event('numbers', name2, 'put', 1) # 2nd number + await history.record_external_event('numbers', name3, 'put', 1) # 1st number await asyncio.sleep(0.1) - await historian.suspend() + await history.suspend() # And another version! @task @@ -257,12 +257,12 @@ async def get_numbers(name): return result - result = historian.run(name1, name2, name3) + result = history.run(name1, name2, name3) await asyncio.sleep(0.1) - await historian.record_external_event('numbers', name2, 'put', 1) # 3rd number - await historian.record_external_event('numbers', name3, 'put', 1) # 2nd number - await historian.record_external_event('numbers', name3, 'put', 1) # 3rd number + await history.record_external_event('numbers', name2, 'put', 1) # 3rd number + await history.record_external_event('numbers', name3, 'put', 1) # 2nd number + await history.record_external_event('numbers', name3, 'put', 1) # 3rd number await asyncio.sleep(0.1) assert (await result) == [ diff --git a/history_test/test_websockets.py b/history_test/test_websockets.py index 6cc7ba09..1c13078a 100644 --- a/history_test/test_websockets.py +++ b/history_test/test_websockets.py @@ -16,8 +16,8 @@ def authorize(headers: dict[str, str]) -> bool: return False -async def serve(manager): - async with Server(manager, 'localhost', 8000, authorizer=authorize): +async def serve(historian): + async with Server(historian, 'localhost', 8000, authorizer=authorize): await asyncio.sleep(1) @@ -43,6 +43,6 @@ async def workflow(): messages = await messages.get() await phrase.set(messages) - manager = create_in_memory_historian({'workflow': workflow}) - manager.start_soon('workflow', 'workflow1') - await asyncio.gather(serve(manager), connect()) + historian = create_in_memory_historian({'workflow': workflow}) + historian.start_soon('workflow', 'workflow1') + await asyncio.gather(serve(historian), connect()) diff --git a/history_test/test_workflow_metrics.py b/history_test/test_workflow_metrics.py index f9257d81..1332a34e 100644 --- a/history_test/test_workflow_metrics.py +++ b/history_test/test_workflow_metrics.py @@ -21,12 +21,12 @@ async def workflow2(): await gate2.wait() return "done" - async with create_in_memory_historian({'w1': workflow1, 'w2': workflow2}) as manager: - manager.start_soon('w1', 'wid1', delete_on_finish=False) - manager.start_soon('w2', 'wid2') + async with create_in_memory_historian({'w1': workflow1, 'w2': workflow2}) as historian: + historian.start_soon('w1', 'wid1', delete_on_finish=False) + historian.start_soon('w2', 'wid2') # Scheduled workflows are counted as "running" - assert len(manager.get_metrics()) == 2 + assert len(historian.get_metrics()) == 2 # Yield to let the workflows start await asyncio.sleep(0.1) @@ -36,17 +36,17 @@ async def workflow2(): await asyncio.sleep(0.1) # Finished workflows with a stored result are still "running" - assert len(manager.get_metrics()) == 2 + assert len(historian.get_metrics()) == 2 # Allow wid1 to return result and clean up - await manager.get_result('wid1', delete=True) + await historian.get_result('wid1', delete=True) - assert len(manager.get_metrics()) == 1 + assert len(historian.get_metrics()) == 1 gate2.set() await asyncio.sleep(0.1) - assert len(manager.get_metrics()) == 0 + assert len(historian.get_metrics()) == 0 @pytest.mark.asyncio @@ -58,23 +58,23 @@ async def sample_workflow(): workflows = { "sample_workflow": sample_workflow } - manager = create_in_memory_historian(workflows=workflows) + historian = create_in_memory_historian(workflows=workflows) - async with manager: + async with historian: # Start workflow but don't delete its result immediately - manager.start_soon('sample_workflow', 'wid2', delete_on_finish=False) - future_wid2 = await manager.get_result('wid2') + historian.start_soon('sample_workflow', 'wid2', delete_on_finish=False) + future_wid2 = await historian.get_result('wid2') await asyncio.sleep(0.1) assert future_wid2 is not None - await manager.delete_workflow('wid2') + await historian.delete_workflow('wid2') with pytest.raises(WorkflowNotFound): - await manager.get_result('wid2') + await historian.get_result('wid2') # Trying to delete a non-existing workflow raises WorkflowNotFound as well with pytest.raises(WorkflowNotFound): - await manager.delete_workflow('not_existing') + await historian.delete_workflow('not_existing') @pytest.mark.asyncio @@ -86,17 +86,17 @@ async def long_running_workflow(): await gate.wait() return "should not reach" - manager = create_in_memory_historian(workflows={"long_workflow": long_running_workflow}) + historian = create_in_memory_historian(workflows={"long_workflow": long_running_workflow}) - async with manager: - manager.start_soon('long_workflow', 'wid1') + async with historian: + historian.start_soon('long_workflow', 'wid1') await asyncio.sleep(0.1) # Cancel the running workflow - assert manager.has('wid1') - await manager.delete_workflow('wid1') + assert historian.has('wid1') + await historian.delete_workflow('wid1') await asyncio.sleep(0.1) - assert not manager.has('wid1') + assert not historian.has('wid1') with pytest.raises(WorkflowNotFound): - await manager.get_result('wid1') + await historian.get_result('wid1') diff --git a/history_test/test_wrappers.py b/history_test/test_wrappers.py index 52448a83..b460530a 100644 --- a/history_test/test_wrappers.py +++ b/history_test/test_wrappers.py @@ -15,8 +15,8 @@ class CallMe: async def __call__(self): await asyncio.sleep(0.01) - historian = History('test', CallMe(), [], serializer=NoopSerializer()) - await historian.run() + history = History('test', CallMe(), [], serializer=NoopSerializer()) + await history.run() @pytest.mark.asyncio @@ -42,24 +42,24 @@ async def workflow(): await useful.foo() await useful.bar() - historian = History('test', workflow, [], serializer=NoopSerializer()) - historian.run() + history = History('test', workflow, [], serializer=NoopSerializer()) + history.run() - with historian.get_resource_stream(None) as resource_stream: + with history.get_resource_stream(None) as resource_stream: updates = aiter(resource_stream) await anext(updates) # First update should be empty resources = await anext(updates) # second event should now show the 'gate' Event assert ('gate', None) in resources - await historian.suspend() + await history.suspend() - wtask = historian.run() + wtask = history.run() - with historian.get_resource_stream(None) as resource_stream: + with history.get_resource_stream(None) as resource_stream: updates = aiter(resource_stream) resources = await anext(updates) # should include 'gate' already because that is where the first run left off assert ('gate', None) in resources - gate = wrap_as_event('gate', None, historian) + gate = wrap_as_event('gate', None, history) await gate.set() await wtask # good hygiene From 0b76ab9f7f9ea1cac5f7009f66b18d17ab7f06bd Mon Sep 17 00:00:00 2001 From: Collin Webb Date: Thu, 10 Apr 2025 20:01:07 -0600 Subject: [PATCH 14/14] rename kyle/youngwoo changes --- .github/workflows/run-pytest.yml | 2 +- history_test/test_websockets.py | 4 ++-- scratch/ainput.py | 8 ++++---- src/history/__init__.py | 2 +- src/history/client.py | 2 +- ...manager_wrappers.py => historian_wrappers.py} | 0 src/history/server.py | 16 ++++++++-------- 7 files changed, 17 insertions(+), 17 deletions(-) rename src/history/{manager_wrappers.py => historian_wrappers.py} (100%) diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index c2931084..1a2790f0 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -7,7 +7,7 @@ on: - main jobs: - quest_test: + history_test: runs-on: ubuntu-latest permissions: diff --git a/history_test/test_websockets.py b/history_test/test_websockets.py index 2265e06f..a15c3fc5 100644 --- a/history_test/test_websockets.py +++ b/history_test/test_websockets.py @@ -58,14 +58,14 @@ async def workflow(): async def test_websockets(): wid = 'test' historian = create_in_memory_historian({'workflow': workflow}) - historian.start_workflow('workflow', wid) + historian.start_soon('workflow', wid) await asyncio.gather(serve(historian, 8000, authorize), connect(wid)) @pytest.mark.asyncio async def test_websockets_exception(): wid = 'test_exception' historian = create_in_memory_historian({'workflow': workflow}) - historian.start_workflow('workflow', wid) + historian.start_soon('workflow', wid) await asyncio.gather(serve(historian, 8000, lambda h: True), connect_exception('fail')) await asyncio.sleep(0.1) await historian.delete_workflow(wid) diff --git a/scratch/ainput.py b/scratch/ainput.py index 1f054c4e..c2540cce 100644 --- a/scratch/ainput.py +++ b/scratch/ainput.py @@ -59,11 +59,11 @@ async def sleep_workflow(): Path('ainput_state'), 'sleep', lambda wid: the_workflow - ) as manager: - if not manager.has('sleep_workflow'): - manager.start_soon('sleep_workflow', 'sleep_workflow') + ) as historian: + if not historian.has('sleep_workflow'): + historian.start_soon('sleep_workflow', 'sleep_workflow') - await manager.get_workflow('sleep_workflow') + await historian.get_workflow('sleep_workflow') # Run the test if __name__ == '__main__': diff --git a/src/history/__init__.py b/src/history/__init__.py index 18573a34..7f11fa58 100644 --- a/src/history/__init__.py +++ b/src/history/__init__.py @@ -6,7 +6,7 @@ from .history import History from .book import Book from .historian import Historian, WorkflowFactory -from .manager_wrappers import alias +from .historian_wrappers import alias from .persistence import LocalFileSystemBlobStorage, PersistentList, BlobStorage, Blob from .serializer import StepSerializer, MasterSerializer, NoopSerializer from .utils import ainput diff --git a/src/history/client.py b/src/history/client.py index 07b6b462..11016f10 100644 --- a/src/history/client.py +++ b/src/history/client.py @@ -3,7 +3,7 @@ import json from websockets.asyncio.client import connect -from quest.utils import deserialize_exception +from history.utils import deserialize_exception def deserialize_response(response): diff --git a/src/history/manager_wrappers.py b/src/history/historian_wrappers.py similarity index 100% rename from src/history/manager_wrappers.py rename to src/history/historian_wrappers.py diff --git a/src/history/server.py b/src/history/server.py index 8e1e4ded..d61ba621 100644 --- a/src/history/server.py +++ b/src/history/server.py @@ -33,18 +33,18 @@ async def serialize_resources(resources): class Server: - def __init__(self, manager: Historian, host: str, port: int, + def __init__(self, historian: Historian, host: str, port: int, authorizer: Callable[[Headers], bool] = lambda headers: True): """ Initialize the server. - :param manager: Workflow manager whose methods will be called remotely. + :param historian: Workflow manager whose methods will be called remotely. :param host: Host address for the server. :param port: Port for the server. :param authorizer: Used to authenticate incoming connections. """ self._server = None - self._manager: Historian = manager + self._historian: Historian = historian self._host = host self._port = port self._authorizer = authorizer @@ -73,7 +73,7 @@ async def handler(self, ws: ServerConnection): """ if not (self._authorizer(ws.request.headers)): await ws.close(reason="Unauthorized") - quest_logger.info(f'Unauthorized attempt to connect from {ws.remote_address[0]}') + history_logger.info(f'Unauthorized attempt to connect from {ws.remote_address[0]}') return history_logger.info(f'New connection from: {ws.remote_address[0]}') @@ -85,7 +85,7 @@ async def handler(self, ws: ServerConnection): case _: response = {'exception': serialize_exception(InvalidPathException(f'Invalid path: {ws.request.path}'))} await ws.send(json.dumps(response)) - quest_logger.info(f'Connection closed from: {ws.remote_address[0]}') + history_logger.info(f'Connection closed from: {ws.remote_address[0]}') async def handle_call(self, ws: ServerConnection): async for message in ws: @@ -98,9 +98,9 @@ async def handle_call(self, ws: ServerConnection): args = data['args'] kwargs = data['kwargs'] - if not hasattr(self._manager, method_name): + if not hasattr(self._historian, method_name): raise MethodNotFoundException(f'{method_name} is not a valid method') - method = getattr(self._manager, method_name) + method = getattr(self._historian, method_name) if not callable(method): raise MethodNotFoundException(f'{method_name} is not callable') @@ -125,7 +125,7 @@ async def handle_stream(self, ws: ServerConnection): ident = params['identity'] # Stream resource updates via ws messages - with self._manager.get_resource_stream(wid, ident) as stream: + with self._historian.get_resource_stream(wid, ident) as stream: async for resources in stream: # Serialize tuple keys into strings joined by '||' resources = await serialize_resources(resources)