From 5a9d526d5c3cf997ed35208b6bb1a6147157f0aa Mon Sep 17 00:00:00 2001 From: kvnkho Date: Thu, 31 Aug 2023 22:15:51 -0500 Subject: [PATCH 1/3] Cleaning x-like --- .readthedocs.yaml | 2 +- tutorials/advanced/x-like.ipynb | 297 ++++++++++++++------------------ 2 files changed, 127 insertions(+), 172 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index aa7f3ee4..d676a714 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,7 +14,7 @@ build: # Build documentation in the docs/ directory with Sphinx sphinx: - configuration: conf.py + configuration: conf.py python: install: diff --git a/tutorials/advanced/x-like.ipynb b/tutorials/advanced/x-like.ipynb index 524bc0eb..f18c5a5e 100644 --- a/tutorials/advanced/x-like.ipynb +++ b/tutorials/advanced/x-like.ipynb @@ -22,9 +22,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "a:int,b:str\n", + "a:int,b_array:[long],c_dict:{x:int,y:str}\n", + "pa schema a: int32\n", + "b: string\n", + "a:int,b:str\n", + "c:str,d:long\n", + "c:str,d:long\n", + "e:str,f:str\n", + "e:str,f:str,g:long\n", + "a:int,b:str\n" + ] + } + ], "source": [ "from fugue import Schema\n", "\n", @@ -44,154 +61,6 @@ "print(Schema(Schema(\"a:int\",\"b:str\"))) # you can separate" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Parameters\n", - "\n", - "`ParamDict` is not that flexible, it can only accept dict or list of tuples just like python dict. `ParamDict` itself is a python dict." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from triad.collections import ParamDict\n", - "\n", - "print(ParamDict())\n", - "print(ParamDict(dict(a=1,b=\"d\")))\n", - "print(ParamDict([(\"a\",1),(\"b\",\"d\")]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DataFrame\n", - "\n", - "Normally, you should create a dataframe from [ExecutionEngine](execution_engine.ipynb) or [FugueWorkflow](dag.ipynb). In general, all execution engines and workflows support list/iterable of python arrays and pandas or Fugue dataframes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from fugue import ExecutionEngine, FugueWorkflow, NativeExecutionEngine, PandasDataFrame\n", - "from fugue_dask import DaskExecutionEngine\n", - "import pandas as pd\n", - "\n", - "def construct_df_by_execution_engine(eng:ExecutionEngine):\n", - " eng.to_df([[0]], \"a:int\", {\"x\":1}).show(title=\"from array\")\n", - " df = PandasDataFrame([[0]], \"a:int\")\n", - " eng.to_df(df).show(title=\"from fugue dataframe\")\n", - " eng.to_df(df.as_pandas()).show(title=\"from pandas dataframe\")\n", - " \n", - "construct_df_by_execution_engine(NativeExecutionEngine())\n", - "construct_df_by_execution_engine(DaskExecutionEngine()) # notice the dataframe types change\n", - "\n", - "print(\"-----------------------------------\")\n", - "\n", - "def construct_df_by_workflow(eng:ExecutionEngine):\n", - " with FugueWorkflow(eng) as dag:\n", - " dag.df([[0]], \"a:int\", {\"x\":1}).show(title=\"from array\")\n", - " df = PandasDataFrame([[0]], \"a:int\")\n", - " dag.df(df).show(title=\"from fugue dataframe\")\n", - " dag.df(df.as_pandas()).show(title=\"from pandas dataframe\")\n", - " \n", - "construct_df_by_workflow(NativeExecutionEngine())\n", - "construct_df_by_workflow(DaskExecutionEngine()) # notice the dataframe types change " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DataFrames\n", - "\n", - "`DataFrames` is a type, it represents a collection of Fugue DataFrames. It can be dict-like where each dataframe has a name, or list-like. It is also an extensively used data structure in the framework" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from fugue import DataFrames, ArrayDataFrame, ArrowDataFrame\n", - "\n", - "df1 = ArrayDataFrame([[0]],\"a:int\")\n", - "df2 = ArrowDataFrame([[1]],\"b:int\")\n", - "\n", - "dfs = DataFrames(df1, df2) # list-like\n", - "assert not dfs.has_key\n", - "assert df1 is dfs[0]\n", - "assert df2 is dfs[1]\n", - "# how to get values as an array in list-like DataFrames\n", - "print(list(dfs.values()))\n", - "\n", - "dfs = DataFrames(x=df1, y=df2) # dict-like\n", - "assert dfs.has_key\n", - "assert df1 is dfs[\"x\"]\n", - "assert df2 is dfs[\"y\"]\n", - "assert isinstance(dfs, dict) # dfs itself is dict, so you know how to iterate\n", - "\n", - "dfs = DataFrames(dict(x=df1,y=df2)) # another equal way to init dict-like\n", - "\n", - "df3 = ArrowDataFrame([[1]],\"a:int\")\n", - "dfs1 = DataFrames(dict(x=df1,y=df2))\n", - "dfs2 = DataFrames(dfs1, z=df3) # DataFrames are immutable, but you can update in this way\n", - "dfs2 = DataFrames(dict(x=df1,y=df2), z=df3)\n", - "\n", - "dfs3 = DataFrames(df1,df2)\n", - "dfs4 = DataFrames(dfs3, df3) # DataFrames are immutable, but you can update in this way\n", - "dfs4 = DataFrames([df1,df2], df3) " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### DataFrames in Programming Interface\n", - "\n", - "In Fugue programming interface, it's common to use DataFrames, it's also very flexible" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from fugue import FugueWorkflow, ArrayDataFrame, DataFrames\n", - "from fugue_dask import DaskExecutionEngine\n", - "\n", - "def show(dfs:DataFrames, title=\"\"):\n", - " if title!=\"\":\n", - " print(title)\n", - " for k,v in dfs.items():\n", - " v.show(title=k)\n", - "\n", - "df1 = ArrayDataFrame([[0]],\"a:int\")\n", - "df2 = ArrayDataFrame([[1]],\"b:int\")\n", - "\n", - "with FugueWorkflow(DaskExecutionEngine) as dag:\n", - " # inside output, it constructs DataFrames(df1, df2) and then convert both to WorkflowDataFrames\n", - " # you can see it's a list like dataframes, but due to the engine, they become DaskDataFrame\n", - " # all these conversions are automatic\n", - " dag.output(df1,df2,using=show, params={\"title\":\"*args from raw dataframes\"})\n", - " dag.output([df1,df2],using=show, params={\"title\":\"list from raw dataframes\"})\n", - " # dict-like input must be passed in as dicts\n", - " dag.output(dict(x=df1,y=df2),dict(z=df1),using=show, params={\"title\":\"dict from raw dataframes\"})\n", - " \n", - " cdf1=dag.df(df1) # you can also convert the dataframes to WorkflowDataFrame explicitly (recommended)\n", - " dag.output(cdf1,df2,using=show, params={\"title\":\"mixed\"}) # and you can mix them, although not recommended " - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -201,9 +70,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "PartitionSpec(num='4', by=['a', 'b'], presort='')" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from fugue import PartitionSpec\n", "\n", @@ -237,31 +117,99 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ab
000
101
211
322
\n", + "
" + ], + "text/plain": [ + " a b\n", + "0 0 0\n", + "1 0 1\n", + "2 1 1\n", + "3 2 2" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import pandas as pd\n", - "from fugue import FugueWorkflow\n", + "import fugue.api as fa\n", "from fugue.rpc import RPCHandler\n", "\n", - "# schema: *\n", - "def print_describe_and_return(df:pd.DataFrame, cb:callable) -> pd.DataFrame:\n", - " cb(str(df.describe()))\n", + "def print_columns_and_return(df:pd.DataFrame, cb:callable) -> pd.DataFrame:\n", + " cb(str(df.columns))\n", " return df\n", "\n", - "dag = FugueWorkflow()\n", - "df = dag.df([[0,0],[1,1],[0,1],[2,2]],\"a:int,b:int\")\n", - "\n", - "# lambda\n", - "df.partition(by=[\"a\"]).transform(print_describe_and_return, callback = lambda x:print(x)).show()\n", - "\n", - "# function\n", "def pt(x):\n", " print(x)\n", "\n", - "df.partition(by=[\"a\"]).transform(print_describe_and_return, callback = pt).show()\n", - "\n", "# RPCHandler\n", "class Handler(RPCHandler):\n", " def __init__(self):\n", @@ -269,10 +217,17 @@ " \n", " def __call__(self, x):\n", " print(x)\n", - " \n", - "df.partition(by=[\"a\"]).transform(print_describe_and_return, callback = Handler()).show()\n", "\n", - "dag.run()" + "df = pd.DataFrame([[0,0],[1,1],[0,1],[2,2]], columns=[\"a\",\"b\"])\n", + "\n", + "# lambda\n", + "fa.transform(df, print_columns_and_return, schema=\"*\", partition={\"by\": \"a\"}, callback = lambda x:print(x))\n", + "\n", + "# function\n", + "fa.transform(df, print_columns_and_return, schema=\"*\", partition={\"by\": \"a\"}, callback = pt)\n", + "\n", + "# RPCHandler class\n", + "fa.transform(df, print_columns_and_return, schema=\"*\", partition={\"by\": \"a\"}, callback = Handler())" ] } ], @@ -292,7 +247,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.9" + "version": "3.8.13" }, "vscode": { "interpreter": { From 78ad1c02f117c2e1618621f3cd09c35549796490 Mon Sep 17 00:00:00 2001 From: kvnkho Date: Sat, 2 Sep 2023 19:30:38 -0500 Subject: [PATCH 2/3] Removing FugueWorkflow from RPC --- tutorials/advanced/rpc.ipynb | 721 +++++++++++++++++++++++++++++++---- 1 file changed, 647 insertions(+), 74 deletions(-) diff --git a/tutorials/advanced/rpc.ipynb b/tutorials/advanced/rpc.ipynb index 10b326dd..002920b1 100644 --- a/tutorials/advanced/rpc.ipynb +++ b/tutorials/advanced/rpc.ipynb @@ -22,23 +22,93 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ab
000
101
211
322
\n", + "
" + ], + "text/plain": [ + " a b\n", + "0 0 0\n", + "1 0 1\n", + "2 1 1\n", + "3 2 2" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import pandas as pd\n", - "from fugue import FugueWorkflow\n", + "import fugue.api as fa\n", "\n", - "# schema: *\n", - "def print_describe_and_return(df:pd.DataFrame, cb:callable) -> pd.DataFrame:\n", - " cb(str(df.describe()))\n", + "def print_columns_and_return(df:pd.DataFrame, cb:callable) -> pd.DataFrame:\n", + " cb(str(df.columns))\n", " return df\n", "\n", - "dag = FugueWorkflow()\n", - "df = dag.df([[0,0],[1,1],[0,1],[2,2]],\"a:int,b:int\")\n", - "df.partition(by=[\"a\"]).transform(print_describe_and_return, callback = lambda x:print(x)).show()\n", + "df = pd.DataFrame([[0,0],[1,1],[0,1],[2,2]], columns=[\"a\",\"b\"])\n", "\n", - "dag.run()" + "# lambda\n", + "fa.transform(df, print_columns_and_return, schema=\"*\", partition={\"by\": \"a\"}, callback = lambda x:print(x))" ] }, { @@ -54,29 +124,96 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n", + "Index(['a', 'b'], dtype='object')\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ab
000
101
211
322
\n", + "
" + ], + "text/plain": [ + " a b\n", + "0 0 0\n", + "1 0 1\n", + "2 1 1\n", + "3 2 2" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from typing import Optional, Callable, Any\n", "\n", - "# schema: *\n", - "def print_describe_and_return(df:pd.DataFrame, cb:Optional[Callable[[Any],None]]) -> pd.DataFrame:\n", + "def print_columns_and_return(df:pd.DataFrame, cb:Optional[Callable[[Any],None]]) -> pd.DataFrame:\n", " if cb is not None:\n", - " cb(str(df.describe()))\n", + " cb(str(df.columns))\n", " return df\n", "\n", - "dag = FugueWorkflow()\n", - "df = dag.df([[0,0],[1,1],[0,1],[2,2]],\"a:int,b:int\")\n", - "df.partition(by=[\"a\"]).transform(print_describe_and_return, callback = lambda x:print(x)).show()\n", + "df = pd.DataFrame([[0,0],[1,1],[0,1],[2,2]], columns=[\"a\",\"b\"])\n", "\n", - "dag.run()\n", + "# with callback\n", + "fa.transform(df, print_columns_and_return, schema=\"*\", partition={\"by\": \"a\"}, callback = lambda x:print(x))\n", "\n", - "dag2 = FugueWorkflow()\n", - "df = dag2.df([[0,0],[1,1],[0,1],[2,2]],\"a:int,b:int\")\n", - "df.partition(by=[\"a\"]).transform(print_describe_and_return).show()\n", - "\n", - "dag2.run()" + "# no callback\n", + "fa.transform(df, print_columns_and_return, schema=\"*\", partition={\"by\": \"a\"})" ] }, { @@ -95,20 +232,245 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "23/09/01 13:45:05 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "23/09/01 13:45:06 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + ] + } + ], "source": [ - "from fugue_spark import SparkExecutionEngine\n", + "from pyspark.sql import SparkSession\n", + "\n", + "spark = SparkSession.builder.getOrCreate()\n", "\n", "conf = {\n", " \"fugue.rpc.server\": \"fugue.rpc.flask.FlaskRPCServer\",\n", " \"fugue.rpc.flask_server.host\": \"0.0.0.0\",\n", " \"fugue.rpc.flask_server.port\": \"1234\",\n", " \"fugue.rpc.flask_server.timeout\": \"2 sec\",\n", - "}\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n", + "Index(['a'], dtype='object')\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
a:long
02
112
226
328
429
530
633
742
848
967
\n", + "
\n", + "PandasDataFrame: a:long" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_columns_and_return(df:pd.DataFrame, cb:Optional[Callable[[Any],None]]) -> pd.DataFrame:\n", + " if cb is not None:\n", + " cb(str(df.columns))\n", + " return df\n", "\n", - "dag.run(SparkExecutionEngine(conf=conf))" + "fa.show(\n", + " fa.transform(df, \n", + " print_columns_and_return,\n", + " schema=\"*\", \n", + " partition={\"by\": \"a\"}, \n", + " engine=spark,\n", + " engine_conf=conf,\n", + " as_local=True,\n", + " callback = lambda x: print(x)\n", + " )\n", + ")" ] }, { @@ -127,9 +489,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ab
000
101
211
322
\n", + "
" + ], + "text/plain": [ + " a b\n", + "0 0 0\n", + "1 0 1\n", + "2 1 1\n", + "3 2 2" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from threading import RLock\n", "\n", @@ -145,17 +570,16 @@ " \n", "callback = Callback()\n", " \n", - " \n", - "# schema: *\n", "def take(df:pd.DataFrame, skip:callable) -> pd.DataFrame:\n", " if not skip():\n", " return df\n", "\n", - "dag = FugueWorkflow()\n", - "df = dag.df([[0,0],[1,1],[0,1],[2,2]],\"a:int,b:int\")\n", - "df.partition(by=[\"a\"]).transform(take, callback = callback.should_skip).show()\n", - "\n", - "dag.run()\n" + "fa.transform(df, \n", + " print_columns_and_return,\n", + " schema=\"*\", \n", + " partition={\"by\": \"a\"}, \n", + " callback = callback.should_skip\n", + ")" ] }, { @@ -174,9 +598,74 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "counter started\n", + "counter stopped\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ab
000
101
211
\n", + "
" + ], + "text/plain": [ + " a b\n", + "0 0 0\n", + "1 0 1\n", + "2 1 1" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from threading import RLock\n", "from fugue.rpc import RPCHandler\n", @@ -208,19 +697,16 @@ " \n", "callback = Callback()\n", " \n", - " \n", - "# schema: *\n", "def take(df:pd.DataFrame, skip:callable) -> pd.DataFrame:\n", " if not skip():\n", " return df\n", "\n", - "dag = FugueWorkflow()\n", - "df = dag.df([[0,0],[1,1],[0,1],[2,2]],\"a:int,b:int\")\n", - "df.partition(by=[\"a\"]).transform(take, callback = callback).show()\n", - "\n", - "print(dag.spec_uuid()) # every time, the id will be different because the Callback is not deterministic\n", - "dag.run()\n", - "\n" + "fa.transform(df, \n", + " take,\n", + " schema=\"*\", \n", + " partition={\"by\": \"a\"}, \n", + " callback = callback\n", + ")" ] }, { @@ -234,11 +720,107 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " a b\n", + "count 2.0 2.000000\n", + "mean 0.0 0.500000\n", + "std 0.0 0.707107\n", + "min 0.0 0.000000\n", + "25% 0.0 0.250000\n", + "50% 0.0 0.500000\n", + "75% 0.0 0.750000\n", + "max 0.0 1.000000\n", + " a b\n", + "count 1.0 1.0\n", + "mean 1.0 1.0\n", + "std NaN NaN\n", + "min 1.0 1.0\n", + "25% 1.0 1.0\n", + "50% 1.0 1.0\n", + "75% 1.0 1.0\n", + "max 1.0 1.0\n", + " a b\n", + "count 1.0 1.0\n", + "mean 2.0 2.0\n", + "std NaN NaN\n", + "min 2.0 2.0\n", + "25% 2.0 2.0\n", + "50% 2.0 2.0\n", + "75% 2.0 2.0\n", + "max 2.0 2.0\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ab
000
101
211
322
\n", + "
" + ], + "text/plain": [ + " a b\n", + "0 0 0\n", + "1 0 1\n", + "2 1 1\n", + "3 2 2" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "from fugue import FugueWorkflow, Transformer\n", + "from fugue import Transformer\n", "\n", "class PrintAndReturn(Transformer):\n", " def get_output_schema(self, df):\n", @@ -248,11 +830,12 @@ " self.callback(str(df.as_pandas().describe()))\n", " return df\n", "\n", - "dag = FugueWorkflow()\n", - "df = dag.df([[0,0],[1,1],[0,1],[2,2]],\"a:int,b:int\")\n", - "df.partition(by=[\"a\"]).transform(PrintAndReturn, callback = lambda x:print(x)).show()\n", - "\n", - "dag.run()" + "fa.transform(df, \n", + " PrintAndReturn,\n", + " schema=\"*\", \n", + " partition={\"by\": \"a\"}, \n", + " callback = lambda x:print(x)\n", + ")" ] }, { @@ -264,12 +847,12 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAp9UlEQVR4nO3deXyV5Z3//9fnnJOFsC+BsIOCsgQQiRjUdtwFW8W2joD7RnTa6WK/0/l1pt+p/dqOrcu0Vjttibi3gtbaqW0Foa61EjCgSFgEBAQx7Pua7fP749wyxxgg5OTkTnLez8fjPHLOdV937g93Dnnnvu9zX5e5OyIikn4iYRcgIiLhUACIiKQpBYCISJpSAIiIpCkFgIhImoqFXcCJ6Natmw8YMCDsMkREWoyFCxduc/fcupa1qAAYMGAApaWlYZchItJimNmHR1umU0AiImlKASAikqYUACIiaUoBICKSphQAIiJp6rgBYGaPmtkWMytLaOtiZnPNbFXwtfNR1r0h6LPKzG5IaB9jZkvMbLWZPWhm1jj/HBERqa/6HAE8Doyv1fZd4GV3Hwy8HLz+FDPrAtwJnAmMBe5MCIpfAVOBwcGj9vcXEZEUO24AuPsbwI5azROBJ4LnTwBX1LHqJcBcd9/h7juBucB4M+sJdHD3Eo+PRf3kUdZvNA++vIrZZeUcrKhO5WZERFqUht4I1sPdy4Pnm4AedfTpDWxIeP1R0NY7eF67vU5mVgQUAfTr1++ECz1QUcUTb61j+/4K2mREOffUXMbn53H+kO60z8444e8nItJaJH0nsLu7maVsVhl3LwaKAQoKCk54OzmZMeb/+wXMX7uDF5eU89LSzcwq20RmLMLnB3djfH5PLhrag445CgMRSS8NDYDNZtbT3cuDUzpb6uizETg34XUf4LWgvU+t9o0NrKNeYtEIZw/qxtmDunHXxHwWfriTWWXlvFS2ib8u30IsYow7uSsT8nty8fAedGuXlcpyRESaBavPlJBmNgD4s7vnB6/vA7a7+0/M7LtAF3f/11rrdAEWAqcHTYuAMe6+w8wWAN8A5gMvAg+5+4vHq6OgoMAbcywgd2fxR7uZVVbOrCWbWL/jABGDMwZ04dIRPblkeB55HbMbbXsiIk3NzBa6e0Gdy44XAGY2g/hf8t2AzcQ/2fM/wLNAP+BD4KrgF3sBcLu73xqsezPw78G3+k93fyxoLyD+6aI2wCzg616PJGrsAEjk7iwr38Pssk3MKtvE6i37ADi9Xycm5PdkfH4efbvkpGTbIiKpklQANCepDIDaVm/Zy6wl8TBYVr4HgPzeHY6Ewcm57ZqkDhGRZCgAkvTh9v3MCo4MFm/YBcCpPdozPj+PCSPyOLVHe3Qvm4g0RwqARvTxroPMLtvE7LJNvP3hDtxhYLe28TDIz2NE744KAxFpNhQAKbJl7yHmLN3M7LJNzFuzneoap3enNkwIjgxG9+1MJKIwEJHwKACawM79FcxdtplZZeW8uXobldVOjw5ZXDI8j/H5eYwd0IVYVGPviUjTUgA0sT2HKnll+RZmlZXz2vtbOVxVQ5e2mVw8rAcTRvRk3EldyYwpDEQk9RQAITpQUcVr72/lxSXlvLpiC/srqumQHePCYT2YkN+Tzw3uRnZGNOwyRaSVUgA0E4cqq/nbqm3MKivnr8s2s+dQFW0zo5w3pDsT8nty3pBccjKTHp1DROSIYwWAfts0oeyMKBcN68FFw3pQUVXDvDXbmV1Wzpylm/nze+VkxSKce2ouE/J7cv7Q7nTQYHUikkI6AmgGqqprWLBux5GPl27Ze5jMaISzB8XHJ7poWA86t80Mu0wRaYF0CqgFqalx3tmw88hdyBt3HSQaMQpP6nJksLru7TU+kYjUjwKghXJ3yjbuiQ9WV7aJtdv2YwZn9O/C+Pz4x0t7dWoTdpki0owpAFoBd+f9zfHxiWaXbeL9zXsBGNW3U/zGs/w8+ndtG3KVItLcKABaoTVb9zEruGawZONuAIb17HDkLuRB3duHXKGINAcKgFZuw44DwTDW5SxavwuAQd3bMSE4TTSsZweNTySSphQAaWTT7kO8tDQeBgvW7qDGoX/XnGCwup6M6qPB6kTSiQIgTW3bdzgYn2gTb63eRlWN06tjNpcEYTCmf2eiGqxOpFVLWQCY2TeBqYABD7v7A7WWfwe4JngZA4YCucHsYeuAvUA1UHW0AhMpABpu94FK5i7fzOyyct5YtY2Kqhpy22dx8bAeXDqiJ2cO1GB1Iq1RSgLAzPKBmcBYoAKYTXw6yNVH6X8ZcIe7nx+8XgcUuPu2+m5TAdA49h2u4pUVW5hdVs6rK7ZysLKazjkZXBSMT3TWoK5kxTQ+kUhrkKqhIIYC8939QLCR14EvA/cepf8UYEYS25NG0i4rxuWjenH5qF4crKjm9ZVbmVVWzotLNvFs6Ue0z4pxwdDujM/vybmn5mqwOpFWKpkjgKHAH4FxwEHgZaDU3b9eR98c4CNgkLvvCNrWAjsBB6a5e/FRtlMEFAH069dvzIcfftigeuX4DldV8/fV25i1ZBNzl29m14FKcjKjnHdqd8bn53HekO60y9LwUSItSSqvAdwCfBXYDywFDrv7t+roNwm41t0vS2jr7e4bzaw7MBf4uru/cazt6RRQ06msrmH+mh3MKivnpaWb2bbvMJmxCJ8fnMuE/DwuHNqDjjkarE6kuWuSTwGZ2d3AR+7+yzqW/QH4nbs/fZR1fwDsc/f7j7UNBUA4qmuc0nU7mFW2iZeWbqJ89yFiEePsQd2YkJ/HRcN60LVdVthlikgdUnkE0N3dt5hZP2AOUOjuu2r16QisBfq6+/6grS0Qcfe9wfO5wF3uPvtY21MAhK+mxln80a7gxrNNrN9xgIjBmQO7MmFEHpcMz6NHBw1WJ9JcpDIA/gZ0BSqBb7v7y2Z2O4C7/zrocyMw3t0nJ6x3EvCH4GUMeNrd//N421MANC/uzrLyPcwu28SLS8r5YGt8sLrT+3U+chdyn845YZcpktZ0I5g0iVWb9zIrODJYXr4HgJF9Oh65C3lgNw1WJ9LUFADS5NZt28/spfEwWLxhFwBD8tozPj+PS0f0ZHD3dhqSQqQJKAAkVBt3HQxmOyun9MOduMNJuW2DYax7MryXBqsTSRUFgDQbW/Yc4qVl8SEpStbsoLrG6dO5TXDNoCej+3YiovGJRBqNAkCapR37K/jrss28WFbO31dvo7LayeuQfWS2szMGdNFgdSJJUgBIs7f7YCWvrNjMrCWbeH3lVg5X1dCtXSYXDYvPdjbu5K5kaLA6kROmAJAWZf/hKl57Pz4+0asrtrC/opqObTK4cGgPJuTncc7gbhqfSKSeFADSYh2qrOaNlVuZXRYfn2jvoSraZcU4f0h3JuTn8Q+n5pKTqfGJRI4mVaOBiqRcdkaUi4fncfHwPCqqanjrg23MLtvEnGWbeWHxx2RnRDj3lO5MGJHH+UO60z5b4xOJ1JeOAKRFqqquYcG6HcHHSzexZe9hMqMRzhn8v+MTdcrJDLtMkdDpFJC0ajU1zqL1O5kVhMHGXQeJRYxxJ3dlfH4eFw/LI7e9BquT9KQAkLTh7izZuPtIGKzdtp+IQcGALlwa3GuQ11GD1Un6UABIWnJ33t+8lxeXxO9CXrl5HwCj+3U6chdy3y4arE5aNwWACPDB1n3BMNbllG2MD1Y3vFeHI3chD+reLuQKRRqfAkCklg07DhwJg0XrdwFwSo92jM/vyYT8PIbktdf4RNIqKABEjqF890FeCoaxfnvdDmocBnTNORIGI/t0VBhIi6UAEKmnbfsOM2fpZmaVlTPvg+1U1Ti9O7XhkuF5XDoij9P7ddZgddKipHJGsG8CUwEDHnb3B2otPxf4I/EpIQGed/e7gmXjgZ8DUWC6u//keNtTAEhT2nWggr8u38KsJeX8bdU2Kqpr6N4+i0uGx8cnGjuwCzGNTyTNXEoCwMzygZnAWKACmA3c7u6rE/qcC/yLu3+x1rpRYCVwEfAR8DYwxd2XHWubCgAJy95DlbyyYguzyzbx2vtbOVhZTZe2mVw0tAfjR+Rx9sndyIwpDKT5SdVQEEOB+e5+INjI68CXgXvrse5YYLW7rwnWnQlMBI4ZACJhaZ+dwcTTejPxtN4crKjm9ZVbmFW2ib8sKeeZ0g0M79WBp245ky5tdfextBzJ/MlSBnzOzLqaWQ5wKdC3jn7jzGyxmc0ys+FBW29gQ0Kfj4K2zzCzIjMrNbPSrVu3JlGuSONokxllfH5Pfj55NAv/40IemHQaq7fsY0pxCVv3Hg67PJF6a3AAuPty4B5gDvHTP+8C1bW6LQL6u/so4CHgfxqwnWJ3L3D3gtzc3IaWK5ISWbEoV4zuzWM3nsH6HQeYXDyPzXsOhV2WSL0kddLS3R9x9zHu/nlgJ/Hz+onL97j7vuD5i0CGmXUDNvLpo4U+QZtIi3TWoG48cfNYNu0+xKRp8/h418GwSxI5rqQCwMy6B1/7ET///3St5XkWfIDazMYG29tO/KLvYDMbaGaZwGTghWRqEQnb2IFdePKWM9m+r4JJxfPYsONA2CWJHFOyH1v4vZktA/4EfM3dd5nZ7WZ2e7D8SqDMzBYDDwKTPa4K+GfgJWA58Ky7L02yFpHQjenfmd/ceia7D1Qyado8Pty+P+ySRI5KN4KJpMDSj3dz7fT5ZMYiPD21kJNzNc6QhONYHwPVB5dFUmB4r47MLBpHdY0zaVoJKzfvDbskkc9QAIikyKl57ZlZNI6IweTiEpZ9vCfskkQ+RQEgkkKDurfjmdvGkRWLMOXhEpZ8tDvskkSOUACIpNjAbm159rZxtMuKcfX0Et5ZvzPskkQABYBIk+jbJYdnbx9Hl7aZXPfIAt5etyPskkQUACJNpXenNjxTNI7uHbK44dEFzPtge9glSZpTAIg0obyO2cwsKqR3pzbc9PgC/rZK41tJeBQAIk2se/t4CAzo2pZbnijl1RVbwi5J0pQCQCQEXdtlMWNqIaf0aEfRU6XMWbop7JIkDSkARELSuW0mv721kOG9OvLV3y7iL++Vh12SpBkFgEiIOrbJ4KlbxjK6Xye+PmMRf3xXg+JK01EAiISsfXYGj980lrEDu/CtZ97ld6Ubjr+SSCNQAIg0A22zYjx241jOGdSN7zz3Hk/PXx92SZIGFAAizUSbzCgPX1/Aeafm8u9/WMKT89aFXZK0cgoAkWYkOyPKr68bw0XDevD9Py5l+t/WhF2StGLJzgj2TTMrM7OlZvatOpZfY2bvmdkSM3vLzEYlLFsXtL9rZhrkXySQFYvyy2tO5wsjevKjvyznv19dHXZJ0krFGrqimeUDU4GxQAUw28z+7O6J79a1wD+4+04zmwAUA2cmLD/P3bc1tAaR1iojGuHnk08jFjXue+l9Kqtr+OYFgwlmWBVpFA0OAGAoMN/dDwCY2evE5wW+95MO7v5WQv8S4pO/i0g9xKIRfnrVaWREIzzw11VUVtfwLxefqhCQRpPMKaAy4HNm1tXMcoBLgb7H6H8LMCvhtQNzzGyhmRUdbSUzKzKzUjMr3bpV46ZIeolGjHu/MpIpY/vx369+wN0vLqclTeMqzVuDjwDcfbmZ3QPMAfYD7wLVdfU1s/OIB8A5Cc3nuPtGM+sOzDWzFe7+Rh3bKSZ+6oiCggK98yXtRCLG3V/KJzNqPPy3tVRWO3deNkxHApK0ZE4B4e6PAI8AmNndwEe1+5jZSGA6MMHdtyesuzH4usXM/kD8WsJnAkBEwMz4weXDyYhGmP7mWg5X1fCfV+QTiSgEpOGSCgAz6x78Au9H/Px/Ya3l/YDngevcfWVCe1sg4u57g+cXA3clU4tIa2dmfO8LQ8mMRfjlax9QWV3DPV8ZSVQhIA2UVAAAvzezrkAl8DV332VmtwO4+6+B7wNdgV8Gh6tV7l4A9AD+ELTFgKfdfXaStYi0embGdy45lcxY/MJwVXUN9//jKGJR3dIjJy7ZU0Cfq6Pt1wnPbwVuraPPGmBU7XYROT4z41sXnkJGNBJ8RNR5YHL800IiJyLZIwARCcnXzhtEVizCj/6ynMrqGh66ejRZsWjYZUkLoj8ZRFqwWz93Ev/v8uHMWbaZf/rNIg5V1vlBPJE6KQBEWrgbzhrA3V8awSsrtjD1yVIOVigEpH4UACKtwNVn9uPeK0fy5upt3Pz42xyoqAq7JGkBFAAircRVBX352VWnMX/tdm54dAF7D1WGXZI0cwoAkVbkitG9eWjK6Sxav4vrHlnA7oMKATk6BYBIK/OFkT355TWns/Tj3Vw7fT67DlSEXZI0UwoAkVbokuF5TLtuDO9v3suUh+ezfd/hsEuSZkgBINJKnT+kB9OvL2DN1n1MebiELXsPhV2SNDMKAJFW7POn5PLYTWewYcdBJheXsGm3QkD+lwJApJU76+RuPHnLWDbvPsSk4nls3HUw7JKkmVAAiKSBMwZ04albz2TH/gomTZvHhh0Hwi5JmgEFgEiaOL1fZ56+tZC9h6qYNG0e67btD7skCZkCQCSNjOjTkRlTCzlUVcNV0+axesu+sEuSECkARNLMsF4dmFlUSI3D5OJ5vL9pb9glSUgUACJp6JQe7ZlZVEjEjMnF81j68e6wS5IQJBUAZvZNMyszs6Vm9q06lpuZPWhmq83sPTM7PWHZDWa2KnjckEwdInLiBnVvx7O3jaNNRpSrH57Pex/tCrskaWINDgAzywemEp/MfRTwRTMbVKvbBGBw8CgCfhWs2wW4EzgzWP9OM+vc0FpEpGEGdGvLM7eNo312jGsens+i9TvDLkmaUDJHAEOB+e5+wN2rgNeJTwyfaCLwpMeVAJ3MrCdwCTDX3Xe4+05gLjA+iVpEpIH6dsnh2dvG0bVdJtdNn8+CtTvCLkmaSDIBUAZ8zsy6mlkOcCnQt1af3sCGhNcfBW1Ha/8MMysys1IzK926dWsS5YrI0fTq1IZnbhtHj47Z3PDoAt5avS3skqQJNDgA3H05cA8wB5gNvAs0+lRE7l7s7gXuXpCbm9vY315EAj06ZPNM0Tj6dmnDTY+/zRsr9QdXa5fURWB3f8Tdx7j754GdwMpaXTby6aOCPkHb0dpFJES57bOYMbWQk3LbcesTpbyyYnPYJUkKJfspoO7B137Ez/8/XavLC8D1waeBCoHd7l4OvARcbGadg4u/FwdtIhKyru2ymDH1TIb0bM9tTy1kdtmmsEuSFEn2PoDfm9ky4E/A19x9l5ndbma3B8tfBNYAq4GHga8CuPsO4IfA28HjrqBNRJqBTjmZ/ObWM8nv3ZGvPb2IP7/3cdglSQqYu4ddQ70VFBR4aWlp2GWIpI19h6u46bEFLPxwJ/911Si+NLpP2CXJCTKzhe5eUNcy3QksIkfVLivGEzeP5cyBXfn2s4t5tnTD8VeSFkMBICLHlJMZ49Ebz+CcQd341+fe47fzPwy7JGkkCgAROa42mVEevr6A84d053t/KOPxv68NuyRpBAoAEamX7Iwov752DJcM78EP/rSM4jc+CLskSZICQETqLTMW4RdXn84XRvbk7hdX8N+vrg67JElCLOwCRKRlyYhG+Pmk08iMRrjvpfepqKrhWxcOxszCLk1OkAJARE5YLBrh/n8cRSxi/PzlVVRU1/Cvl5yqEGhhFAAi0iDRiHHPV0aSGYvwq9c+oKKqhv/7haEKgRZEASAiDRaJGD+6Ip+MaIRH3lxLZXUNP7hsOJGIQqAlUACISFLMjDsvG0ZmLELxG2uorK7hP68YoRBoARQAIpI0M+PfJgwhMxrhF6+upqLKuffKkUQVAs2aAkBEGoWZ8S+XnEpmLMJP566kqqaG//rHUcSi+rR5c6UAEJFG9Y0LBhOLGvfOfp/K6hp+Pnk0GQqBZkkBICKN7qvnDiIzGuFHf1lOZfUifnH1aLJi0bDLkloUyyKSErd+7iTumjicucs2c/tTCzlU2egzxkqSFAAikjLXjxvAj788gtdWbuXWJ0o5WKEQaE6SnRLyDjNbamZlZjbDzLJrLf+Zmb0bPFaa2a6EZdUJy15Ipg4Rab6mjO3HfVeO4q0PtnHT4wvYf7gq7JIk0OAAMLPewDeAAnfPB6LA5MQ+7n6Hu5/m7qcBDwHPJyw++Mkyd7+8oXWISPN35Zg+/GzSaby9bic3PLqAvYcqwy5JSP4UUAxoY2YxIAc41sShU4AZSW5PRFqoiaf15qEpo3l3wy6ufWQBuw8qBMLW4ABw943A/cB6oBzY7e5z6uprZv2BgcArCc3ZZlZqZiVmdsXRtmNmRUG/0q1btza0XBFpBi4d0ZNfXnM6yz7ezTXTS9i5vyLsktJaMqeAOgMTif9i7wW0NbNrj9J9MvCcuydeAeofTFR8NfCAmZ1c14ruXuzuBe5ekJub29ByRaSZuHh4HsXXF7By8z6mPFzCtn2Hwy4pbSVzCuhCYK27b3X3SuLn9886St/J1Dr9ExxB4O5rgNeA0UnUIiItyHmndufRG85g3fb9TCkuYcueQ2GXlJaSCYD1QKGZ5Vh8/NcLgOW1O5nZEKAzMC+hrbOZZQXPuwFnA8uSqEVEWphzBnfjsRvHsnHXQSYXl7Bpt0KgqSVzDWA+8BywCFgSfK9iM7vLzBI/1TMZmOnuntA2FCg1s8XAq8BP3F0BIJJmxp3clSdvHsuWvYeZVDyPjbsOhl1SWrFP/15u3goKCry0tDTsMkSkkb2zfifXP7qADtkZzJhaSL+uOWGX1GqY2cLgeutn6E5gEQnd6H6dmTG1kP0VVUwqnsfabfvDLiktKABEpFnI792RGVMLOVxVw1XT5rF6y96wS2r1FAAi0mwM7dmBmUWFuMOkaSWs2LQn7JJaNQWAiDQrp/RozzO3FRKLGlOKSyjbuDvsklotBYCINDsn57bj2dvGkZMZ4+qHS1i8YVfYJbVKCgARaZb6d23LzKJCOuZkcO30+Sz8cEfYJbU6CgARabb6dsnhmaJxdGufxfWPLGD+mu1hl9SqKABEpFnr1akNzxQVktcxmxsfe5u/r94WdkmthgJARJq97h2ymVk0jn5dcrj58bd5faVGBm4MCgARaRFy22cxo6iQk3PbMfWJUv66bHPYJbV4CgARaTG6tM1kxtRChvZsz+2/WcjssvKwS2rRFAAi0qJ0zMngqVvPZGSfjnzt6Xf40+JjTUQox6IAEJEWp0N2Bk/eciZj+nfmmzPf4flFH4VdUoukABCRFqldVozHbzqDwpO68n9+t5hn394QdkktjgJARFqsnMwYj954Bp8bnMu//v49nir5MOySWpSkAsDM7jCzpWZWZmYzzCy71vIbzWyrmb0bPG5NWHaDma0KHjckU4eIpK/sjCjF143hgiHd+Y//KePRN9eGXVKLkcyk8L2BbwAF7p4PRInP/lXbM+5+WvCYHqzbBbgTOBMYC9wZTDIvInLCsjOi/OraMYwfnsddf17GtNc/CLukFiHZU0AxoI2ZxYAcoL6X4y8B5rr7DnffCcwFxidZi4ikscxYhIeuHs1lo3rx41kreOjlVWGX1OzFGrqiu280s/uJTw5/EJjj7nPq6PoVM/s8sBK4w903AL2BxCs2HwVtn2FmRUARQL9+/RparoikgYxohAcmnUZGxPivuSuprK7hjotOwczCLq1ZSuYUUGdgIjAQ6AW0NbNra3X7EzDA3UcS/yv/iRPdjrsXu3uBuxfk5uY2tFwRSRPRiHHfP45iUkFfHnxlNT+ZvYKWNPd5U0rmFNCFwFp33+rulcDzwFmJHdx9u7sfDl5OB8YEzzcCfRO69gnaRESSFo0YP/7yCK4t7Me019fwwz8vVwjUocGngIif+ik0sxzip4AuAEoTO5hZT3f/5F7ty4HlwfOXgLsTLvxeDPxbErWIiHxKJGL8cGI+GdEIj/59LZXVNfy/y4cTieh00CeSuQYw38yeAxYBVcA7QLGZ3QWUuvsLwDfM7PJg+Q7gxmDdHWb2Q+Dt4Nvd5e6a7UFEGpWZ8f0vDiMzFmHa62uorK7h7i+NUAgErCUdFhUUFHhpaenxO4qIJHB3fjZ3JQ++spovn96b+64cRTRNQsDMFrp7QV3LkjkFJCLSIpgZ3774VGLRCD+du5LKauenV40iI5regyEoAEQkbXzjgsFkxiL8ZNYKqqpr+Pnk0WTG0jcE0vdfLiJp6fZ/OJn/+OIwZpVt4qu/XcjhquqwSwqNAkBE0s4t5wzkh1fk89flWyh6ciGHKtMzBBQAIpKWrivszz1fGcEbq7ZyyxNvc6CiKuySmpwCQETS1qQz+nH/laOY98F2bnzsbfYdTq8QUACISFr7ypg+PDB5NAs/3MkNjy5gz6HKsEtqMgoAEUl7l4/qxS+mjGbxhl1cN30+uw+kRwgoAEREgAkjevLra8ewvHwvV08vYcf+irBLSjkFgIhI4MJhPSi+fgyrtuzj6odL2Lbv8PFXasEUACIiCc49tTuP3XgG67bvZ3JxCVv2HAq7pJRRAIiI1HL2oG48ftNYPt51kEnFJZTvPhh2SSmhABARqUPhSV156paxbNt7mEnTSvho54GwS2p0CgARkaMY078LT916JrsOVDBpWgkfbt8fdkmNSgEgInIMp/XtxNNTC9lfUcWkaSWs2bov7JIajQJAROQ48nt3ZGZRIZXVNUwqLmHV5r1hl9QokgoAM7vDzJaaWZmZzTCz7FrLv21my8zsPTN72cz6JyyrNrN3g8cLydQhIpJqQ/I6MLOoEIDJxSUsL98TckXJa3AAmFlv4BtAgbvnA1Fgcq1u7wTLRwLPAfcmLDvo7qcFj8sbWoeISFMZ3KM9zxQVkhGNMOXhEso27g67pKQkewooBrQxsxiQA3ycuNDdX3X3Ty6dlwB9ktyeiEioTsptx7O3jaNtZoyrHy7h3Q27wi6pwRocAO6+EbgfWA+UA7vdfc4xVrkFmJXwOtvMSs2sxMyuONpKZlYU9CvdunVrQ8sVEWk0/brm8MxthXTKyeTa6fMpXbcj7JIaJJlTQJ2BicBAoBfQ1syuPUrfa4EC4L6E5v7BRMVXAw+Y2cl1revuxe5e4O4Fubm5DS1XRKRR9ekcD4Hu7bO4/tEFlKzZHnZJJyyZU0AXAmvdfau7VwLPA2fV7mRmFwLfAy539yMDawRHELj7GuA1YHQStYiINLmeHdsws6iQ3p3acONjC3hz1bawSzohyQTAeqDQzHLMzIALgOWJHcxsNDCN+C//LQntnc0sK3jeDTgbWJZELSIioejeIZsZRYUM6NqWm594m1ff33L8lZqJZK4BzCf+yZ5FwJLgexWb2V1m9smneu4D2gG/q/Vxz6FAqZktBl4FfuLuCgARaZG6tctixtRCBndvx21PLmTuss1hl1Qv5u5h11BvBQUFXlpaGnYZIiJ12n2gkusfW8DSjbt5aMpoJozoGXZJmNnC4HrrZ+hOYBGRRtIxJ4Pf3DKWUX078c8z3uGP724Mu6RjUgCIiDSi9tkZPHnzWAr6d+aOZ97luYUfhV3SUSkAREQaWdusGI/fNJazTu7Gd55bzMwF68MuqU4KABGRFGiTGWX6DQV8fnAu331+CU/NWxd2SZ+hABARSZHsjCjF14/hwqE9+I8/LuWRN9eGXdKnKABERFIoKxbll9eczoT8PH7452X86rUPwi7pCAWAiEiKZcYiPDRlNJeP6sU9s1fw4Murwi4JiI/mKSIiKRaLRvjZpNOIRY2fzl1JRVUN/+fiU4gPpBBSTaFtWUQkzUQjxv1XjiIzGuEXr66msrqG704YEloIKABERJpQJGLc/aURZEQjTHtjDRXVNXz/i8NCCQEFgIhIE4tEjLsmDiczFuGRN9dSUVXDDyfmE4k0bQgoAEREQmBm/N8vDCUjGuHXr39AZXUNP/7ySKJNGAIKABGRkJgZ/9/4U8mMRXjw5VVUVjv3XTmSWLRpPqCpABARCZGZ8e2LTiEzatw/ZyWV1TX8bNJpZDRBCCgARESagX8+fzCZsQh3v7iCyuoaHppyOpmx1IaAbgQTEWkmij5/MndeNoyXlm7mn36zkEOV1SndXlIBYGZ3mNlSMyszsxlmll1reZaZPWNmq81svpkNSFj2b0H7+2Z2STJ1iIi0FjedPZAfXZHPyyu2MPXJ0pSGQIMDwMx6A98ACtw9H4gCk2t1uwXY6e6DgJ8B9wTrDgv6DgfGA780s2hDaxERaU2uLezPvV8ZyZurt3Hz429zoKIqJdtJ9hRQDGhjZjEgB/i41vKJwBPB8+eAC4IJ5CcCM939sLuvBVYDY5OsRUSk1bjqjL789KpRlKzZzo2PpiYEGnwR2N03mtn9wHrgIDDH3efU6tYb2BD0rzKz3UDXoL0kod9HQdtnmFkRUATQr1+/hpYrItLifGl0H2KRCG+u2kZ2rPFPkiRzCqgz8b/kBwK9gLZmdm1jFfYJdy929wJ3L8jNzW3sby8i0qxdNqoX91w5MiV3CSdzCuhCYK27b3X3SuB54KxafTYCfQGC00Qdge2J7YE+QZuIiDSRZAJgPVBoZjnBef0LgOW1+rwA3BA8vxJ4xd09aJ8cfEpoIDAYWJBELSIicoKSuQYw38yeAxYBVcA7QLGZ3QWUuvsLwCPAU2a2GthB8Ckhd19qZs8Cy4J1v+buqf3Aq4iIfIrF/yBvGQoKCry0tDTsMkREWgwzW+juBXUt053AIiJpSgEgIpKmFAAiImlKASAikqZa1EVgM9sKfNjA1bsB2xqxnMaiuk6M6joxquvEtMa6+rt7nXfRtqgASIaZlR7tSniYVNeJUV0nRnWdmHSrS6eARETSlAJARCRNpVMAFIddwFGorhOjuk6M6joxaVVX2lwDEBGRT0unIwAREUmgABARSVMtPgDMbHwwsfxqM/tuHctDmZi+HnV928yWmdl7ZvaymfVPWFZtZu8GjxeauK4bzWxrwvZvTVh2g5mtCh431F43xXX9LKGmlWa2K2FZKvfXo2a2xczKjrLczOzBoO73zOz0hGWp3F/Hq+uaoJ4lZvaWmY1KWLYuaH/XzBp1dMV61HWume1O+Hl9P2HZMd8DKa7rOwk1lQXvqS7BslTur75m9mrwu2CpmX2zjj6pe4+5e4t9EJ+I/gPgJCATWAwMq9Xnq8Cvg+eTgWeC58OC/lnEZzX7AIg2YV3nATnB83/6pK7g9b4Q99eNwC/qWLcLsCb42jl43rmp6qrV/+vAo6neX8H3/jxwOlB2lOWXArMAAwqB+aneX/Ws66xPtgdM+KSu4PU6oFtI++tc4M/Jvgcau65afS8jPndJU+yvnsDpwfP2wMo6/k+m7D3W0o8AxgKr3X2Nu1cAM4lPU5kojInpj1uXu7/q7geClyXEZ0VLtfrsr6O5BJjr7jvcfScwFxgfUl1TgBmNtO1jcvc3iM9lcTQTgSc9rgToZGY9Se3+Om5d7v5WsF1ouvdXffbX0STz3mzsupry/VXu7ouC53uJT6pVe370lL3HWnoAHJl0PlDX5PKfmpgeSJyY/njrprKuRLcQT/hPZJtZqZmVmNkVjVTTidT1leBQ8zkz+2Tqzmaxv4JTZQOBVxKaU7W/6uNotadyf52o2u8vB+aY2UIzKwqhnnFmttjMZpnZ8KCtWewvM8sh/kv09wnNTbK/LH56ejQwv9ailL3HGjwjmDQOM7sWKAD+IaG5v7tvNLOTgFfMbIm7f9BEJf0JmOHuh83sNuJHT+c30bbrYzLwnH96Brkw91ezZmbnEQ+AcxKazwn2V3dgrpmtCP5CbgqLiP+89pnZpcD/EJ8Strm4DPi7uyceLaR8f5lZO+Kh8y1339OY3/tYWvoRQH0mlw9jYvp6fW8zuxD4HnC5ux/+pN3dNwZf1wCvEf+roEnqcvftCbVMB8bUd91U1pVgMrUOz1O4v+rjaLWncn/Vi5mNJP4znOju2z9pT9hfW4A/0HinPo/L3fe4+77g+YtAhpl1oxnsr8Cx3l8p2V9mlkH8l/9v3f35Orqk7j2WigsbTfUgfgSzhvgpgU8uHA2v1edrfPoi8LPB8+F8+iLwGhrvInB96hpN/KLX4FrtnYGs4Hk3YBWNdDGsnnX1THj+JaDE//eC09qgvs7B8y5NVVfQbwjxC3LWFPsrYRsDOPpFzS/w6Qt0C1K9v+pZVz/i17XOqtXeFmif8PwtYHwT1pX3yc+P+C/S9cG+q9d7IFV1Bcs7Er9O0Lap9lfwb38SeOAYfVL2Hmu0nRvWg/gV8pXEf5l+L2i7i/hf1QDZwO+C/wwLgJMS1v1esN77wIQmruuvwGbg3eDxQtB+FrAk+A+wBLiliev6MbA02P6rwJCEdW8O9uNq4KamrCt4/QPgJ7XWS/X+mgGUA5XEz7HeAtwO3B4sN+C/g7qXAAVNtL+OV9d0YGfC+6s0aD8p2FeLg5/z95q4rn9OeH+VkBBQdb0HmqquoM+NxD8YkrheqvfXOcSvMbyX8LO6tKneYxoKQkQkTbX0awAiItJACgARkTSlABARSVMKABGRNKUAEBFJUwoAEZE0pQAQEUlT/z9Nl5tDosnuGwAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAAAp9UlEQVR4nO3deXyV5Z3//9fnnJOFsC+BsIOCsgQQiRjUdtwFW8W2joD7RnTa6WK/0/l1pt+p/dqOrcu0Vjttibi3gtbaqW0Foa61EjCgSFgEBAQx7Pua7fP749wyxxgg5OTkTnLez8fjPHLOdV937g93Dnnnvu9zX5e5OyIikn4iYRcgIiLhUACIiKQpBYCISJpSAIiIpCkFgIhImoqFXcCJ6Natmw8YMCDsMkREWoyFCxduc/fcupa1qAAYMGAApaWlYZchItJimNmHR1umU0AiImlKASAikqYUACIiaUoBICKSphQAIiJp6rgBYGaPmtkWMytLaOtiZnPNbFXwtfNR1r0h6LPKzG5IaB9jZkvMbLWZPWhm1jj/HBERqa/6HAE8Doyv1fZd4GV3Hwy8HLz+FDPrAtwJnAmMBe5MCIpfAVOBwcGj9vcXEZEUO24AuPsbwI5azROBJ4LnTwBX1LHqJcBcd9/h7juBucB4M+sJdHD3Eo+PRf3kUdZvNA++vIrZZeUcrKhO5WZERFqUht4I1sPdy4Pnm4AedfTpDWxIeP1R0NY7eF67vU5mVgQUAfTr1++ECz1QUcUTb61j+/4K2mREOffUXMbn53H+kO60z8444e8nItJaJH0nsLu7maVsVhl3LwaKAQoKCk54OzmZMeb/+wXMX7uDF5eU89LSzcwq20RmLMLnB3djfH5PLhrag445CgMRSS8NDYDNZtbT3cuDUzpb6uizETg34XUf4LWgvU+t9o0NrKNeYtEIZw/qxtmDunHXxHwWfriTWWXlvFS2ib8u30IsYow7uSsT8nty8fAedGuXlcpyRESaBavPlJBmNgD4s7vnB6/vA7a7+0/M7LtAF3f/11rrdAEWAqcHTYuAMe6+w8wWAN8A5gMvAg+5+4vHq6OgoMAbcywgd2fxR7uZVVbOrCWbWL/jABGDMwZ04dIRPblkeB55HbMbbXsiIk3NzBa6e0Gdy44XAGY2g/hf8t2AzcQ/2fM/wLNAP+BD4KrgF3sBcLu73xqsezPw78G3+k93fyxoLyD+6aI2wCzg616PJGrsAEjk7iwr38Pssk3MKtvE6i37ADi9Xycm5PdkfH4efbvkpGTbIiKpklQANCepDIDaVm/Zy6wl8TBYVr4HgPzeHY6Ewcm57ZqkDhGRZCgAkvTh9v3MCo4MFm/YBcCpPdozPj+PCSPyOLVHe3Qvm4g0RwqARvTxroPMLtvE7LJNvP3hDtxhYLe28TDIz2NE744KAxFpNhQAKbJl7yHmLN3M7LJNzFuzneoap3enNkwIjgxG9+1MJKIwEJHwKACawM79FcxdtplZZeW8uXobldVOjw5ZXDI8j/H5eYwd0IVYVGPviUjTUgA0sT2HKnll+RZmlZXz2vtbOVxVQ5e2mVw8rAcTRvRk3EldyYwpDEQk9RQAITpQUcVr72/lxSXlvLpiC/srqumQHePCYT2YkN+Tzw3uRnZGNOwyRaSVUgA0E4cqq/nbqm3MKivnr8s2s+dQFW0zo5w3pDsT8nty3pBccjKTHp1DROSIYwWAfts0oeyMKBcN68FFw3pQUVXDvDXbmV1Wzpylm/nze+VkxSKce2ouE/J7cv7Q7nTQYHUikkI6AmgGqqprWLBux5GPl27Ze5jMaISzB8XHJ7poWA86t80Mu0wRaYF0CqgFqalx3tmw88hdyBt3HSQaMQpP6nJksLru7TU+kYjUjwKghXJ3yjbuiQ9WV7aJtdv2YwZn9O/C+Pz4x0t7dWoTdpki0owpAFoBd+f9zfHxiWaXbeL9zXsBGNW3U/zGs/w8+ndtG3KVItLcKABaoTVb9zEruGawZONuAIb17HDkLuRB3duHXKGINAcKgFZuw44DwTDW5SxavwuAQd3bMSE4TTSsZweNTySSphQAaWTT7kO8tDQeBgvW7qDGoX/XnGCwup6M6qPB6kTSiQIgTW3bdzgYn2gTb63eRlWN06tjNpcEYTCmf2eiGqxOpFVLWQCY2TeBqYABD7v7A7WWfwe4JngZA4YCucHsYeuAvUA1UHW0AhMpABpu94FK5i7fzOyyct5YtY2Kqhpy22dx8bAeXDqiJ2cO1GB1Iq1RSgLAzPKBmcBYoAKYTXw6yNVH6X8ZcIe7nx+8XgcUuPu2+m5TAdA49h2u4pUVW5hdVs6rK7ZysLKazjkZXBSMT3TWoK5kxTQ+kUhrkKqhIIYC8939QLCR14EvA/cepf8UYEYS25NG0i4rxuWjenH5qF4crKjm9ZVbmVVWzotLNvFs6Ue0z4pxwdDujM/vybmn5mqwOpFWKpkjgKHAH4FxwEHgZaDU3b9eR98c4CNgkLvvCNrWAjsBB6a5e/FRtlMEFAH069dvzIcfftigeuX4DldV8/fV25i1ZBNzl29m14FKcjKjnHdqd8bn53HekO60y9LwUSItSSqvAdwCfBXYDywFDrv7t+roNwm41t0vS2jr7e4bzaw7MBf4uru/cazt6RRQ06msrmH+mh3MKivnpaWb2bbvMJmxCJ8fnMuE/DwuHNqDjjkarE6kuWuSTwGZ2d3AR+7+yzqW/QH4nbs/fZR1fwDsc/f7j7UNBUA4qmuc0nU7mFW2iZeWbqJ89yFiEePsQd2YkJ/HRcN60LVdVthlikgdUnkE0N3dt5hZP2AOUOjuu2r16QisBfq6+/6grS0Qcfe9wfO5wF3uPvtY21MAhK+mxln80a7gxrNNrN9xgIjBmQO7MmFEHpcMz6NHBw1WJ9JcpDIA/gZ0BSqBb7v7y2Z2O4C7/zrocyMw3t0nJ6x3EvCH4GUMeNrd//N421MANC/uzrLyPcwu28SLS8r5YGt8sLrT+3U+chdyn845YZcpktZ0I5g0iVWb9zIrODJYXr4HgJF9Oh65C3lgNw1WJ9LUFADS5NZt28/spfEwWLxhFwBD8tozPj+PS0f0ZHD3dhqSQqQJKAAkVBt3HQxmOyun9MOduMNJuW2DYax7MryXBqsTSRUFgDQbW/Yc4qVl8SEpStbsoLrG6dO5TXDNoCej+3YiovGJRBqNAkCapR37K/jrss28WFbO31dvo7LayeuQfWS2szMGdNFgdSJJUgBIs7f7YCWvrNjMrCWbeH3lVg5X1dCtXSYXDYvPdjbu5K5kaLA6kROmAJAWZf/hKl57Pz4+0asrtrC/opqObTK4cGgPJuTncc7gbhqfSKSeFADSYh2qrOaNlVuZXRYfn2jvoSraZcU4f0h3JuTn8Q+n5pKTqfGJRI4mVaOBiqRcdkaUi4fncfHwPCqqanjrg23MLtvEnGWbeWHxx2RnRDj3lO5MGJHH+UO60z5b4xOJ1JeOAKRFqqquYcG6HcHHSzexZe9hMqMRzhn8v+MTdcrJDLtMkdDpFJC0ajU1zqL1O5kVhMHGXQeJRYxxJ3dlfH4eFw/LI7e9BquT9KQAkLTh7izZuPtIGKzdtp+IQcGALlwa3GuQ11GD1Un6UABIWnJ33t+8lxeXxO9CXrl5HwCj+3U6chdy3y4arE5aNwWACPDB1n3BMNbllG2MD1Y3vFeHI3chD+reLuQKRRqfAkCklg07DhwJg0XrdwFwSo92jM/vyYT8PIbktdf4RNIqKABEjqF890FeCoaxfnvdDmocBnTNORIGI/t0VBhIi6UAEKmnbfsOM2fpZmaVlTPvg+1U1Ti9O7XhkuF5XDoij9P7ddZgddKipHJGsG8CUwEDHnb3B2otPxf4I/EpIQGed/e7gmXjgZ8DUWC6u//keNtTAEhT2nWggr8u38KsJeX8bdU2Kqpr6N4+i0uGx8cnGjuwCzGNTyTNXEoCwMzygZnAWKACmA3c7u6rE/qcC/yLu3+x1rpRYCVwEfAR8DYwxd2XHWubCgAJy95DlbyyYguzyzbx2vtbOVhZTZe2mVw0tAfjR+Rx9sndyIwpDKT5SdVQEEOB+e5+INjI68CXgXvrse5YYLW7rwnWnQlMBI4ZACJhaZ+dwcTTejPxtN4crKjm9ZVbmFW2ib8sKeeZ0g0M79WBp245ky5tdfextBzJ/MlSBnzOzLqaWQ5wKdC3jn7jzGyxmc0ys+FBW29gQ0Kfj4K2zzCzIjMrNbPSrVu3JlGuSONokxllfH5Pfj55NAv/40IemHQaq7fsY0pxCVv3Hg67PJF6a3AAuPty4B5gDvHTP+8C1bW6LQL6u/so4CHgfxqwnWJ3L3D3gtzc3IaWK5ISWbEoV4zuzWM3nsH6HQeYXDyPzXsOhV2WSL0kddLS3R9x9zHu/nlgJ/Hz+onL97j7vuD5i0CGmXUDNvLpo4U+QZtIi3TWoG48cfNYNu0+xKRp8/h418GwSxI5rqQCwMy6B1/7ET///3St5XkWfIDazMYG29tO/KLvYDMbaGaZwGTghWRqEQnb2IFdePKWM9m+r4JJxfPYsONA2CWJHFOyH1v4vZktA/4EfM3dd5nZ7WZ2e7D8SqDMzBYDDwKTPa4K+GfgJWA58Ky7L02yFpHQjenfmd/ceia7D1Qyado8Pty+P+ySRI5KN4KJpMDSj3dz7fT5ZMYiPD21kJNzNc6QhONYHwPVB5dFUmB4r47MLBpHdY0zaVoJKzfvDbskkc9QAIikyKl57ZlZNI6IweTiEpZ9vCfskkQ+RQEgkkKDurfjmdvGkRWLMOXhEpZ8tDvskkSOUACIpNjAbm159rZxtMuKcfX0Et5ZvzPskkQABYBIk+jbJYdnbx9Hl7aZXPfIAt5etyPskkQUACJNpXenNjxTNI7uHbK44dEFzPtge9glSZpTAIg0obyO2cwsKqR3pzbc9PgC/rZK41tJeBQAIk2se/t4CAzo2pZbnijl1RVbwi5J0pQCQCQEXdtlMWNqIaf0aEfRU6XMWbop7JIkDSkARELSuW0mv721kOG9OvLV3y7iL++Vh12SpBkFgEiIOrbJ4KlbxjK6Xye+PmMRf3xXg+JK01EAiISsfXYGj980lrEDu/CtZ97ld6Ubjr+SSCNQAIg0A22zYjx241jOGdSN7zz3Hk/PXx92SZIGFAAizUSbzCgPX1/Aeafm8u9/WMKT89aFXZK0cgoAkWYkOyPKr68bw0XDevD9Py5l+t/WhF2StGLJzgj2TTMrM7OlZvatOpZfY2bvmdkSM3vLzEYlLFsXtL9rZhrkXySQFYvyy2tO5wsjevKjvyznv19dHXZJ0krFGrqimeUDU4GxQAUw28z+7O6J79a1wD+4+04zmwAUA2cmLD/P3bc1tAaR1iojGuHnk08jFjXue+l9Kqtr+OYFgwlmWBVpFA0OAGAoMN/dDwCY2evE5wW+95MO7v5WQv8S4pO/i0g9xKIRfnrVaWREIzzw11VUVtfwLxefqhCQRpPMKaAy4HNm1tXMcoBLgb7H6H8LMCvhtQNzzGyhmRUdbSUzKzKzUjMr3bpV46ZIeolGjHu/MpIpY/vx369+wN0vLqclTeMqzVuDjwDcfbmZ3QPMAfYD7wLVdfU1s/OIB8A5Cc3nuPtGM+sOzDWzFe7+Rh3bKSZ+6oiCggK98yXtRCLG3V/KJzNqPPy3tVRWO3deNkxHApK0ZE4B4e6PAI8AmNndwEe1+5jZSGA6MMHdtyesuzH4usXM/kD8WsJnAkBEwMz4weXDyYhGmP7mWg5X1fCfV+QTiSgEpOGSCgAz6x78Au9H/Px/Ya3l/YDngevcfWVCe1sg4u57g+cXA3clU4tIa2dmfO8LQ8mMRfjlax9QWV3DPV8ZSVQhIA2UVAAAvzezrkAl8DV332VmtwO4+6+B7wNdgV8Gh6tV7l4A9AD+ELTFgKfdfXaStYi0embGdy45lcxY/MJwVXUN9//jKGJR3dIjJy7ZU0Cfq6Pt1wnPbwVuraPPGmBU7XYROT4z41sXnkJGNBJ8RNR5YHL800IiJyLZIwARCcnXzhtEVizCj/6ynMrqGh66ejRZsWjYZUkLoj8ZRFqwWz93Ev/v8uHMWbaZf/rNIg5V1vlBPJE6KQBEWrgbzhrA3V8awSsrtjD1yVIOVigEpH4UACKtwNVn9uPeK0fy5upt3Pz42xyoqAq7JGkBFAAircRVBX352VWnMX/tdm54dAF7D1WGXZI0cwoAkVbkitG9eWjK6Sxav4vrHlnA7oMKATk6BYBIK/OFkT355TWns/Tj3Vw7fT67DlSEXZI0UwoAkVbokuF5TLtuDO9v3suUh+ezfd/hsEuSZkgBINJKnT+kB9OvL2DN1n1MebiELXsPhV2SNDMKAJFW7POn5PLYTWewYcdBJheXsGm3QkD+lwJApJU76+RuPHnLWDbvPsSk4nls3HUw7JKkmVAAiKSBMwZ04albz2TH/gomTZvHhh0Hwi5JmgEFgEiaOL1fZ56+tZC9h6qYNG0e67btD7skCZkCQCSNjOjTkRlTCzlUVcNV0+axesu+sEuSECkARNLMsF4dmFlUSI3D5OJ5vL9pb9glSUgUACJp6JQe7ZlZVEjEjMnF81j68e6wS5IQJBUAZvZNMyszs6Vm9q06lpuZPWhmq83sPTM7PWHZDWa2KnjckEwdInLiBnVvx7O3jaNNRpSrH57Pex/tCrskaWINDgAzywemEp/MfRTwRTMbVKvbBGBw8CgCfhWs2wW4EzgzWP9OM+vc0FpEpGEGdGvLM7eNo312jGsens+i9TvDLkmaUDJHAEOB+e5+wN2rgNeJTwyfaCLwpMeVAJ3MrCdwCTDX3Xe4+05gLjA+iVpEpIH6dsnh2dvG0bVdJtdNn8+CtTvCLkmaSDIBUAZ8zsy6mlkOcCnQt1af3sCGhNcfBW1Ha/8MMysys1IzK926dWsS5YrI0fTq1IZnbhtHj47Z3PDoAt5avS3skqQJNDgA3H05cA8wB5gNvAs0+lRE7l7s7gXuXpCbm9vY315EAj06ZPNM0Tj6dmnDTY+/zRsr9QdXa5fURWB3f8Tdx7j754GdwMpaXTby6aOCPkHb0dpFJES57bOYMbWQk3LbcesTpbyyYnPYJUkKJfspoO7B137Ez/8/XavLC8D1waeBCoHd7l4OvARcbGadg4u/FwdtIhKyru2ymDH1TIb0bM9tTy1kdtmmsEuSFEn2PoDfm9ky4E/A19x9l5ndbma3B8tfBNYAq4GHga8CuPsO4IfA28HjrqBNRJqBTjmZ/ObWM8nv3ZGvPb2IP7/3cdglSQqYu4ddQ70VFBR4aWlp2GWIpI19h6u46bEFLPxwJ/911Si+NLpP2CXJCTKzhe5eUNcy3QksIkfVLivGEzeP5cyBXfn2s4t5tnTD8VeSFkMBICLHlJMZ49Ebz+CcQd341+fe47fzPwy7JGkkCgAROa42mVEevr6A84d053t/KOPxv68NuyRpBAoAEamX7Iwov752DJcM78EP/rSM4jc+CLskSZICQETqLTMW4RdXn84XRvbk7hdX8N+vrg67JElCLOwCRKRlyYhG+Pmk08iMRrjvpfepqKrhWxcOxszCLk1OkAJARE5YLBrh/n8cRSxi/PzlVVRU1/Cvl5yqEGhhFAAi0iDRiHHPV0aSGYvwq9c+oKKqhv/7haEKgRZEASAiDRaJGD+6Ip+MaIRH3lxLZXUNP7hsOJGIQqAlUACISFLMjDsvG0ZmLELxG2uorK7hP68YoRBoARQAIpI0M+PfJgwhMxrhF6+upqLKuffKkUQVAs2aAkBEGoWZ8S+XnEpmLMJP566kqqaG//rHUcSi+rR5c6UAEJFG9Y0LBhOLGvfOfp/K6hp+Pnk0GQqBZkkBICKN7qvnDiIzGuFHf1lOZfUifnH1aLJi0bDLkloUyyKSErd+7iTumjicucs2c/tTCzlU2egzxkqSFAAikjLXjxvAj788gtdWbuXWJ0o5WKEQaE6SnRLyDjNbamZlZjbDzLJrLf+Zmb0bPFaa2a6EZdUJy15Ipg4Rab6mjO3HfVeO4q0PtnHT4wvYf7gq7JIk0OAAMLPewDeAAnfPB6LA5MQ+7n6Hu5/m7qcBDwHPJyw++Mkyd7+8oXWISPN35Zg+/GzSaby9bic3PLqAvYcqwy5JSP4UUAxoY2YxIAc41sShU4AZSW5PRFqoiaf15qEpo3l3wy6ufWQBuw8qBMLW4ABw943A/cB6oBzY7e5z6uprZv2BgcArCc3ZZlZqZiVmdsXRtmNmRUG/0q1btza0XBFpBi4d0ZNfXnM6yz7ezTXTS9i5vyLsktJaMqeAOgMTif9i7wW0NbNrj9J9MvCcuydeAeofTFR8NfCAmZ1c14ruXuzuBe5ekJub29ByRaSZuHh4HsXXF7By8z6mPFzCtn2Hwy4pbSVzCuhCYK27b3X3SuLn9886St/J1Dr9ExxB4O5rgNeA0UnUIiItyHmndufRG85g3fb9TCkuYcueQ2GXlJaSCYD1QKGZ5Vh8/NcLgOW1O5nZEKAzMC+hrbOZZQXPuwFnA8uSqEVEWphzBnfjsRvHsnHXQSYXl7Bpt0KgqSVzDWA+8BywCFgSfK9iM7vLzBI/1TMZmOnuntA2FCg1s8XAq8BP3F0BIJJmxp3clSdvHsuWvYeZVDyPjbsOhl1SWrFP/15u3goKCry0tDTsMkSkkb2zfifXP7qADtkZzJhaSL+uOWGX1GqY2cLgeutn6E5gEQnd6H6dmTG1kP0VVUwqnsfabfvDLiktKABEpFnI792RGVMLOVxVw1XT5rF6y96wS2r1FAAi0mwM7dmBmUWFuMOkaSWs2LQn7JJaNQWAiDQrp/RozzO3FRKLGlOKSyjbuDvsklotBYCINDsn57bj2dvGkZMZ4+qHS1i8YVfYJbVKCgARaZb6d23LzKJCOuZkcO30+Sz8cEfYJbU6CgARabb6dsnhmaJxdGufxfWPLGD+mu1hl9SqKABEpFnr1akNzxQVktcxmxsfe5u/r94WdkmthgJARJq97h2ymVk0jn5dcrj58bd5faVGBm4MCgARaRFy22cxo6iQk3PbMfWJUv66bHPYJbV4CgARaTG6tM1kxtRChvZsz+2/WcjssvKwS2rRFAAi0qJ0zMngqVvPZGSfjnzt6Xf40+JjTUQox6IAEJEWp0N2Bk/eciZj+nfmmzPf4flFH4VdUoukABCRFqldVozHbzqDwpO68n9+t5hn394QdkktjgJARFqsnMwYj954Bp8bnMu//v49nir5MOySWpSkAsDM7jCzpWZWZmYzzCy71vIbzWyrmb0bPG5NWHaDma0KHjckU4eIpK/sjCjF143hgiHd+Y//KePRN9eGXVKLkcyk8L2BbwAF7p4PRInP/lXbM+5+WvCYHqzbBbgTOBMYC9wZTDIvInLCsjOi/OraMYwfnsddf17GtNc/CLukFiHZU0AxoI2ZxYAcoL6X4y8B5rr7DnffCcwFxidZi4ikscxYhIeuHs1lo3rx41kreOjlVWGX1OzFGrqiu280s/uJTw5/EJjj7nPq6PoVM/s8sBK4w903AL2BxCs2HwVtn2FmRUARQL9+/RparoikgYxohAcmnUZGxPivuSuprK7hjotOwczCLq1ZSuYUUGdgIjAQ6AW0NbNra3X7EzDA3UcS/yv/iRPdjrsXu3uBuxfk5uY2tFwRSRPRiHHfP45iUkFfHnxlNT+ZvYKWNPd5U0rmFNCFwFp33+rulcDzwFmJHdx9u7sfDl5OB8YEzzcCfRO69gnaRESSFo0YP/7yCK4t7Me019fwwz8vVwjUocGngIif+ik0sxzip4AuAEoTO5hZT3f/5F7ty4HlwfOXgLsTLvxeDPxbErWIiHxKJGL8cGI+GdEIj/59LZXVNfy/y4cTieh00CeSuQYw38yeAxYBVcA7QLGZ3QWUuvsLwDfM7PJg+Q7gxmDdHWb2Q+Dt4Nvd5e6a7UFEGpWZ8f0vDiMzFmHa62uorK7h7i+NUAgErCUdFhUUFHhpaenxO4qIJHB3fjZ3JQ++spovn96b+64cRTRNQsDMFrp7QV3LkjkFJCLSIpgZ3774VGLRCD+du5LKauenV40iI5regyEoAEQkbXzjgsFkxiL8ZNYKqqpr+Pnk0WTG0jcE0vdfLiJp6fZ/OJn/+OIwZpVt4qu/XcjhquqwSwqNAkBE0s4t5wzkh1fk89flWyh6ciGHKtMzBBQAIpKWrivszz1fGcEbq7ZyyxNvc6CiKuySmpwCQETS1qQz+nH/laOY98F2bnzsbfYdTq8QUACISFr7ypg+PDB5NAs/3MkNjy5gz6HKsEtqMgoAEUl7l4/qxS+mjGbxhl1cN30+uw+kRwgoAEREgAkjevLra8ewvHwvV08vYcf+irBLSjkFgIhI4MJhPSi+fgyrtuzj6odL2Lbv8PFXasEUACIiCc49tTuP3XgG67bvZ3JxCVv2HAq7pJRRAIiI1HL2oG48ftNYPt51kEnFJZTvPhh2SSmhABARqUPhSV156paxbNt7mEnTSvho54GwS2p0CgARkaMY078LT916JrsOVDBpWgkfbt8fdkmNSgEgInIMp/XtxNNTC9lfUcWkaSWs2bov7JIajQJAROQ48nt3ZGZRIZXVNUwqLmHV5r1hl9QokgoAM7vDzJaaWZmZzTCz7FrLv21my8zsPTN72cz6JyyrNrN3g8cLydQhIpJqQ/I6MLOoEIDJxSUsL98TckXJa3AAmFlv4BtAgbvnA1Fgcq1u7wTLRwLPAfcmLDvo7qcFj8sbWoeISFMZ3KM9zxQVkhGNMOXhEso27g67pKQkewooBrQxsxiQA3ycuNDdX3X3Ty6dlwB9ktyeiEioTsptx7O3jaNtZoyrHy7h3Q27wi6pwRocAO6+EbgfWA+UA7vdfc4xVrkFmJXwOtvMSs2sxMyuONpKZlYU9CvdunVrQ8sVEWk0/brm8MxthXTKyeTa6fMpXbcj7JIaJJlTQJ2BicBAoBfQ1syuPUrfa4EC4L6E5v7BRMVXAw+Y2cl1revuxe5e4O4Fubm5DS1XRKRR9ekcD4Hu7bO4/tEFlKzZHnZJJyyZU0AXAmvdfau7VwLPA2fV7mRmFwLfAy539yMDawRHELj7GuA1YHQStYiINLmeHdsws6iQ3p3acONjC3hz1bawSzohyQTAeqDQzHLMzIALgOWJHcxsNDCN+C//LQntnc0sK3jeDTgbWJZELSIioejeIZsZRYUM6NqWm594m1ff33L8lZqJZK4BzCf+yZ5FwJLgexWb2V1m9smneu4D2gG/q/Vxz6FAqZktBl4FfuLuCgARaZG6tctixtRCBndvx21PLmTuss1hl1Qv5u5h11BvBQUFXlpaGnYZIiJ12n2gkusfW8DSjbt5aMpoJozoGXZJmNnC4HrrZ+hOYBGRRtIxJ4Pf3DKWUX078c8z3uGP724Mu6RjUgCIiDSi9tkZPHnzWAr6d+aOZ97luYUfhV3SUSkAREQaWdusGI/fNJazTu7Gd55bzMwF68MuqU4KABGRFGiTGWX6DQV8fnAu331+CU/NWxd2SZ+hABARSZHsjCjF14/hwqE9+I8/LuWRN9eGXdKnKABERFIoKxbll9eczoT8PH7452X86rUPwi7pCAWAiEiKZcYiPDRlNJeP6sU9s1fw4Murwi4JiI/mKSIiKRaLRvjZpNOIRY2fzl1JRVUN/+fiU4gPpBBSTaFtWUQkzUQjxv1XjiIzGuEXr66msrqG704YEloIKABERJpQJGLc/aURZEQjTHtjDRXVNXz/i8NCCQEFgIhIE4tEjLsmDiczFuGRN9dSUVXDDyfmE4k0bQgoAEREQmBm/N8vDCUjGuHXr39AZXUNP/7ySKJNGAIKABGRkJgZ/9/4U8mMRXjw5VVUVjv3XTmSWLRpPqCpABARCZGZ8e2LTiEzatw/ZyWV1TX8bNJpZDRBCCgARESagX8+fzCZsQh3v7iCyuoaHppyOpmx1IaAbgQTEWkmij5/MndeNoyXlm7mn36zkEOV1SndXlIBYGZ3mNlSMyszsxlmll1reZaZPWNmq81svpkNSFj2b0H7+2Z2STJ1iIi0FjedPZAfXZHPyyu2MPXJ0pSGQIMDwMx6A98ACtw9H4gCk2t1uwXY6e6DgJ8B9wTrDgv6DgfGA780s2hDaxERaU2uLezPvV8ZyZurt3Hz429zoKIqJdtJ9hRQDGhjZjEgB/i41vKJwBPB8+eAC4IJ5CcCM939sLuvBVYDY5OsRUSk1bjqjL789KpRlKzZzo2PpiYEGnwR2N03mtn9wHrgIDDH3efU6tYb2BD0rzKz3UDXoL0kod9HQdtnmFkRUATQr1+/hpYrItLifGl0H2KRCG+u2kZ2rPFPkiRzCqgz8b/kBwK9gLZmdm1jFfYJdy929wJ3L8jNzW3sby8i0qxdNqoX91w5MiV3CSdzCuhCYK27b3X3SuB54KxafTYCfQGC00Qdge2J7YE+QZuIiDSRZAJgPVBoZjnBef0LgOW1+rwA3BA8vxJ4xd09aJ8cfEpoIDAYWJBELSIicoKSuQYw38yeAxYBVcA7QLGZ3QWUuvsLwCPAU2a2GthB8Ckhd19qZs8Cy4J1v+buqf3Aq4iIfIrF/yBvGQoKCry0tDTsMkREWgwzW+juBXUt053AIiJpSgEgIpKmFAAiImlKASAikqZa1EVgM9sKfNjA1bsB2xqxnMaiuk6M6joxquvEtMa6+rt7nXfRtqgASIaZlR7tSniYVNeJUV0nRnWdmHSrS6eARETSlAJARCRNpVMAFIddwFGorhOjuk6M6joxaVVX2lwDEBGRT0unIwAREUmgABARSVMtPgDMbHwwsfxqM/tuHctDmZi+HnV928yWmdl7ZvaymfVPWFZtZu8GjxeauK4bzWxrwvZvTVh2g5mtCh431F43xXX9LKGmlWa2K2FZKvfXo2a2xczKjrLczOzBoO73zOz0hGWp3F/Hq+uaoJ4lZvaWmY1KWLYuaH/XzBp1dMV61HWume1O+Hl9P2HZMd8DKa7rOwk1lQXvqS7BslTur75m9mrwu2CpmX2zjj6pe4+5e4t9EJ+I/gPgJCATWAwMq9Xnq8Cvg+eTgWeC58OC/lnEZzX7AIg2YV3nATnB83/6pK7g9b4Q99eNwC/qWLcLsCb42jl43rmp6qrV/+vAo6neX8H3/jxwOlB2lOWXArMAAwqB+aneX/Ws66xPtgdM+KSu4PU6oFtI++tc4M/Jvgcau65afS8jPndJU+yvnsDpwfP2wMo6/k+m7D3W0o8AxgKr3X2Nu1cAM4lPU5kojInpj1uXu7/q7geClyXEZ0VLtfrsr6O5BJjr7jvcfScwFxgfUl1TgBmNtO1jcvc3iM9lcTQTgSc9rgToZGY9Se3+Om5d7v5WsF1ouvdXffbX0STz3mzsupry/VXu7ouC53uJT6pVe370lL3HWnoAHJl0PlDX5PKfmpgeSJyY/njrprKuRLcQT/hPZJtZqZmVmNkVjVTTidT1leBQ8zkz+2Tqzmaxv4JTZQOBVxKaU7W/6uNotadyf52o2u8vB+aY2UIzKwqhnnFmttjMZpnZ8KCtWewvM8sh/kv09wnNTbK/LH56ejQwv9ailL3HGjwjmDQOM7sWKAD+IaG5v7tvNLOTgFfMbIm7f9BEJf0JmOHuh83sNuJHT+c30bbrYzLwnH96Brkw91ezZmbnEQ+AcxKazwn2V3dgrpmtCP5CbgqLiP+89pnZpcD/EJ8Strm4DPi7uyceLaR8f5lZO+Kh8y1339OY3/tYWvoRQH0mlw9jYvp6fW8zuxD4HnC5ux/+pN3dNwZf1wCvEf+roEnqcvftCbVMB8bUd91U1pVgMrUOz1O4v+rjaLWncn/Vi5mNJP4znOju2z9pT9hfW4A/0HinPo/L3fe4+77g+YtAhpl1oxnsr8Cx3l8p2V9mlkH8l/9v3f35Orqk7j2WigsbTfUgfgSzhvgpgU8uHA2v1edrfPoi8LPB8+F8+iLwGhrvInB96hpN/KLX4FrtnYGs4Hk3YBWNdDGsnnX1THj+JaDE//eC09qgvs7B8y5NVVfQbwjxC3LWFPsrYRsDOPpFzS/w6Qt0C1K9v+pZVz/i17XOqtXeFmif8PwtYHwT1pX3yc+P+C/S9cG+q9d7IFV1Bcs7Er9O0Lap9lfwb38SeOAYfVL2Hmu0nRvWg/gV8pXEf5l+L2i7i/hf1QDZwO+C/wwLgJMS1v1esN77wIQmruuvwGbg3eDxQtB+FrAk+A+wBLiliev6MbA02P6rwJCEdW8O9uNq4KamrCt4/QPgJ7XWS/X+mgGUA5XEz7HeAtwO3B4sN+C/g7qXAAVNtL+OV9d0YGfC+6s0aD8p2FeLg5/z95q4rn9OeH+VkBBQdb0HmqquoM+NxD8YkrheqvfXOcSvMbyX8LO6tKneYxoKQkQkTbX0awAiItJACgARkTSlABARSVMKABGRNKUAEBFJUwoAEZE0pQAQEUlT/z9Nl5tDosnuGwAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -281,7 +864,6 @@ } ], "source": [ - "from fugue import FugueWorkflow\n", "from fugue.rpc import RPCHandler\n", "import pandas as pd\n", "import random\n", @@ -337,12 +919,12 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfH0lEQVR4nO3deXhU9d338fd3JgmBsARIQCCEJYCAgETCIiKCYIsbaF1ApfejdUGBVrs+tne329reVlurVVyoonV/3FqxtaJVFrUKBAQVZEcgoBBl3wLJfJ8/ZqApBgkwyZmZfF7XlYuZc05mPnDBJz9+c87vmLsjIiLJLxR0ABERiQ8VuohIilChi4ikCBW6iEiKUKGLiKSItKDeOCcnx9u3bx/U24uIJKV58+Z97u65Ve0LrNDbt29PcXFxUG8vIpKUzGzN4fZpykVEJEWo0EVEUoQKXUQkRajQRURShApdRCRFqNBFRFKECl1EJEUkXaEv37iDO19fxu595UFHERFJKElX6G8u2cQf31jOsN/P5KUF69F67iIiUUlX6OPOKOC560+lecMMbnxmAZc88C4flmwLOpaISOCSrtAB+rZvxksTBvHbi3ryyRe7GDnpbf7v8x9QuqMs6GgiIoFJykIHCIeM0X3zefMHQ7j29I68+H4JZ/5uBn+atYp95ZGg44mI1LqkLfQDGmem85NzujHtpsH07dCMX7/yMV+/axZvLtkYdDQRkVqV9IV+QMfchky5si+PXNUXM/jWo8Vc+cgcVmzaGXQ0EZFakTKFfsDQE1sw7abB/PTcbsxbs4URd83iV39bzLY9+4OOJiJSo6pV6GY2wsyWmtkKM7u5iv35ZjbdzN43sw/M7Jz4R62+9HCIa07vyPQfDOGSorZMeWc1Z/5uBk/PWUtFRKc5ikhqOmKhm1kYmAScDXQHLjOz7occ9lPgWXcvBMYA98U76LHIaViP//1GT16eOIiC3Ib8+MUPOf+et5m96ougo4mIxF11Ruj9gBXuvsrd9wHPAKMOOcaBxrHHTYAN8Yt4/Hq0acL/GzeAey8vZOvufYye/B4TnprP+q17go4mIhI31bkFXRtgXaXnJUD/Q475JfCamX0byAKGxyVdHJkZ5/VqzbCuLZk8axX3z1zBPxdv5PozCrj+jALqZ4SDjigiclzi9aHoZcCj7p4HnAM8bmZfem0zu87Mis2suLS0NE5vfXTqZ4S5cXhn3vj+EL520gnc/cZyhv1+BlMXbtAyAiKS1KpT6OuBtpWe58W2VXY18CyAu78LZAI5h76Qu0929yJ3L8rNrfKm1bWmTXZ97rmskGfHnUrTrAy+8/T7XPrgu3y0XssIiEhyqk6hzwU6m1kHM8sg+qHn1EOOWQsMAzCzbkQLPZgh+FHq16EZUycO4rZv9GRV6S7Ov/dtbn7hAz7fqWUERCS5HLHQ3b0cmAhMAz4mejbLIjO7xcxGxg77PnCtmS0Engau9CSavwiHjDH98pn+wyFcM6gDz88rYegdM3joLS0jICLJw4Lq3aKiIi8uLg7kvY9kZelObv3bYqYvLaVjThY/O687Q7u2CDqWiAhmNs/di6ral3JXisZDQW5DHrmqH49c2ReAqx6dy1WPzGFlqZYREJHEpUL/CkO7tuDV2DICxZ9s4et/mMWtf1vM9r1aRkBEEo8K/Qgy0mLLCPxwCJcU5fHwO6sZescMntEyAiKSYFTo1RRdRqAXL08cRMfcLG5+8UNG3vs2c1ZvDjqaiAigQj9qPdo04dlxp3LPZYVs2bWPSx98l4laRkBEEoAK/RiYGeef3Jo3vj+EG4d15vXFGxn2+xlMmr5CV5uKSGBU6MehfkaY757VhTd/MIQzuuRyx7SlzFr+edCxRKSOUqHHQXQZgVNo1SSTSW+uCDqOiNRRKvQ4yUgLcd3gjsz5ZLM+KBWRQKjQ42hM33yaZ2Vw73SN0kWk9qnQ46h+RpirT+/ArGWlfFCyNeg4IlLHqNDj7JsD2tE4M41JGqWLSC1TocdZo8x0rhzYnmmLNrJs446g44hIHaJCrwFXndaBBhlh7tMoXURqkQq9BjTNyuDyfvlMXbiBtV/sDjqOiNQRKvQacu3gjqSFQtw/c2XQUUSkjlCh15CWjTO5pCiPF+aV8Nm2vUHHEZE6QIVeg64/o4AKdybPWhV0FBGpA1ToNahtswaM6t2ap+as4QvddFpEapgKvYaNH9KJsvIIU95ZHXQUEUlxKvQa1qlFQ87ucQKP/WsN2/bo1nUiUnNU6LVg/JBO7Cgr5/F3Pwk6ioikMBV6LejRpglDT8zl4bdXs3tfedBxRCRFqdBrycQzO7Fl936emr026CgikqJU6LWkT7tmDOjYjD+9tYqy8oqg44hIClKh16IJQzuxcXsZz88rCTqKiKQgFXotGtQph5PzmvDAzJWUV0SCjiMiKUaFXovMjAlDO7Fu8x5e/mBD0HFEJMWo0GvZ8G4tObFlI+6bvpJIxIOOIyIpRIVey0IhY/zQApZv2slriz8LOo6IpBAVegDO69Wa9s0bcO/0FbhrlC4i8aFCD0A4ZNwwpICP1m9n5rLSoOOISIpQoQfkwsI8WjfJ1M2kRSRuVOgByUgLcd3gjsz9ZAuzV30RdBwRSQEq9ACN6ZdPTsMM7tUoXUTiQIUeoMz0MFcP6shbyz9n4bqtQccRkSSnQg/Y2AH5NM5M01y6iBw3FXrAGmWmc+XA9ry2eCPLNu4IOo6IJLFqFbqZjTCzpWa2wsxuPswxl5rZYjNbZGZPxTdmarvqtA40yAhzn0bpInIcjljoZhYGJgFnA92By8ys+yHHdAZ+DJzm7icBN8U/aupqmpXBFf3zmbpwA2u+2BV0HBFJUtUZofcDVrj7KnffBzwDjDrkmGuBSe6+BcDdN8U3Zuq79vSOpIVDPDBzZdBRRCRJVafQ2wDrKj0viW2rrAvQxczeMbP3zGxEvALWFS0aZ3JpUR7Pzyvh0217go4jIkkoXh+KpgGdgSHAZcCfzCz70IPM7DozKzaz4tJSXfJ+qHGDC4g4TJ61KugoIpKEqlPo64G2lZ7nxbZVVgJMdff97r4aWEa04P+Du0929yJ3L8rNzT3WzCmrbbMGXNC7DU/PWcvnO8uCjiMiSaY6hT4X6GxmHcwsAxgDTD3kmL8SHZ1jZjlEp2A0zDwG44cWUFYeYcrbq4OOIiJJ5oiF7u7lwERgGvAx8Ky7LzKzW8xsZOywacAXZrYYmA780N21QMkxKMhtyDk9WvH4u2vYtmd/0HFEJIlYUOtxFxUVeXFxcSDvnegWbdjGuX98m++f1YVvD/vSzJWI1GFmNs/di6rapytFE9BJrZtwZtcWTHlnNbvKyoOOIyJJQoWeoCYMLWDL7v08PWdt0FFEJEmo0BNUn3bNGNCxGZNnraKsvCLoOCKSBFToCWzi0M5s2lHG8/NKgo4iIklAhZ7ATuvUnJPbZvPAzJWUV0SCjiMiCU6FnsDMjIlDO7Fu8x6mLtwQdBwRSXAq9AQ3rGsLup7QiPtmrCQSCeYUUxFJDir0BBcKGeOHdmLFpp1MW/RZ0HFEJIGp0JPAuT1b0SEni3unryCoC8FEJPGp0JNAOGTccEYBizZsZ8YyrVIpIlVToSeJCwrb0LpJJpPe1ChdRKqmQk8SGWkhxp1RQPGaLcxevTnoOCKSgFToSWR037bkNMxgkm4mLSJVUKEnkcz0MFcP6shbyz9n4bqtQccRkQSjQk8yYwfk0zgzTaN0EfkSFXqSaZSZzpWndeC1xRtZ+tmOoOOISAJRoSehqwa2p0FGmPtmaJQuIv+mQk9CTbMyGDugHS8v3MAnn+8KOo6IJAgVepK6ZlAH0sIhHpi5MugoIpIgVOhJqkXjTEYXteWF+SVs2Lon6DgikgBU6Els3BkdcYfJs1YFHUVEEoAKPYnlNW3ABYVteGbuWj7fWRZ0HBEJmAo9yd0wpICy8ggPv7066CgiEjAVepIryG3IOT1b8fi7a9i2e3/QcUQkQCr0FDB+SAE7y8r587ufBB1FRAKkQk8BJ7VuwpldWzDlndXsKisPOo6IBESFniImDO3E1t37eXrO2qCjiEhAVOgpok+7ppzasTmTZ61i7/6KoOOISABU6Clk4pmd2LSjjOfnlQQdRUQCoEJPIQMLmtO7bTYPzFzJ/opI0HFEpJap0FOImTFxaCdKtuxh6oINQccRkVqmQk8xw7q1oOsJjbhvxgoiEd1MWqQuUaGnGDNjwtBOrCzdxauLPgs6jojUIhV6CjqnZys65mQxafoK3DVKF6krVOgpKBwyrh9SwKIN25mxtDToOCJSS1ToKerCwja0ya7PvRqli9QZKvQUlR4Ocd3gjsxbs4XZqzcHHUdEaoEKPYWN7tuWnIb1mDRdN5MWqQuqVehmNsLMlprZCjO7+SuOu8jM3MyK4hdRjlVmephrT+/AW8s/13rpInVA2pEOMLMwMAk4CygB5prZVHdffMhxjYAbgdk1EVSOzbcGdWDBuq386m+LcXeuOb1j0JFEpIZUZ4TeD1jh7qvcfR/wDDCqiuN+BfwW2BvHfHKc0sMh/nhZIef2bMWtf/+YybNWBh1JRGpIdQq9DbCu0vOS2LaDzOwUoK27//2rXsjMrjOzYjMrLi3V6XS1JT0c4u4xvTmvVyt+88oSHpipUhdJRUeccjkSMwsBdwJXHulYd58MTAYoKirSuXS1KC0c4q7RvTEzbvvHEiLujB/SKehYIhJH1Sn09UDbSs/zYtsOaAT0AGaYGcAJwFQzG+nuxfEKKscvLRziD5eeTMjg9leXEok4E8/sHHQsEYmT6hT6XKCzmXUgWuRjgMsP7HT3bUDOgedmNgP4gco8MaWFQ9x5aW9CZvzutWVEHL4zTKUukgqOWOjuXm5mE4FpQBiY4u6LzOwWoNjdp9Z0SImvcMj43SUnYwZ3vr6MiDs3De8SdCwROU7VmkN391eAVw7Z9vPDHDvk+GNJTQuHjDsuPpmQGXf9czkRh+8O70xs2kxEktBxfygqySscMm6/qBchgz++sRx353tndVGpiyQpFXodFwoZt32jFyEz7nlzBRF3fvC1E1XqIklIhS6EQsZvLuyJmTFp+koiDj/6ukpdJNmo0AWIlvqvL+hByOD+GSuJuHPziK4qdZEkokKXg0Ih49YLehAy48GZq3CHH5+tUhdJFip0+Q9mxi2jTiJkMHnWKioizk/P7aZSF0kCKnT5EjPjlyNPwsx4+O3VRNz5+XndVeoiCU6FLlUyM35xfndCZkx5ZzXu8IvzVeoiiUyFLodlZvzsvG6EDB6KjdT/JzZyF5HEo0KXr2Rm/Pe53QiFjMmzVhFx55aRPQiFVOoiiUaFLkdkZrGzXeDBmauIONw6SqUukmhU6FItZsbNI7oSMuP+GStxd359QU+VukgCUaFLtZkZP/r6iYTNuHf6CiIR+N9vqNRFEoUKXY6KmfH9r3WJLugVW/vltxf1UqmLJAAVuhw1M+N7sQW87n4juvTu7Rf3IqxSFwmUCl2O2XfP6oIZ3PXP6NK7d1xyskpdJEAqdDkuNw3vQsjs4J2Pfn9pb5W6SEBU6HLcvjOsc/QOSNOW4sDvLzmZtHAo6FgidY4KXeJiwtBOmMHtry4l4vCHS1XqIrVNhS5xM35IJ0Jm3PaPJUTcuWt0b9JV6iK1RoUucXX9GQWEDH7zyhLcnbvHFKrURWqJCl3i7rrBBYTMuPXvHxOJvM89l6vURWqD/pVJjbjm9I787LzuvLroMyY+NZ995ZGgI4mkPBW61JirB3Xgl+d3Z9qijUxQqYvUOBW61KgrT+vALaNO4vXFGxn/5DzKyiuCjiSSslToUuP+69T2/GrUSfzz403c8MR8lbpIDVGhS6345qntufWCHry5ZBPXPz6PvftV6iLxpkKXWjN2QDt+c2FPpi8tZZxKXSTudNqi1KrL++cTDsHNL37If02Zw8CC5qSHQ6SFjLRwiPSwkRY68Dy2LbYvLWykh2K/HjgubAe/Pz12TFoo9jqVtmt9GakLVOhS60b3zcfM+MVLi5izenOtvKcZB38YhENV/RD49+P66WG6tWpMYX42vds2pX3zBroxtiQFc/dA3rioqMiLi4sDeW9JHJGIsz8SobzCKa/49+P9FRHKI055RYT9FU555MDz2LZD9x3me/ZXOBWRr/qeL3//zr3lLNqwjV37olNC2Q3SOTkvO1bw0a/sBhkB/8lJXWVm89y9qKp9GqFLoEIho14oTL0E+5tYEXGWb9rBgrVbWbBuK++v3crdbyznwPinY05WtNzzsyls25SurRrpalgJnEboItW0s6ycD0qi5b5gXfSrdEcZAPXSQvRo04TCWMn3bptNm+z6mqqRuPuqEboKXeQYuTvrt+6Jlnus5D9cv42y2BWxOQ3rHZymKWybTa+22TRMtP+KSNLRlItIDTAz8po2IK9pA87r1RqA/RURlny6gwXrthwcyb++eGPseOjSotG/p2rys+ncopHOwJG40QhdpIZt3b3v4BTNga+tu/cDkJURpmdeEwrzmx4cybdonBlwYklkGqGLBCi7QQZDTmzBkBNbANGpmk++2M2CdVtYsHYr76/byp9mraI8Eh1ctW6SebDge+dn07NNEzLTw0H+FiRJqNBFapmZ0SEniw45WVxYmAfA3v0VLNqwnffXbjk4iv/7h58CkBYyurZqRFG7ZkwY2oncRvWCjC8JrFqFbmYjgLuBMPCQu992yP7vAdcA5UAp8C13XxPnrCIpKzM9TJ92TenTrunBbaU7ymLlHi35p+as5fXFG5lyZV9OPKFRgGklUR1xDt3MwsAy4CygBJgLXObuiysdMxSY7e67zewGYIi7j/6q19UcusjR+bBkG1f/eS6791Vwz+WFDI1N4Ujd8lVz6NW5EqIfsMLdV7n7PuAZYFTlA9x9urvvjj19D8g7nsAi8mU985rw0sTTaNe8AVc/OpdH31kddCRJMNUp9DbAukrPS2LbDudq4B9V7TCz68ys2MyKS0tLq59SRABo1aQ+z447lWHdWvLLlxfz85c+orxCd4KSqLheq2xmY4Ei4I6q9rv7ZHcvcvei3NzceL61SJ2RVS+NB8f2Ydzgjjz27hq+9editu/dH3QsSQDVKfT1QNtKz/Ni2/6DmQ0H/hsY6e5l8YknIlUJhYwfn9ON317Uk3+t+JyL7vsX6zbvPvI3SkqrTqHPBTqbWQczywDGAFMrH2BmhcCDRMt8U/xjikhVRvfN57Gr+7FpRxkXTHqHeWtqZzliSUxHLHR3LwcmAtOAj4Fn3X2Rmd1iZiNjh90BNASeM7MFZjb1MC8nInE2sCCHv4wfSKPMNC7702xeWvCl/0BLHaFL/0VSxJZd+7j+iXnMXr2Z7wzrzHeHd9ZqjynoeE9bFJEk0DQrg8ev7s8lffL44xvL+c4zC3Tf1jpGl/6LpJCMtBC3X9yLjrkN+e2rSyjZspvJ3yzScgF1hEboIinGzLhhSAEPjD2Fjz/dzgWT3mHpZzuCjiW1QIUukqJG9GjFc+MGsr8iwkX3/4vpS3UCWqpToYukMC0XULeo0EVSnJYLqDtU6CJ1QFa9NB7QcgEpT4UuUkeEY8sF3PYNLReQqlToInXMmH75PPatfmzcvlfLBaQYFbpIHTSwUw5/nXCalgtIMSp0kTqqY25D/jL+NArbZnPjMwu48/VlBLUUiMSHCl2kDjuwXMDFWi4gJejSf5E6LiMtxB0X96JAywUkPY3QRUTLBaQIFbqIHKTlApKbCl1E/sOB5QLym2m5gGSjQheRL2nVpD7PXX8qZ3bVcgHJRIUuIlXKqpfGg9/sw3VaLiBpqNBF5LDCIeMnWi4gaajQReSItFxAclChi0i1DOyUw1+0XEBCU6GLSLUVxJYL6K3lAhKSCl1EjkrTrAyeOGS5gM279gUdS9Cl/yJyDCovF3D7tCVM++gzzul5AmMHtKNPu6aYWdAR6yQVuogckwPLBQzr1oIn31vDi/PX89cFG+h6QiOu6J/PBYVtaJSZHnTMOsWCmv8qKiry4uLiQN5bROJvV1k5Uxdu4In31rBow3ayMsKMKmzD2P7t6N66cdDxUoaZzXP3oir3qdBFJJ7cnYUl23jivTW8vHADZeURCvOzGdu/Hef2akVmejjoiElNhS4igdi6ex8vzF/Pk7PXsKp0F9kN0rn4lDyuGNCODjlZQcdLSip0EQmUu/Puyi94cvZapi36jPKIc1qn5ozt347h3VuSHtYJd9WlQheRhLFp+16eLV7H03PWsX7rHlo0qseYvm0Z0y+f1tn1g46X8FToIpJwKiLO9CWbeHL2GmYsK8WAYd1ackX/fAZ3ziUU0qmPVfmqQtdpiyISiHDIGN69JcO7t2Td5t08NWctz85dx+uLN5LfrAGX98/nkj55NG+oW+FVl0boIpIwysormLZoI0+8t4Y5qzeTEQ5xduyCpSJdsARoykVEktCyjTt4avZaXphXwo6yck5s2YgrBuRzYR2/YEmFLiJJa/e+cl5euIEn3lvLh+u30SAjzKjerbmifzt6tGkSdLxap0IXkZSwcN1Wnpy9hqkLN7B3f4TebbO5on8+55/cus5csKRCF5GUsm33fl6YX8KTs9ewsnQXTeqnc3GfPC7vn09BbsOg49UoFbqIpCR3571Vm3li9hqmfRS9YGlgQXPGDmjHWSl6wdJxn7ZoZiOAu4Ew8JC733bI/nrAY0Af4AtgtLt/cjyhRUSOxMw4taA5pxY0Z9OOvTxXXMJTs9cy/sn55DTMoF3zLLLqpdGwXpiG9dJij9MO/zgzjazYsfXTw0l3Vs0RR+hmFgaWAWcBJcBc4DJ3X1zpmPFAL3e/3szGABe6++ivel2N0EWkJlREnJnLNvHX9zfwxa4ydpZVsHPvfnaVVbCrrJyd+8qpzsREyKiy9KOFn07DeuHotszY9oxKj//j+8JkZaTF7UKp4x2h9wNWuPuq2Is9A4wCFlc6ZhTwy9jj54F7zcxc96YSkVoWDhlndm3JmV1bVrk/EnH27I+W+46y8mjJl5Wzc285u/aVs/NA8e+Nbj+4P/Z404697CqrOLitIlK9msvK+PcPgJuGd2Hkya3j+dsGqlfobYB1lZ6XAP0Pd4y7l5vZNqA58Hnlg8zsOuA6gPz8/GOMLCJy7EIhIys2im5xnK/l7pSVRw6W/Y690V937TvwuOJLPxB2lpXTtEHNnEdfq5f+u/tkYDJEp1xq871FROLNzMhMD5OZHiYnAZYoqM5HwOuBtpWe58W2VXmMmaUBTYh+OCoiIrWkOoU+F+hsZh3MLAMYA0w95JipwP+JPb4YeFPz5yIiteuIUy6xOfGJwDSipy1OcfdFZnYLUOzuU4GHgcfNbAWwmWjpi4hILarWHLq7vwK8csi2n1d6vBe4JL7RRETkaKTeZVQiInWUCl1EJEWo0EVEUoQKXUQkRQS22qKZlQJrjvHbczjkKtQEoVxHR7mOXqJmU66jczy52rl7blU7Aiv042FmxYdbnCZIynV0lOvoJWo25To6NZVLUy4iIilChS4ikiKStdAnBx3gMJTr6CjX0UvUbMp1dGokV1LOoYuIyJcl6whdREQOoUIXEUkRSVfoZjbCzJaa2QozuznoPABmNsXMNpnZR0FnqczM2prZdDNbbGaLzOzGoDMBmFmmmc0xs4WxXP8TdKbKzCxsZu+b2d+CznKAmX1iZh+a2QIzS5ib8ZpZtpk9b2ZLzOxjMzs1ATKdGPtzOvC13cxuCjoXgJl9N/Z3/iMze9rMMuP6+sk0h16dG1YHlGswsBN4zN17BJmlMjNrBbRy9/lm1giYB1yQAH9eBmS5+04zSwfeBm509/eCzHWAmX0PKAIau/t5QeeBaKEDRe6eUBfJmNmfgbfc/aHY/RIauPvWgGMdFOuM9UB/dz/WCxnjlaUN0b/r3d19j5k9C7zi7o/G6z2SbYR+8IbV7r4POHDD6kC5+yyi68AnFHf/1N3nxx7vAD4mev/XQHnUztjT9NhXQowszCwPOBd4KOgsic7MmgCDid4PAXffl0hlHjMMWBl0mVeSBtSP3dmtAbAhni+ebIVe1Q2rAy+oZGBm7YFCYHbAUYCD0xoLgE3A6+6eELmAu4AfAZGAcxzKgdfMbF7sZuuJoANQCjwSm6J6yMyygg51iDHA00GHAHD39cDvgLXAp8A2d38tnu+RbIUux8DMGgIvADe5+/ag8wC4e4W79yZ6j9p+Zhb4VJWZnQdscvd5QWepwiB3PwU4G5gQm+YLWhpwCnC/uxcCu4CE+FwLIDYFNBJ4LugsAGbWlOiMQgegNZBlZmPj+R7JVujVuWG1VBKbo34BeNLdXww6z6Fi/0WfDowIOArAacDI2Hz1M8CZZvZEsJGiYqM73H0T8Bei049BKwFKKv3v6nmiBZ8ozgbmu/vGoIPEDAdWu3upu+8HXgQGxvMNkq3Qq3PDaomJffj4MPCxu98ZdJ4DzCzXzLJjj+sT/ZB7SaChAHf/sbvnuXt7on+33nT3uI6gjoWZZcU+1CY2pfE1IPAzqtz9M2CdmZ0Y2zQMCPQD90NcRoJMt8SsBQaYWYPYv81hRD/Xiptq3VM0URzuhtUBx8LMngaGADlmVgL8wt0fDjYVEB1xfhP4MDZfDfCT2D1ig9QK+HPsDIQQ8Ky7J8wpggmoJfCXaAeQBjzl7q8GG+mgbwNPxgZYq4CrAs4DHPzBdxYwLugsB7j7bDN7HpgPlAPvE+clAJLqtEURETm8ZJtyERGRw1Chi4ikCBW6iEiKUKGLiKQIFbqISIpQoYuIpAgVuohIivj/KR1KREmtfbgAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfH0lEQVR4nO3deXhU9d338fd3JgmBsARIQCCEJYCAgETCIiKCYIsbaF1ApfejdUGBVrs+tne329reVlurVVyoonV/3FqxtaJVFrUKBAQVZEcgoBBl3wLJfJ8/ZqApBgkwyZmZfF7XlYuZc05mPnDBJz9+c87vmLsjIiLJLxR0ABERiQ8VuohIilChi4ikCBW6iEiKUKGLiKSItKDeOCcnx9u3bx/U24uIJKV58+Z97u65Ve0LrNDbt29PcXFxUG8vIpKUzGzN4fZpykVEJEWo0EVEUoQKXUQkRajQRURShApdRCRFqNBFRFKECl1EJEUkXaEv37iDO19fxu595UFHERFJKElX6G8u2cQf31jOsN/P5KUF69F67iIiUUlX6OPOKOC560+lecMMbnxmAZc88C4flmwLOpaISOCSrtAB+rZvxksTBvHbi3ryyRe7GDnpbf7v8x9QuqMs6GgiIoFJykIHCIeM0X3zefMHQ7j29I68+H4JZ/5uBn+atYp95ZGg44mI1LqkLfQDGmem85NzujHtpsH07dCMX7/yMV+/axZvLtkYdDQRkVqV9IV+QMfchky5si+PXNUXM/jWo8Vc+cgcVmzaGXQ0EZFakTKFfsDQE1sw7abB/PTcbsxbs4URd83iV39bzLY9+4OOJiJSo6pV6GY2wsyWmtkKM7u5iv35ZjbdzN43sw/M7Jz4R62+9HCIa07vyPQfDOGSorZMeWc1Z/5uBk/PWUtFRKc5ikhqOmKhm1kYmAScDXQHLjOz7occ9lPgWXcvBMYA98U76LHIaViP//1GT16eOIiC3Ib8+MUPOf+et5m96ougo4mIxF11Ruj9gBXuvsrd9wHPAKMOOcaBxrHHTYAN8Yt4/Hq0acL/GzeAey8vZOvufYye/B4TnprP+q17go4mIhI31bkFXRtgXaXnJUD/Q475JfCamX0byAKGxyVdHJkZ5/VqzbCuLZk8axX3z1zBPxdv5PozCrj+jALqZ4SDjigiclzi9aHoZcCj7p4HnAM8bmZfem0zu87Mis2suLS0NE5vfXTqZ4S5cXhn3vj+EL520gnc/cZyhv1+BlMXbtAyAiKS1KpT6OuBtpWe58W2VXY18CyAu78LZAI5h76Qu0929yJ3L8rNrfKm1bWmTXZ97rmskGfHnUrTrAy+8/T7XPrgu3y0XssIiEhyqk6hzwU6m1kHM8sg+qHn1EOOWQsMAzCzbkQLPZgh+FHq16EZUycO4rZv9GRV6S7Ov/dtbn7hAz7fqWUERCS5HLHQ3b0cmAhMAz4mejbLIjO7xcxGxg77PnCtmS0Engau9CSavwiHjDH98pn+wyFcM6gDz88rYegdM3joLS0jICLJw4Lq3aKiIi8uLg7kvY9kZelObv3bYqYvLaVjThY/O687Q7u2CDqWiAhmNs/di6ral3JXisZDQW5DHrmqH49c2ReAqx6dy1WPzGFlqZYREJHEpUL/CkO7tuDV2DICxZ9s4et/mMWtf1vM9r1aRkBEEo8K/Qgy0mLLCPxwCJcU5fHwO6sZescMntEyAiKSYFTo1RRdRqAXL08cRMfcLG5+8UNG3vs2c1ZvDjqaiAigQj9qPdo04dlxp3LPZYVs2bWPSx98l4laRkBEEoAK/RiYGeef3Jo3vj+EG4d15vXFGxn2+xlMmr5CV5uKSGBU6MehfkaY757VhTd/MIQzuuRyx7SlzFr+edCxRKSOUqHHQXQZgVNo1SSTSW+uCDqOiNRRKvQ4yUgLcd3gjsz5ZLM+KBWRQKjQ42hM33yaZ2Vw73SN0kWk9qnQ46h+RpirT+/ArGWlfFCyNeg4IlLHqNDj7JsD2tE4M41JGqWLSC1TocdZo8x0rhzYnmmLNrJs446g44hIHaJCrwFXndaBBhlh7tMoXURqkQq9BjTNyuDyfvlMXbiBtV/sDjqOiNQRKvQacu3gjqSFQtw/c2XQUUSkjlCh15CWjTO5pCiPF+aV8Nm2vUHHEZE6QIVeg64/o4AKdybPWhV0FBGpA1ToNahtswaM6t2ap+as4QvddFpEapgKvYaNH9KJsvIIU95ZHXQUEUlxKvQa1qlFQ87ucQKP/WsN2/bo1nUiUnNU6LVg/JBO7Cgr5/F3Pwk6ioikMBV6LejRpglDT8zl4bdXs3tfedBxRCRFqdBrycQzO7Fl936emr026CgikqJU6LWkT7tmDOjYjD+9tYqy8oqg44hIClKh16IJQzuxcXsZz88rCTqKiKQgFXotGtQph5PzmvDAzJWUV0SCjiMiKUaFXovMjAlDO7Fu8x5e/mBD0HFEJMWo0GvZ8G4tObFlI+6bvpJIxIOOIyIpRIVey0IhY/zQApZv2slriz8LOo6IpBAVegDO69Wa9s0bcO/0FbhrlC4i8aFCD0A4ZNwwpICP1m9n5rLSoOOISIpQoQfkwsI8WjfJ1M2kRSRuVOgByUgLcd3gjsz9ZAuzV30RdBwRSQEq9ACN6ZdPTsMM7tUoXUTiQIUeoMz0MFcP6shbyz9n4bqtQccRkSSnQg/Y2AH5NM5M01y6iBw3FXrAGmWmc+XA9ry2eCPLNu4IOo6IJLFqFbqZjTCzpWa2wsxuPswxl5rZYjNbZGZPxTdmarvqtA40yAhzn0bpInIcjljoZhYGJgFnA92By8ys+yHHdAZ+DJzm7icBN8U/aupqmpXBFf3zmbpwA2u+2BV0HBFJUtUZofcDVrj7KnffBzwDjDrkmGuBSe6+BcDdN8U3Zuq79vSOpIVDPDBzZdBRRCRJVafQ2wDrKj0viW2rrAvQxczeMbP3zGxEvALWFS0aZ3JpUR7Pzyvh0217go4jIkkoXh+KpgGdgSHAZcCfzCz70IPM7DozKzaz4tJSXfJ+qHGDC4g4TJ61KugoIpKEqlPo64G2lZ7nxbZVVgJMdff97r4aWEa04P+Du0929yJ3L8rNzT3WzCmrbbMGXNC7DU/PWcvnO8uCjiMiSaY6hT4X6GxmHcwsAxgDTD3kmL8SHZ1jZjlEp2A0zDwG44cWUFYeYcrbq4OOIiJJ5oiF7u7lwERgGvAx8Ky7LzKzW8xsZOywacAXZrYYmA780N21QMkxKMhtyDk9WvH4u2vYtmd/0HFEJIlYUOtxFxUVeXFxcSDvnegWbdjGuX98m++f1YVvD/vSzJWI1GFmNs/di6rapytFE9BJrZtwZtcWTHlnNbvKyoOOIyJJQoWeoCYMLWDL7v08PWdt0FFEJEmo0BNUn3bNGNCxGZNnraKsvCLoOCKSBFToCWzi0M5s2lHG8/NKgo4iIklAhZ7ATuvUnJPbZvPAzJWUV0SCjiMiCU6FnsDMjIlDO7Fu8x6mLtwQdBwRSXAq9AQ3rGsLup7QiPtmrCQSCeYUUxFJDir0BBcKGeOHdmLFpp1MW/RZ0HFEJIGp0JPAuT1b0SEni3unryCoC8FEJPGp0JNAOGTccEYBizZsZ8YyrVIpIlVToSeJCwrb0LpJJpPe1ChdRKqmQk8SGWkhxp1RQPGaLcxevTnoOCKSgFToSWR037bkNMxgkm4mLSJVUKEnkcz0MFcP6shbyz9n4bqtQccRkQSjQk8yYwfk0zgzTaN0EfkSFXqSaZSZzpWndeC1xRtZ+tmOoOOISAJRoSehqwa2p0FGmPtmaJQuIv+mQk9CTbMyGDugHS8v3MAnn+8KOo6IJAgVepK6ZlAH0sIhHpi5MugoIpIgVOhJqkXjTEYXteWF+SVs2Lon6DgikgBU6Els3BkdcYfJs1YFHUVEEoAKPYnlNW3ABYVteGbuWj7fWRZ0HBEJmAo9yd0wpICy8ggPv7066CgiEjAVepIryG3IOT1b8fi7a9i2e3/QcUQkQCr0FDB+SAE7y8r587ufBB1FRAKkQk8BJ7VuwpldWzDlndXsKisPOo6IBESFniImDO3E1t37eXrO2qCjiEhAVOgpok+7ppzasTmTZ61i7/6KoOOISABU6Clk4pmd2LSjjOfnlQQdRUQCoEJPIQMLmtO7bTYPzFzJ/opI0HFEpJap0FOImTFxaCdKtuxh6oINQccRkVqmQk8xw7q1oOsJjbhvxgoiEd1MWqQuUaGnGDNjwtBOrCzdxauLPgs6jojUIhV6CjqnZys65mQxafoK3DVKF6krVOgpKBwyrh9SwKIN25mxtDToOCJSS1ToKerCwja0ya7PvRqli9QZKvQUlR4Ocd3gjsxbs4XZqzcHHUdEaoEKPYWN7tuWnIb1mDRdN5MWqQuqVehmNsLMlprZCjO7+SuOu8jM3MyK4hdRjlVmephrT+/AW8s/13rpInVA2pEOMLMwMAk4CygB5prZVHdffMhxjYAbgdk1EVSOzbcGdWDBuq386m+LcXeuOb1j0JFEpIZUZ4TeD1jh7qvcfR/wDDCqiuN+BfwW2BvHfHKc0sMh/nhZIef2bMWtf/+YybNWBh1JRGpIdQq9DbCu0vOS2LaDzOwUoK27//2rXsjMrjOzYjMrLi3V6XS1JT0c4u4xvTmvVyt+88oSHpipUhdJRUeccjkSMwsBdwJXHulYd58MTAYoKirSuXS1KC0c4q7RvTEzbvvHEiLujB/SKehYIhJH1Sn09UDbSs/zYtsOaAT0AGaYGcAJwFQzG+nuxfEKKscvLRziD5eeTMjg9leXEok4E8/sHHQsEYmT6hT6XKCzmXUgWuRjgMsP7HT3bUDOgedmNgP4gco8MaWFQ9x5aW9CZvzutWVEHL4zTKUukgqOWOjuXm5mE4FpQBiY4u6LzOwWoNjdp9Z0SImvcMj43SUnYwZ3vr6MiDs3De8SdCwROU7VmkN391eAVw7Z9vPDHDvk+GNJTQuHjDsuPpmQGXf9czkRh+8O70xs2kxEktBxfygqySscMm6/qBchgz++sRx353tndVGpiyQpFXodFwoZt32jFyEz7nlzBRF3fvC1E1XqIklIhS6EQsZvLuyJmTFp+koiDj/6ukpdJNmo0AWIlvqvL+hByOD+GSuJuHPziK4qdZEkokKXg0Ih49YLehAy48GZq3CHH5+tUhdJFip0+Q9mxi2jTiJkMHnWKioizk/P7aZSF0kCKnT5EjPjlyNPwsx4+O3VRNz5+XndVeoiCU6FLlUyM35xfndCZkx5ZzXu8IvzVeoiiUyFLodlZvzsvG6EDB6KjdT/JzZyF5HEo0KXr2Rm/Pe53QiFjMmzVhFx55aRPQiFVOoiiUaFLkdkZrGzXeDBmauIONw6SqUukmhU6FItZsbNI7oSMuP+GStxd359QU+VukgCUaFLtZkZP/r6iYTNuHf6CiIR+N9vqNRFEoUKXY6KmfH9r3WJLugVW/vltxf1UqmLJAAVuhw1M+N7sQW87n4juvTu7Rf3IqxSFwmUCl2O2XfP6oIZ3PXP6NK7d1xyskpdJEAqdDkuNw3vQsjs4J2Pfn9pb5W6SEBU6HLcvjOsc/QOSNOW4sDvLzmZtHAo6FgidY4KXeJiwtBOmMHtry4l4vCHS1XqIrVNhS5xM35IJ0Jm3PaPJUTcuWt0b9JV6iK1RoUucXX9GQWEDH7zyhLcnbvHFKrURWqJCl3i7rrBBYTMuPXvHxOJvM89l6vURWqD/pVJjbjm9I787LzuvLroMyY+NZ995ZGgI4mkPBW61JirB3Xgl+d3Z9qijUxQqYvUOBW61KgrT+vALaNO4vXFGxn/5DzKyiuCjiSSslToUuP+69T2/GrUSfzz403c8MR8lbpIDVGhS6345qntufWCHry5ZBPXPz6PvftV6iLxpkKXWjN2QDt+c2FPpi8tZZxKXSTudNqi1KrL++cTDsHNL37If02Zw8CC5qSHQ6SFjLRwiPSwkRY68Dy2LbYvLWykh2K/HjgubAe/Pz12TFoo9jqVtmt9GakLVOhS60b3zcfM+MVLi5izenOtvKcZB38YhENV/RD49+P66WG6tWpMYX42vds2pX3zBroxtiQFc/dA3rioqMiLi4sDeW9JHJGIsz8SobzCKa/49+P9FRHKI055RYT9FU555MDz2LZD9x3me/ZXOBWRr/qeL3//zr3lLNqwjV37olNC2Q3SOTkvO1bw0a/sBhkB/8lJXWVm89y9qKp9GqFLoEIho14oTL0E+5tYEXGWb9rBgrVbWbBuK++v3crdbyznwPinY05WtNzzsyls25SurRrpalgJnEboItW0s6ycD0qi5b5gXfSrdEcZAPXSQvRo04TCWMn3bptNm+z6mqqRuPuqEboKXeQYuTvrt+6Jlnus5D9cv42y2BWxOQ3rHZymKWybTa+22TRMtP+KSNLRlItIDTAz8po2IK9pA87r1RqA/RURlny6gwXrthwcyb++eGPseOjSotG/p2rys+ncopHOwJG40QhdpIZt3b3v4BTNga+tu/cDkJURpmdeEwrzmx4cybdonBlwYklkGqGLBCi7QQZDTmzBkBNbANGpmk++2M2CdVtYsHYr76/byp9mraI8Eh1ctW6SebDge+dn07NNEzLTw0H+FiRJqNBFapmZ0SEniw45WVxYmAfA3v0VLNqwnffXbjk4iv/7h58CkBYyurZqRFG7ZkwY2oncRvWCjC8JrFqFbmYjgLuBMPCQu992yP7vAdcA5UAp8C13XxPnrCIpKzM9TJ92TenTrunBbaU7ymLlHi35p+as5fXFG5lyZV9OPKFRgGklUR1xDt3MwsAy4CygBJgLXObuiysdMxSY7e67zewGYIi7j/6q19UcusjR+bBkG1f/eS6791Vwz+WFDI1N4Ujd8lVz6NW5EqIfsMLdV7n7PuAZYFTlA9x9urvvjj19D8g7nsAi8mU985rw0sTTaNe8AVc/OpdH31kddCRJMNUp9DbAukrPS2LbDudq4B9V7TCz68ys2MyKS0tLq59SRABo1aQ+z447lWHdWvLLlxfz85c+orxCd4KSqLheq2xmY4Ei4I6q9rv7ZHcvcvei3NzceL61SJ2RVS+NB8f2Ydzgjjz27hq+9editu/dH3QsSQDVKfT1QNtKz/Ni2/6DmQ0H/hsY6e5l8YknIlUJhYwfn9ON317Uk3+t+JyL7vsX6zbvPvI3SkqrTqHPBTqbWQczywDGAFMrH2BmhcCDRMt8U/xjikhVRvfN57Gr+7FpRxkXTHqHeWtqZzliSUxHLHR3LwcmAtOAj4Fn3X2Rmd1iZiNjh90BNASeM7MFZjb1MC8nInE2sCCHv4wfSKPMNC7702xeWvCl/0BLHaFL/0VSxJZd+7j+iXnMXr2Z7wzrzHeHd9ZqjynoeE9bFJEk0DQrg8ev7s8lffL44xvL+c4zC3Tf1jpGl/6LpJCMtBC3X9yLjrkN+e2rSyjZspvJ3yzScgF1hEboIinGzLhhSAEPjD2Fjz/dzgWT3mHpZzuCjiW1QIUukqJG9GjFc+MGsr8iwkX3/4vpS3UCWqpToYukMC0XULeo0EVSnJYLqDtU6CJ1QFa9NB7QcgEpT4UuUkeEY8sF3PYNLReQqlToInXMmH75PPatfmzcvlfLBaQYFbpIHTSwUw5/nXCalgtIMSp0kTqqY25D/jL+NArbZnPjMwu48/VlBLUUiMSHCl2kDjuwXMDFWi4gJejSf5E6LiMtxB0X96JAywUkPY3QRUTLBaQIFbqIHKTlApKbCl1E/sOB5QLym2m5gGSjQheRL2nVpD7PXX8qZ3bVcgHJRIUuIlXKqpfGg9/sw3VaLiBpqNBF5LDCIeMnWi4gaajQReSItFxAclChi0i1DOyUw1+0XEBCU6GLSLUVxJYL6K3lAhKSCl1EjkrTrAyeOGS5gM279gUdS9Cl/yJyDCovF3D7tCVM++gzzul5AmMHtKNPu6aYWdAR6yQVuogckwPLBQzr1oIn31vDi/PX89cFG+h6QiOu6J/PBYVtaJSZHnTMOsWCmv8qKiry4uLiQN5bROJvV1k5Uxdu4In31rBow3ayMsKMKmzD2P7t6N66cdDxUoaZzXP3oir3qdBFJJ7cnYUl23jivTW8vHADZeURCvOzGdu/Hef2akVmejjoiElNhS4igdi6ex8vzF/Pk7PXsKp0F9kN0rn4lDyuGNCODjlZQcdLSip0EQmUu/Puyi94cvZapi36jPKIc1qn5ozt347h3VuSHtYJd9WlQheRhLFp+16eLV7H03PWsX7rHlo0qseYvm0Z0y+f1tn1g46X8FToIpJwKiLO9CWbeHL2GmYsK8WAYd1ackX/fAZ3ziUU0qmPVfmqQtdpiyISiHDIGN69JcO7t2Td5t08NWctz85dx+uLN5LfrAGX98/nkj55NG+oW+FVl0boIpIwysormLZoI0+8t4Y5qzeTEQ5xduyCpSJdsARoykVEktCyjTt4avZaXphXwo6yck5s2YgrBuRzYR2/YEmFLiJJa/e+cl5euIEn3lvLh+u30SAjzKjerbmifzt6tGkSdLxap0IXkZSwcN1Wnpy9hqkLN7B3f4TebbO5on8+55/cus5csKRCF5GUsm33fl6YX8KTs9ewsnQXTeqnc3GfPC7vn09BbsOg49UoFbqIpCR3571Vm3li9hqmfRS9YGlgQXPGDmjHWSl6wdJxn7ZoZiOAu4Ew8JC733bI/nrAY0Af4AtgtLt/cjyhRUSOxMw4taA5pxY0Z9OOvTxXXMJTs9cy/sn55DTMoF3zLLLqpdGwXpiG9dJij9MO/zgzjazYsfXTw0l3Vs0RR+hmFgaWAWcBJcBc4DJ3X1zpmPFAL3e/3szGABe6++ivel2N0EWkJlREnJnLNvHX9zfwxa4ydpZVsHPvfnaVVbCrrJyd+8qpzsREyKiy9KOFn07DeuHotszY9oxKj//j+8JkZaTF7UKp4x2h9wNWuPuq2Is9A4wCFlc6ZhTwy9jj54F7zcxc96YSkVoWDhlndm3JmV1bVrk/EnH27I+W+46y8mjJl5Wzc285u/aVs/NA8e+Nbj+4P/Z404697CqrOLitIlK9msvK+PcPgJuGd2Hkya3j+dsGqlfobYB1lZ6XAP0Pd4y7l5vZNqA58Hnlg8zsOuA6gPz8/GOMLCJy7EIhIys2im5xnK/l7pSVRw6W/Y690V937TvwuOJLPxB2lpXTtEHNnEdfq5f+u/tkYDJEp1xq871FROLNzMhMD5OZHiYnAZYoqM5HwOuBtpWe58W2VXmMmaUBTYh+OCoiIrWkOoU+F+hsZh3MLAMYA0w95JipwP+JPb4YeFPz5yIiteuIUy6xOfGJwDSipy1OcfdFZnYLUOzuU4GHgcfNbAWwmWjpi4hILarWHLq7vwK8csi2n1d6vBe4JL7RRETkaKTeZVQiInWUCl1EJEWo0EVEUoQKXUQkRQS22qKZlQJrjvHbczjkKtQEoVxHR7mOXqJmU66jczy52rl7blU7Aiv042FmxYdbnCZIynV0lOvoJWo25To6NZVLUy4iIilChS4ikiKStdAnBx3gMJTr6CjX0UvUbMp1dGokV1LOoYuIyJcl6whdREQOoUIXEUkRSVfoZjbCzJaa2QozuznoPABmNsXMNpnZR0FnqczM2prZdDNbbGaLzOzGoDMBmFmmmc0xs4WxXP8TdKbKzCxsZu+b2d+CznKAmX1iZh+a2QIzS5ib8ZpZtpk9b2ZLzOxjMzs1ATKdGPtzOvC13cxuCjoXgJl9N/Z3/iMze9rMMuP6+sk0h16dG1YHlGswsBN4zN17BJmlMjNrBbRy9/lm1giYB1yQAH9eBmS5+04zSwfeBm509/eCzHWAmX0PKAIau/t5QeeBaKEDRe6eUBfJmNmfgbfc/aHY/RIauPvWgGMdFOuM9UB/dz/WCxnjlaUN0b/r3d19j5k9C7zi7o/G6z2SbYR+8IbV7r4POHDD6kC5+yyi68AnFHf/1N3nxx7vAD4mev/XQHnUztjT9NhXQowszCwPOBd4KOgsic7MmgCDid4PAXffl0hlHjMMWBl0mVeSBtSP3dmtAbAhni+ebIVe1Q2rAy+oZGBm7YFCYHbAUYCD0xoLgE3A6+6eELmAu4AfAZGAcxzKgdfMbF7sZuuJoANQCjwSm6J6yMyygg51iDHA00GHAHD39cDvgLXAp8A2d38tnu+RbIUux8DMGgIvADe5+/ag8wC4e4W79yZ6j9p+Zhb4VJWZnQdscvd5QWepwiB3PwU4G5gQm+YLWhpwCnC/uxcCu4CE+FwLIDYFNBJ4LugsAGbWlOiMQgegNZBlZmPj+R7JVujVuWG1VBKbo34BeNLdXww6z6Fi/0WfDowIOArAacDI2Hz1M8CZZvZEsJGiYqM73H0T8Bei049BKwFKKv3v6nmiBZ8ozgbmu/vGoIPEDAdWu3upu+8HXgQGxvMNkq3Qq3PDaomJffj4MPCxu98ZdJ4DzCzXzLJjj+sT/ZB7SaChAHf/sbvnuXt7on+33nT3uI6gjoWZZcU+1CY2pfE1IPAzqtz9M2CdmZ0Y2zQMCPQD90NcRoJMt8SsBQaYWYPYv81hRD/Xiptq3VM0URzuhtUBx8LMngaGADlmVgL8wt0fDjYVEB1xfhP4MDZfDfCT2D1ig9QK+HPsDIQQ8Ky7J8wpggmoJfCXaAeQBjzl7q8GG+mgbwNPxgZYq4CrAs4DHPzBdxYwLugsB7j7bDN7HpgPlAPvE+clAJLqtEURETm8ZJtyERGRw1Chi4ikCBW6iEiKUKGLiKQIFbqISIpQoYuIpAgVuohIivj/KR1KREmtfbgAAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -354,8 +936,7 @@ } ], "source": [ - "def create() -> pd.DataFrame:\n", - " return pd.DataFrame([[100-x] for x in range(100)], columns=[\"a\"])\n", + "df = pd.DataFrame([[100-x] for x in range(100)], columns=[\"a\"])\n", "\n", "def plot(df:pd.DataFrame, p:callable) -> None:\n", " random.seed(0)\n", @@ -363,8 +944,7 @@ " p(random.random()*v)\n", " sleep(0.2)\n", "\n", - "with FugueWorkflow() as dag:\n", - " dag.create(create).out_transform(plot, callback=PlotMinNow())" + "fa.out_transform(df, plot, callback=PlotMinNow())" ] }, { @@ -401,13 +981,6 @@ "| Optional | Yes | No | **Local only**, you can use it to test your callbacks using `NativeExecutionEngine` only. If you use a distributed engine, serialization exception will be thrown\n", "| Optional | Yes | Yes | **Common**, a typical way to use callbacks both locally and distributedly" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -426,7 +999,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.9" + "version": "3.8.13" }, "vscode": { "interpreter": { From 6a7ba4d914b6bf0d5f309994c3333efea6d2c4e4 Mon Sep 17 00:00:00 2001 From: kvnkho Date: Fri, 17 Nov 2023 19:17:14 -0600 Subject: [PATCH 3/3] Cleaning checkpoint tutorial to remove FugueWorkflow --- tutorials/advanced/checkpoint.ipynb | 284 ++++++++++---- tutorials/advanced/index.md | 6 +- .../recipes/loading_databases.ipynb | 141 ------- .../recipes/loading_text_files.ipynb | 360 ------------------ 4 files changed, 205 insertions(+), 586 deletions(-) delete mode 100644 tutorials/applications/recipes/loading_databases.ipynb delete mode 100644 tutorials/applications/recipes/loading_text_files.ipynb diff --git a/tutorials/advanced/checkpoint.ipynb b/tutorials/advanced/checkpoint.ipynb index a0495eb7..8779bc0f 100644 --- a/tutorials/advanced/checkpoint.ipynb +++ b/tutorials/advanced/checkpoint.ipynb @@ -52,12 +52,12 @@ "\n", "Lazy | Filesystem | Permanent | Deterministic | Explicit | Spark Equivalent | Fugue Equivalent\n", ":---:|:---:|:---:|:---:|:---:|:---|:---\n", - "Yes | No | No | No | No | `persist()` | `weak_checkpoint(lazy=True)`\n", - "No | No | No | No | No | | `persist()` or `weak_checkpoint(lazy=False)`\n", - "Yes | Yes | No | No | No | | `strong_checkpoint(lazy=True)`\n", + "Yes | No | No | No | No | `persist()` | `persist(lazy=True)`\n", + "No | No | No | No | No | | `persist()` or `persist(lazy=False)`\n", + "Yes | Yes | No | No | No | | `transform(..., save_path=None, checkpoint=True)`\n", "No | Yes | No | No | No | | `checkpoint()` or `strong_checkpoint(lazy=False)`\n", "Yes | Yes | Yes | No | No | | \n", - "No | Yes | Yes | No | No | `checkpoint` | `yield_as`\n", + "No | Yes | Yes | No | No | `checkpoint()` | `yield_as`\n", "Yes | Yes | Yes | Yes | No | | `deteriministic_checkpoint(lazy=True)`\n", "No | Yes | Yes | Yes | No | | `deteriministic_checkpoint()`\n", "Yes | Yes | Yes | Yes | Yes | | \n", @@ -70,26 +70,26 @@ "\n", "### Weak Checkpoint\n", "\n", - "Weak checkpoint is in memory checkpoint. We don't use 'in memory' because it may also use executor's local disk space, depending on the execution engine we use. `persist()` is an alias of `weak_checkpoint(lazy=False)`. \n", + "Weak checkpoint is an in memory checkpoint. We don't use 'in memory' because it may also use executor's local disk space, depending on the execution engine we use. To invoke this, use `persist(lazy=False)`. \n", "\n", - "Note Fugue persist is eager but Spark persist is lazy." + "Note Fugue persist is eager by default but Spark persist is lazy." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ - "from fugue import FugueWorkflow\n", - "from fugue_spark import SparkExecutionEngine\n", "from time import sleep\n", "import pandas as pd\n", + "import fugue.api as fa\n", "\n", - "# schema: *\n", "def just_wait(df:pd.DataFrame) -> pd.DataFrame:\n", " sleep(5)\n", - " return df" + " return df\n", + "\n", + "df = pd.DataFrame({\"a\": [1,2,3], \"b\": [1,2,3]})" ] }, { @@ -101,37 +101,141 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "%%time\n", - "with FugueWorkflow(SparkExecutionEngine()) as dag:\n", - " df = dag.df([[0]],\"a:int\")\n", - " df = df.transform(just_wait)\n", - " df.show()\n", - " df.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the following code, the two shows, due to Spark lineage, will duplicae the transform step, and will take 10 sec" + "from pyspark.sql import SparkSession\n", + "\n", + "spark = SparkSession.builder.getOrCreate()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
a:longb:long
011
122
233
\n", + "
\n", + "SparkDataFrame: a:long,b:long" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
a:longb:long
011
122
233
\n", + "
\n", + "SparkDataFrame: a:long,b:long" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 49.5 ms, sys: 13.7 ms, total: 63.2 ms\n", + "Wall time: 10.6 s\n" + ] + } + ], "source": [ "%%time\n", - "with FugueWorkflow(SparkExecutionEngine()) as dag:\n", - " df = dag.df([[0]],\"a:int\")\n", - " df = df.transform(just_wait).persist()\n", - " df.show()\n", - " df.show()" + "with fa.engine_context(spark):\n", + " res = fa.transform(df, just_wait, schema=\"*\")\n", + " fa.show(res)\n", + " fa.show(res)" ] }, { @@ -143,35 +247,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 26.4 ms, sys: 6.85 ms, total: 33.2 ms\n", + "Wall time: 83.9 ms\n" + ] + } + ], "source": [ "%%time\n", - "with FugueWorkflow(SparkExecutionEngine()) as dag:\n", - " df = dag.df([[0]],\"a:int\")\n", - " df = df.transform(just_wait).weak_checkpoint(lazy=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following takes 5 sec because the first show triggers the persist and the second uses the cache. But it is pointless to use a lazy checkpoint in a predefined dag like this." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "with FugueWorkflow(SparkExecutionEngine()) as dag:\n", - " df = dag.df([[0]],\"a:int\")\n", - " df = df.transform(just_wait).weak_checkpoint(lazy=True)\n", - " df.show()\n", - " df.show()" + "with fa.engine_context(spark):\n", + " res = fa.transform(df, just_wait, schema=\"*\")\n", + " fa.persist(res, lazy=True)" ] }, { @@ -180,7 +272,7 @@ "source": [ "### Strong Checkpoint\n", "\n", - "Strong checkpoints are in file but non-permanent checkpoints. They are removed after the DAG execution done.\n", + "Strong checkpoints are in file but non-permanent checkpoints. They are removed after the execution is completed.\n", "\n", "Note Fugue checkpoint is non-permanent but Spark checkpoint is permanent.\n", "\n", @@ -189,32 +281,40 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "engine = SparkExecutionEngine(conf={\"fugue.workflow.checkpoint.path\":\"/tmp\"})\n", - "with FugueWorkflow(engine) as dag:\n", - " df = dag.df([[0]],\"a:int\")\n", - " df = df.transform(just_wait).checkpoint()\n", - " df.show()\n", - " df.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---+---+\n", + "| a| b|\n", + "+---+---+\n", + "| 3| 3|\n", + "| 1| 1|\n", + "| 2| 2|\n", + "+---+---+\n", + "\n", + "+---+---+\n", + "| a| b|\n", + "+---+---+\n", + "| 3| 3|\n", + "| 1| 1|\n", + "| 2| 2|\n", + "+---+---+\n", + "\n", + "CPU times: user 36.3 ms, sys: 9.8 ms, total: 46.1 ms\n", + "Wall time: 5.57 s\n" + ] + } + ], "source": [ "%%time\n", - "engine = SparkExecutionEngine(conf={\"fugue.workflow.checkpoint.path\":\"/tmp\"})\n", - "with FugueWorkflow(engine) as dag:\n", - " df = dag.df([[0]],\"a:int\")\n", - " df = df.transform(just_wait).strong_checkpoint(lazy=True)\n", - " df.show()\n", - " df.show()" + "with fa.engine_context(spark, engine_conf={\"fugue.workflow.checkpoint.path\":\"/tmp\"}):\n", + " res = fa.transform(df, just_wait, schema=\"*\", checkpoint=True)\n", + " res.show()\n", + " res.show()" ] }, { @@ -223,7 +323,7 @@ "source": [ "### Deterministic Checkpoint\n", "\n", - "Deterministic checkpoint is mainly for 'resuming'. Expecially when in an interactive environment, you may not want to rerun the whole DAG every time you make a minor change. But note that, practically, you should avoid using deterministic checkpoint in production, also whenever you use deterministic checkpoint, ask yourself, is the dag overly complicated? Can you make it more modulized?\n", + "Deterministic checkpoint is mainly for 'resuming'. Expecially when in an interactive environment, you may not want to rerun the whole DAG every time you make a minor change. But note that, practically, you should avoid using deterministic checkpoint in production, also whenever you use deterministic checkpoint, ask yourself, is the dag overly complicated? Can you make it more modularized?\n", "\n", "See this example" ] @@ -319,6 +419,22 @@ "It is common to see you have a good separation of your logic, and they are divided into several DAGs, and you need to pass the dataframes between the DAGs. This can be done by `save` and `load` an explicit file path, but for intermediate data, you don't always want to specify paths for them, also you want determinism to work cross DAGs. This is reason to use `yield_as`" ] }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "query = \"\"\"\n", + "TRANSFORM df USING just_wait SCHEMA *\n", + "DETERMINISTIC CHECKPOINT\n", + "YIELD DATAFRAME AS result\n", + "\"\"\"\n", + "\n", + "res = fa.fugue_sql_flow(query).run(engine=spark, conf={\"fugue.workflow.checkpoint.path\":\"/tmp\"})\n", + "res['result']" + ] + }, { "cell_type": "code", "execution_count": null, @@ -383,7 +499,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.9" + "version": "3.8.13" }, "vscode": { "interpreter": { diff --git a/tutorials/advanced/index.md b/tutorials/advanced/index.md index 469babdd..dc1f128d 100644 --- a/tutorials/advanced/index.md +++ b/tutorials/advanced/index.md @@ -12,6 +12,7 @@ Since you already have experience in Spark or distributed computing in general, :hidden: useful_config +checkpoint execution_engine validation schema_dataframes @@ -30,8 +31,11 @@ These configurations can have significant impact on building and running the Fug ## [Execution Engine](execution_engine.ipynb) The heart of Fugue. It is the layer that unifies many of the core concepts of distributed computing, and separates the underlying computing frameworks from user level logic. Normally you don't directly interact with execution engines. But it's good to understand some basics. +## [Checkpoint](checkpoint.ipynb) +These are the operations to avoid duplicate computation when working with larger datasets. Fugue has more granular checkpoint operations than Spark. + ## [Validation](validation.ipynb) -Fugue applies input validation. +Fugue applies input validation to make sure the data has the expected format before a transformation is applied. ## [Data Type, Schema & DataFrames](schema_dataframes.ipynb) Fugue data types and schema are strictly based on [Apache Arrow](https://arrow.apache.org/docs/index.html). Dataframe is an abstract concept with several built-in implementations to adapt to different dataframes. In this tutorial, we will go through the basic APIs and focus on the most common use cases. diff --git a/tutorials/applications/recipes/loading_databases.ipynb b/tutorials/applications/recipes/loading_databases.ipynb deleted file mode 100644 index 02f872b7..00000000 --- a/tutorials/applications/recipes/loading_databases.ipynb +++ /dev/null @@ -1,141 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Loading database tables\n", - "\n", - "We can either use a custom `Creator` load from a database to a `DataFrame` or we can use the database engine service itself to output a file which can then be loaded in `fugue`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's dump some data into a `sqlite` database & read it in `fugue`" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import sqlite3\n", - "\n", - "from fugue import FugueWorkflow\n", - "import pandas as pd" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "def create_sqlite_db(path: str, content: str, table: str):\n", - " uri = f\"file:{path}\"\n", - " lines = content.split(\"\\n\")\n", - " headers = lines[0]\n", - " rows = lines[1:]\n", - " with sqlite3.connect(uri, uri=True) as con:\n", - " cur = con.cursor()\n", - " cur.execute(f\"CREATE TABLE {table}({headers})\")\n", - " values = \"(\" + '),('.join(row for row in rows) + \")\"\n", - " cur.execute(f\"INSERT INTO {table} VALUES {values}\")\n", - " con.commit()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "content = '''\\\n", - "\"TIMESTAMP\",\"RECORD\",\"WS_80m_90deg_Avg\",\"WS_80m_90deg_Std\",\"WS_80m_90deg_3sGust_Max\",\"WS_80m_90deg_Max\",\"WS_80m_270deg_Avg\",\"WS_80m_270deg_Std\",\"WS_80m_270deg_3sGust_Max\",\"WS_80m_270deg_Max\",\"WS_65m_90deg_Avg\",\"WS_65m_90deg_Std\",\"WS_65m_90deg_3sGust_Max\",\"WS_65m_90deg_Max\",\"WS_65m_270deg_Avg\",\"WS_65m_270deg_Std\",\"WS_65m_270deg_3sGust_Max\",\"WS_65m_270deg_Max\",\"WS_50m_90deg_Avg\",\"WS_50m_90deg_Std\",\"WS_50m_90deg_3sGust_Max\",\"WS_50m_90deg_Max\",\"WS_50m_270deg_Avg\",\"WS_50m_270deg_Std\",\"WS_50m_270deg_3sGust_Max\",\"WS_50m_270deg_Max\",\"WS_30m_90deg_Avg\",\"WS_30m_90deg_Std\",\"WS_30m_90deg_3sGust_Max\",\"WS_30m_90deg_Max\",\"WS_30m_270deg_Avg\",\"WS_30m_270deg_Std\",\"WS_30m_270deg_3sGust_Max\",\"WS_30m_270deg_Max\",\"Dir_78m_90deg_avg\",\"Dir_78m_90deg_std\",\"Dir_63m_90deg_avg\",\"Dir_63m_90deg_std\",\"Dir_28m_90deg_avg\",\"Dir_28m_90deg_std\",\"Batt_Volt_Min\",\"Press_Avg\",\"Temp_C80_Avg\",\"Temp_C15_Avg\",\"Hum_Avg\"\n", - "\"2012-05-31 12:20:00\",1,1.383,0.6,2.75,3.37,1.368,0.439,2.673,2.74,1.332,0.478,2.75,2.75,1.242,0.379,2.74,2.79,1.162,0.535,2.337,2.75,1.159,0.354,2.34,2.39,1.27,0.614,2.337,2.75,1.322,0.416,2.157,2.24,240.3,46,242,45.39,222,33.45,13.79,1009,13.84,14.08,65.67\n", - "\"2012-05-31 12:30:00\",2,1.183,0.449,1.923,2.13,1.135,0.324,1.94,1.99,0.948,0.524,1.923,2.13,1.068,0.303,1.723,1.74,0.701,0.547,1.923,2.13,0.913,0.308,1.673,1.74,0.771,0.539,1.717,2.13,0.997,0.28,1.657,1.74,282,26.79,264.3,30.25,278.5,62.87,13.73,1009,14.04,14.45,64.51'''" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def load_db(table: str, con: str) -> pd.DataFrame:\n", - " return pd.read_sql_table(table, uri)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "PandasDataFrame\n", - "TIMESTAMP:str |RECORD:long|WS_80m_90deg_Avg:double|WS_80m_90deg_Std:double|WS_80m_90deg_3sGust_Max:double|WS_80m_90deg_Max:double|WS_80m_270deg_Avg:double|WS_80m_270deg_Std:double|WS_80m_270deg_3sGust_Max:double|WS_80m_270deg_Max:double|WS_65m_90deg_Avg:double|WS_65m_90deg_Std:double|WS_65m_90deg_3sGust_Max:double|WS_65m_90deg_Max:double|WS_65m_270deg_Avg:double|WS_65m_270deg_Std:double|WS_65m_270deg_3sGust_Max:double|WS_65m_270deg_Max:double|WS_50m_90deg_Avg:double|WS_50m_90deg_Std:double|WS_50m_90deg_3sGust_Max:double|WS_50m_90deg_Max:double|WS_50m_270deg_Avg:double|WS_50m_270deg_Std:double|WS_50m_270deg_3sGust_Max:double|WS_50m_270deg_Max:double|WS_30m_90deg_Avg:double|WS_30m_90deg_Std:double|WS_30m_90deg_3sGust_Max:double|WS_30m_90deg_Max:double|WS_30m_270deg_Avg:double|WS_30m_270deg_Std:double|WS_30m_270deg_3sGust_Max:double|WS_30m_270deg_Max:double|Dir_78m_90deg_avg:double|Dir_78m_90deg_std:double|Dir_63m_90deg_avg:double|Dir_63m_90deg_std:double|Dir_28m_90deg_avg:double|Dir_28m_90deg_std:double|Batt_Volt_Min:double|Press_Avg:long|Temp_C80_Avg:double|Temp_C15_Avg:double|Hum_Avg:double\n", - "--------------+-----------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+--------------------+--------------+-------------------+-------------------+--------------\n", - "2012-05-31 12:|1 |1.383 |0.6 |2.75 |3.37 |1.368 |0.439 |2.673 |2.74 |1.332 |0.478 |2.75 |2.75 |1.242 |0.379 |2.74 |2.79 |1.162 |0.535 |2.337 |2.75 |1.159 |0.354 |2.34 |2.39 |1.27 |0.614 |2.337 |2.75 |1.322 |0.416 |2.157 |2.24 |240.3 |46.0 |242.0 |45.39 |222.0 |33.45 |13.79 |1009 |13.84 |14.08 |65.67 \n", - "20:00 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | \n", - "2012-05-31 12:|2 |1.183 |0.449 |1.923 |2.13 |1.135 |0.324 |1.94 |1.99 |0.948 |0.524 |1.923 |2.13 |1.068 |0.303 |1.723 |1.74 |0.701 |0.547 |1.923 |2.13 |0.913 |0.308 |1.673 |1.74 |0.771 |0.539 |1.717 |2.13 |0.997 |0.28 |1.657 |1.74 |282.0 |26.79 |264.3 |30.25 |278.5 |62.87 |13.73 |1009 |14.04 |14.45 |64.51 \n", - "30:00 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | \n", - "Total count: 2\n", - "\n" - ] - } - ], - "source": [ - "db = \"/tmp/example.db\"\n", - "uri = f\"sqlite:///{db}\"\n", - "create_sqlite_db(db, content, table=\"sensors\")\n", - "\n", - "with FugueWorkflow() as dag:\n", - " df = dag.create(load_db, params={\"table\": \"sensors\", \"con\": uri})\n", - " df.show()\n", - "\n", - "os.unlink(db)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.10.6 ('fugue')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "787dcf4b06485f091e5ea0894a1563cc39da567670ce0a93adb376c1b3122bd1" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tutorials/applications/recipes/loading_text_files.ipynb b/tutorials/applications/recipes/loading_text_files.ipynb deleted file mode 100644 index f4b0a8a0..00000000 --- a/tutorials/applications/recipes/loading_text_files.ipynb +++ /dev/null @@ -1,360 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Loading text files\n", - "\n", - "`fugue` can read text files natively via `load` or by dropping into an execution engine\n", - "\n", - "You might find it useful to use the execution engine directly for loading non-standard files or files that are not natively supported by `fugue`. \n", - "\n", - "We'll demonstrate `pandas`, `duckdb` & `dask` here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here's an example where the headers are on the 2nd line & the data starts on the 5th:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\"SITENAME\"\n", - "\"TIMESTAMP\",\"RECORD\",\"WS_80m_90deg_Avg\",\"WS_80m_90deg_Std\",\"WS_80m_90deg_3sGust_Max\",\"WS_80m_90deg_Max\",\"WS_80m_270deg_Avg\",\"WS_80m_270deg_Std\",\"WS_80m_270deg_3sGust_Max\",\"WS_80m_270deg_Max\",\"WS_65m_90deg_Avg\",\"WS_65m_90deg_Std\",\"WS_65m_90deg_3sGust_Max\",\"WS_65m_90deg_Max\",\"WS_65m_270deg_Avg\",\"WS_65m_270deg_Std\",\"WS_65m_270deg_3sGust_Max\",\"WS_65m_270deg_Max\",\"WS_50m_90deg_Avg\",\"WS_50m_90deg_Std\",\"WS_50m_90deg_3sGust_Max\",\"WS_50m_90deg_Max\",\"WS_50m_270deg_Avg\",\"WS_50m_270deg_Std\",\"WS_50m_270deg_3sGust_Max\",\"WS_50m_270deg_Max\",\"WS_30m_90deg_Avg\",\"WS_30m_90deg_Std\",\"WS_30m_90deg_3sGust_Max\",\"WS_30m_90deg_Max\",\"WS_30m_270deg_Avg\",\"WS_30m_270deg_Std\",\"WS_30m_270deg_3sGust_Max\",\"WS_30m_270deg_Max\",\"Dir_78m_90deg_avg\",\"Dir_78m_90deg_std\",\"Dir_63m_90deg_avg\",\"Dir_63m_90deg_std\",\"Dir_28m_90deg_avg\",\"Dir_28m_90deg_std\",\"Batt_Volt_Min\",\"Press_Avg\",\"Temp_C80_Avg\",\"Temp_C15_Avg\",\"Hum_Avg\"\n", - "\"TS\",\"RN\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"meters/second\",\"meters/second\",\"meters/second\",\"meters/second\",\"Volts\",\"mB\",\"DegC\",\"DegC\",\"%\"\n", - "\"\",\"\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"WVc\",\"WVc\",\"WVc\",\"WVc\",\"WVc\",\"WVc\",\"Min\",\"Avg\",\"Avg\",\"Avg\",\"Avg\"\n", - "\"2012-05-31 12:20:00\",1,1.383,0.6,2.75,3.37,1.368,0.439,2.673,2.74,1.332,0.478,2.75,2.75,1.242,0.379,2.74,2.79,1.162,0.535,2.337,2.75,1.159,0.354,2.34,2.39,1.27,0.614,2.337,2.75,1.322,0.416,2.157,2.24,240.3,46,242,45.39,222,33.45,13.79,1009,13.84,14.08,65.67\n", - "\"2012-05-31 12:30:00\",2,1.183,0.449,1.923,2.13,1.135,0.324,1.94,1.99,0.948,0.524,1.923,2.13,1.068,0.303,1.723,1.74,0.701,0.547,1.923,2.13,0.913,0.308,1.673,1.74,0.771,0.539,1.717,2.13,0.997,0.28,1.657,1.74,282,26.79,264.3,30.25,278.5,62.87,13.73,1009,14.04,14.45,64.51" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To read it we'll need to create a custom `Creator` in `fugue`!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import typing\n", - "\n", - "import duckdb\n", - "from fugue import DataFrame\n", - "from fugue import ExecutionEngine\n", - "from fugue import FugueWorkflow\n", - "from fugue import NativeExecutionEngine\n", - "from fugue_dask import DaskExecutionEngine\n", - "from fugue_duckdb import DuckExecutionEngine\n", - "import pandas as pd" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Standard text files" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First let's create a sample `standard` text file ..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "content = \"\"\"\\\n", - "a,b,c\n", - "1,2,3\n", - "1,2,3\"\"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can read it natively" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "PandasDataFrame\n", - "a:str|b:str|c:str\n", - "-----+-----+-----\n", - "1 |2 |3 \n", - "1 |2 |3 \n", - "Total count: 2\n", - "\n" - ] - } - ], - "source": [ - "file = \"/tmp/fugue_example_std_1.csv\"\n", - "with open(file, \"w\") as f:\n", - " f.write(content)\n", - "\n", - "with FugueWorkflow() as dag:\n", - " df = dag.load(file, header=True)\n", - " df.show()\n", - "\n", - "os.unlink(file)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can read multiple files using a wildcard `*` " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "PandasDataFrame\n", - "a:str|b:str|c:str\n", - "-----+-----+-----\n", - "1 |2 |3 \n", - "1 |2 |3 \n", - "1 |2 |3 \n", - "1 |2 |3 \n", - "Total count: 4\n", - "\n" - ] - } - ], - "source": [ - "file_1 = \"/tmp/fugue_example_std_2.csv\"\n", - "file_2 = \"/tmp/fugue_example_std_3.csv\"\n", - "with open(file_1, \"w\") as f1, open(file_2, \"w\") as f2:\n", - " f1.write(content)\n", - " f2.write(content)\n", - "wildcard = \"/tmp/fugue_example_std_*.csv\"\n", - "\n", - "with FugueWorkflow() as dag:\n", - " df = dag.load(wildcard, header=True)\n", - " df.show()\n", - "\n", - "os.unlink(file_1)\n", - "os.unlink(file_2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Non-standard text files\n", - "\n", - "Or, if your input file is non-standard, we can use the execution engine directly " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "content = '''\\\n", - "\"SITENAME\"\n", - "\"TIMESTAMP\",\"RECORD\",\"WS_80m_90deg_Avg\",\"WS_80m_90deg_Std\",\"WS_80m_90deg_3sGust_Max\",\"WS_80m_90deg_Max\",\"WS_80m_270deg_Avg\",\"WS_80m_270deg_Std\",\"WS_80m_270deg_3sGust_Max\",\"WS_80m_270deg_Max\",\"WS_65m_90deg_Avg\",\"WS_65m_90deg_Std\",\"WS_65m_90deg_3sGust_Max\",\"WS_65m_90deg_Max\",\"WS_65m_270deg_Avg\",\"WS_65m_270deg_Std\",\"WS_65m_270deg_3sGust_Max\",\"WS_65m_270deg_Max\",\"WS_50m_90deg_Avg\",\"WS_50m_90deg_Std\",\"WS_50m_90deg_3sGust_Max\",\"WS_50m_90deg_Max\",\"WS_50m_270deg_Avg\",\"WS_50m_270deg_Std\",\"WS_50m_270deg_3sGust_Max\",\"WS_50m_270deg_Max\",\"WS_30m_90deg_Avg\",\"WS_30m_90deg_Std\",\"WS_30m_90deg_3sGust_Max\",\"WS_30m_90deg_Max\",\"WS_30m_270deg_Avg\",\"WS_30m_270deg_Std\",\"WS_30m_270deg_3sGust_Max\",\"WS_30m_270deg_Max\",\"Dir_78m_90deg_avg\",\"Dir_78m_90deg_std\",\"Dir_63m_90deg_avg\",\"Dir_63m_90deg_std\",\"Dir_28m_90deg_avg\",\"Dir_28m_90deg_std\",\"Batt_Volt_Min\",\"Press_Avg\",\"Temp_C80_Avg\",\"Temp_C15_Avg\",\"Hum_Avg\"\n", - "\"TS\",\"RN\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"\",\"meters/second\",\"meters/second\",\"meters/second\",\"meters/second\",\"meters/second\",\"meters/second\",\"meters/second\",\"Volts\",\"mB\",\"DegC\",\"DegC\",\"%\"\n", - "\"\",\"\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"Avg\",\"Std\",\"Max\",\"Max\",\"WVc\",\"WVc\",\"WVc\",\"WVc\",\"WVc\",\"WVc\",\"Min\",\"Avg\",\"Avg\",\"Avg\",\"Avg\"\n", - "\"2012-05-31 12:20:00\",1,1.383,0.6,2.75,3.37,1.368,0.439,2.673,2.74,1.332,0.478,2.75,2.75,1.242,0.379,2.74,2.79,1.162,0.535,2.337,2.75,1.159,0.354,2.34,2.39,1.27,0.614,2.337,2.75,1.322,0.416,2.157,2.24,240.3,46,242,45.39,222,33.45,13.79,1009,13.84,14.08,65.67\n", - "\"2012-05-31 12:30:00\",2,1.183,0.449,1.923,2.13,1.135,0.324,1.94,1.99,0.948,0.524,1.923,2.13,1.068,0.303,1.723,1.74,0.701,0.547,1.923,2.13,0.913,0.308,1.673,1.74,0.771,0.539,1.717,2.13,0.997,0.28,1.657,1.74,282,26.79,264.3,30.25,278.5,62.87,13.73,1009,14.04,14.45,64.51\n", - "'''" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's read the headers on the 2nd line separately to loading the text file " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def read_header(filepath: str) -> typing.List[str]:\n", - " row_1 = pd.read_csv(filepath, skiprows=1, nrows=0).columns\n", - " header = [row_1[0].replace(\"columns: \", \"\"), *row_1[1:]]\n", - " return header" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "& specify these headers as the column names of the data that we are loading in\n", - "\n", - "> **Note:** `skip` & `columns` for `DuckExecutionEngine` correspond to `skiprows` & `names` for `pandas.read_csv` as `duckdb` `csv` has different conventions " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def read_text_file(engine: ExecutionEngine, filepath: str) -> DataFrame:\n", - " headers = read_header(filepath)\n", - " if isinstance(engine, NativeExecutionEngine):\n", - " # load_df uses pandas.read_csv\n", - " df = engine.load_df(filepath, infer_schema=True, header=True, skiprows=3, names=headers)\n", - " elif isinstance(engine, DuckExecutionEngine):\n", - " # load_df uses duckdb read_csv_auto\n", - " df = engine.load_df(filepath, infer_schema=True, skip=4, columns=headers)\n", - " elif isinstance(engine, DaskExecutionEngine):\n", - " # load_df uses dask.dataframe.read_csv\n", - " df = engine.load_df(filepath, infer_schema=True, header=True, skiprows=3, names=headers)\n", - " else:\n", - " supported_engines = {NativeExecutionEngine, DuckExecutionEngine, DaskExecutionEngine} \n", - " raise ValueError(f\"Engine {engine} is not supported, must be one of {supported_engines}\")\n", - " return df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%html\n", - "\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "PandasDataFrame\n", - "TIMESTAMP:str |RECORD:long|WS_80m_90deg_Avg:double|WS_80m_90deg_Std:double|WS_80m_90deg_3sGust_Max:double|WS_80m_90deg_Max:double|WS_80m_270deg_Avg:double|WS_80m_270deg_Std:double|WS_80m_270deg_3sGust_Max:double|WS_80m_270deg_Max:double|WS_65m_90deg_Avg:double|WS_65m_90deg_Std:double|WS_65m_90deg_3sGust_Max:double|WS_65m_90deg_Max:double|WS_65m_270deg_Avg:double|WS_65m_270deg_Std:double|WS_65m_270deg_3sGust_Max:double|WS_65m_270deg_Max:double|WS_50m_90deg_Avg:double|WS_50m_90deg_Std:double|WS_50m_90deg_3sGust_Max:double|WS_50m_90deg_Max:double|WS_50m_270deg_Avg:double|WS_50m_270deg_Std:double|WS_50m_270deg_3sGust_Max:double|WS_50m_270deg_Max:double|WS_30m_90deg_Avg:double|WS_30m_90deg_Std:double|WS_30m_90deg_3sGust_Max:double|WS_30m_90deg_Max:double|WS_30m_270deg_Avg:double|WS_30m_270deg_Std:double|WS_30m_270deg_3sGust_Max:double|WS_30m_270deg_Max:double|Dir_78m_90deg_avg:double|Dir_78m_90deg_std:double|Dir_63m_90deg_avg:double|Dir_63m_90deg_std:double|Dir_28m_90deg_avg:double|Dir_28m_90deg_std:double|Batt_Volt_Min:double|Press_Avg:long|Temp_C80_Avg:double|Temp_C15_Avg:double|Hum_Avg:double\n", - "--------------+-----------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+--------------------+--------------+-------------------+-------------------+--------------\n", - "2012-05-31 12:|1 |1.383 |0.6 |2.75 |3.37 |1.368 |0.439 |2.673 |2.74 |1.332 |0.478 |2.75 |2.75 |1.242 |0.379 |2.74 |2.79 |1.162 |0.535 |2.337 |2.75 |1.159 |0.354 |2.34 |2.39 |1.27 |0.614 |2.337 |2.75 |1.322 |0.416 |2.157 |2.24 |240.3 |46.0 |242.0 |45.39 |222.0 |33.45 |13.79 |1009 |13.84 |14.08 |65.67 \n", - "20:00 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | \n", - "2012-05-31 12:|2 |1.183 |0.449 |1.923 |2.13 |1.135 |0.324 |1.94 |1.99 |0.948 |0.524 |1.923 |2.13 |1.068 |0.303 |1.723 |1.74 |0.701 |0.547 |1.923 |2.13 |0.913 |0.308 |1.673 |1.74 |0.771 |0.539 |1.717 |2.13 |0.997 |0.28 |1.657 |1.74 |282.0 |26.79 |264.3 |30.25 |278.5 |62.87 |13.73 |1009 |14.04 |14.45 |64.51 \n", - "30:00 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | \n", - "Total count: 2\n", - "\n", - "DuckDataFrame\n", - "timestamp:datetime |RECORD:int|WS_80m_90deg_Avg:double|WS_80m_90deg_Std:double|WS_80m_90deg_3sGust_Max:double|WS_80m_90deg_Max:double|WS_80m_270deg_Avg:double|WS_80m_270deg_Std:double|WS_80m_270deg_3sGust_Max:double|WS_80m_270deg_Max:double|WS_65m_90deg_Avg:double|WS_65m_90deg_Std:double|WS_65m_90deg_3sGust_Max:double|WS_65m_90deg_Max:double|WS_65m_270deg_Avg:double|WS_65m_270deg_Std:double|WS_65m_270deg_3sGust_Max:double|WS_65m_270deg_Max:double|WS_50m_90deg_Avg:double|WS_50m_90deg_Std:double|WS_50m_90deg_3sGust_Max:double|WS_50m_90deg_Max:double|WS_50m_270deg_Avg:double|WS_50m_270deg_Std:double|WS_50m_270deg_3sGust_Max:double|WS_50m_270deg_Max:double|WS_30m_90deg_Avg:double|WS_30m_90deg_Std:double|WS_30m_90deg_3sGust_Max:double|WS_30m_90deg_Max:double|WS_30m_270deg_Avg:double|WS_30m_270deg_Std:double|WS_30m_270deg_3sGust_Max:double|WS_30m_270deg_Max:double|Dir_78m_90deg_avg:int|Dir_78m_90deg_std:double|Dir_63m_90deg_avg:double|Dir_63m_90deg_std:double|Dir_28m_90deg_avg:double|Dir_28m_90deg_std:double|Batt_Volt_Min:double|Press_Avg:int|Temp_C80_Avg:double|Temp_C15_Avg:double|Hum_Avg:double\n", - "-------------------+----------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+---------------------+------------------------+------------------------+------------------------+------------------------+------------------------+--------------------+-------------+-------------------+-------------------+--------------\n", - "2012-05-31 12:20:00|1 |1.383 |0.6 |2.75 |3.37 |1.368 |0.439 |2.673 |2.74 |1.332 |0.478 |2.75 |2.75 |1.242 |0.379 |2.74 |2.79 |1.162 |0.535 |2.337 |2.75 |1.159 |0.354 |2.34 |2.39 |1.27 |0.614 |2.337 |2.75 |1.322 |0.416 |2.157 |2.24 |240 |46.0 |242.0 |45.39 |222.0 |33.45 |13.79 |1009 |13.84 |14.08 |65.67 \n", - "2012-05-31 12:30:00|2 |1.183 |0.449 |1.923 |2.13 |1.135 |0.324 |1.94 |1.99 |0.948 |0.524 |1.923 |2.13 |1.068 |0.303 |1.723 |1.74 |0.701 |0.547 |1.923 |2.13 |0.913 |0.308 |1.673 |1.74 |0.771 |0.539 |1.717 |2.13 |0.997 |0.28 |1.657 |1.74 |282 |26.79 |264.3 |30.25 |278.5 |62.87 |13.73 |1009 |14.04 |14.45 |64.51 \n", - "Total count: 2\n", - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DaskDataFrame\n", - "TIMESTAMP:str |RECORD:long|WS_80m_90deg_Avg:double|WS_80m_90deg_Std:double|WS_80m_90deg_3sGust_Max:double|WS_80m_90deg_Max:double|WS_80m_270deg_Avg:double|WS_80m_270deg_Std:double|WS_80m_270deg_3sGust_Max:double|WS_80m_270deg_Max:double|WS_65m_90deg_Avg:double|WS_65m_90deg_Std:double|WS_65m_90deg_3sGust_Max:double|WS_65m_90deg_Max:double|WS_65m_270deg_Avg:double|WS_65m_270deg_Std:double|WS_65m_270deg_3sGust_Max:double|WS_65m_270deg_Max:double|WS_50m_90deg_Avg:double|WS_50m_90deg_Std:double|WS_50m_90deg_3sGust_Max:double|WS_50m_90deg_Max:double|WS_50m_270deg_Avg:double|WS_50m_270deg_Std:double|WS_50m_270deg_3sGust_Max:double|WS_50m_270deg_Max:double|WS_30m_90deg_Avg:double|WS_30m_90deg_Std:double|WS_30m_90deg_3sGust_Max:double|WS_30m_90deg_Max:double|WS_30m_270deg_Avg:double|WS_30m_270deg_Std:double|WS_30m_270deg_3sGust_Max:double|WS_30m_270deg_Max:double|Dir_78m_90deg_avg:double|Dir_78m_90deg_std:double|Dir_63m_90deg_avg:double|Dir_63m_90deg_std:double|Dir_28m_90deg_avg:double|Dir_28m_90deg_std:double|Batt_Volt_Min:double|Press_Avg:long|Temp_C80_Avg:double|Temp_C15_Avg:double|Hum_Avg:double\n", - "--------------+-----------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+-----------------------+-----------------------+------------------------------+-----------------------+------------------------+------------------------+-------------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+--------------------+--------------+-------------------+-------------------+--------------\n", - "2012-05-31 12:|1 |1.383 |0.6 |2.75 |3.37 |1.368 |0.439 |2.673 |2.74 |1.332 |0.478 |2.75 |2.75 |1.242 |0.379 |2.74 |2.79 |1.162 |0.535 |2.337 |2.75 |1.159 |0.354 |2.34 |2.39 |1.27 |0.614 |2.337 |2.75 |1.322 |0.416 |2.157 |2.24 |240.3 |46.0 |242.0 |45.39 |222.0 |33.45 |13.79 |1009 |13.84 |14.08 |65.67 \n", - "20:00 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | \n", - "2012-05-31 12:|2 |1.183 |0.449 |1.923 |2.13 |1.135 |0.324 |1.94 |1.99 |0.948 |0.524 |1.923 |2.13 |1.068 |0.303 |1.723 |1.74 |0.701 |0.547 |1.923 |2.13 |0.913 |0.308 |1.673 |1.74 |0.771 |0.539 |1.717 |2.13 |0.997 |0.28 |1.657 |1.74 |282.0 |26.79 |264.3 |30.25 |278.5 |62.87 |13.73 |1009 |14.04 |14.45 |64.51 \n", - "30:00 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | \n", - "Total count: 2\n", - "\n" - ] - } - ], - "source": [ - "file = \"/tmp/fugue_example_nonstd.csv\"\n", - "with open(file, \"w\") as f:\n", - " f.write(content)\n", - "\n", - "with FugueWorkflow() as dag:\n", - " df = dag.create(read_text_file, params={\"filepath\": file})\n", - " df.show()\n", - "\n", - "with FugueWorkflow(engine=\"duckdb\") as dag:\n", - " df = dag.create(read_text_file, params={\"filepath\": file})\n", - " df.show()\n", - "\n", - "with FugueWorkflow(engine=\"dask\") as dag:\n", - " df = dag.create(read_text_file, params={\"filepath\": file})\n", - " df.show()\n", - "\n", - "os.unlink(file)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.10.6 ('fugue')", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.10.6" - }, - "vscode": { - "interpreter": { - "hash": "787dcf4b06485f091e5ea0894a1563cc39da567670ce0a93adb376c1b3122bd1" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}