From 592ff644f6bf6eb79493209005c2fc415c93ebe1 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Mon, 9 Feb 2026 01:48:57 -0800 Subject: [PATCH 01/55] init --- .../controller/WorkflowScheduler.scala | 7 +- .../execution/RegionExecution.scala | 2 - .../execution/WorkflowExecution.scala | 6 +- .../architecture/scheduling/Schedule.scala | 9 +++ .../promisehandlers/AssignPortHandler.scala | 3 +- .../InitializeExecutorHandler.scala | 1 + .../workflow/WorkflowExecutionsResource.scala | 2 + .../texera/amber/operator/LogicalOp.scala | 31 +++------ .../amber/operator/loop/LoopEndOpDesc.scala | 53 +++++++++++++++ .../amber/operator/loop/LoopEndOpExec.scala | 8 +++ .../amber/operator/loop/LoopStartOpDesc.scala | 62 ++++++++++++++++++ .../amber/operator/loop/LoopStartOpExec.scala | 44 +++++++++++++ .../workflow-editor.component.ts | 1 - .../src/assets/operator_images/LoopEnd.png | Bin 0 -> 5865 bytes .../src/assets/operator_images/LoopStart.png | Bin 0 -> 2138 bytes 15 files changed, 195 insertions(+), 34 deletions(-) create mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala create mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala create mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala create mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala create mode 100644 frontend/src/assets/operator_images/LoopEnd.png create mode 100644 frontend/src/assets/operator_images/LoopStart.png diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala index 9dcf3ad4bfc..e2239f99a9b 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala @@ -52,6 +52,9 @@ class WorkflowScheduler( this.physicalPlan = updatedPhysicalPlan } - def getNextRegions: Set[Region] = if (!schedule.hasNext) Set() else schedule.next() - + def getNextRegions: Set[Region] = { + val region : Set[Region] = if (!schedule.hasNext) Set() else schedule.loopNext() + println("current Region: " + region) + region + } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala index d5939c2e3b1..e905c2b0449 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala @@ -59,8 +59,6 @@ case class RegionExecution(region: Region) { physicalOpId: PhysicalOpIdentity, inheritOperatorExecution: Option[OperatorExecution] = None ): OperatorExecution = { - assert(!operatorExecutions.contains(physicalOpId), "OperatorExecution already exists.") - operatorExecutions.getOrElseUpdate( physicalOpId, inheritOperatorExecution diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala index dea9b692a4f..31409b180af 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala @@ -44,11 +44,7 @@ case class WorkflowExecution() { * @throws AssertionError if the `RegionExecution` has already been initialized. */ def initRegionExecution(region: Region): RegionExecution = { - // ensure the region execution hasn't been initialized already. - assert( - !regionExecutions.contains(region.id), - s"RegionExecution of ${region.id} already initialized." - ) + regionExecutions.remove(region.id) regionExecutions.getOrElseUpdate(region.id, RegionExecution(region)) } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala index 6f34c9ed1e5..d0ba5268091 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala @@ -21,6 +21,7 @@ package org.apache.texera.amber.engine.architecture.scheduling case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterator[Set[Region]] { private var currentLevel = levelSets.keys.minOption.getOrElse(0) + private var loopStartLevel = currentLevel def getRegions: List[Region] = levelSets.values.flatten.toList @@ -28,6 +29,14 @@ case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterat override def next(): Set[Region] = { val regions = levelSets(currentLevel) + if(regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopStart-operator-")))) loopStartLevel = currentLevel + currentLevel += 1 + regions + } + + def loopNext(): Set[Region] = { + val regions = levelSets(currentLevel) + if(regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-")))) currentLevel = loopStartLevel currentLevel += 1 regions } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala index fe959733abb..1cc725dff83 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala @@ -41,6 +41,7 @@ trait AssignPortHandler { this: DataProcessorRPCHandlerInitializer => override def assignPort(msg: AssignPortRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { + println("ergergerge") val schema = Schema.fromRawSchema(msg.schema) if (msg.input) { val inputPortURIStrs = msg.storageUris.toList @@ -55,7 +56,7 @@ trait AssignPortHandler { // Same as AddInputChannelHandler dp.inputGateway.getChannel(channelId).setPortId(msg.portId) dp.inputManager.getPort(msg.portId).channels.add(channelId) - dp.stateManager.assertState(READY, RUNNING, PAUSED) + //dp.stateManager.assertState(READY, RUNNING, PAUSED) } } else { val storageURIOption: Option[URI] = msg.storageUris.head match { diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala index bf45d8eff9a..cc1a32594b6 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala @@ -39,6 +39,7 @@ trait InitializeExecutorHandler { req: InitializeExecutorRequest, ctx: AsyncRPCContext ): Future[EmptyReturn] = { + println(s"Initializing executor with request: $req") dp.serializationManager.setOpInitialization(req) val workerIdx = VirtualIdentityUtils.getWorkerIndex(actorId) val workerCount = req.totalWorkerCount diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala index 72fb1c364e5..92582afdd2b 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala @@ -247,6 +247,8 @@ object WorkflowExecutionsResource { OPERATOR_PORT_EXECUTIONS.RESULT_URI ) .values(eid.id.toInt, globalPortId.serializeAsString, uri.toString) + .onConflict() + .doNothing() .execute() } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala index eb319a82d1d..ee57514212b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala @@ -24,15 +24,8 @@ import com.fasterxml.jackson.annotation._ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OperatorExecutor import org.apache.texera.amber.core.tuple.Schema -import org.apache.texera.amber.core.virtualidentity.{ - ExecutionIdentity, - OperatorIdentity, - WorkflowIdentity -} -import org.apache.texera.amber.core.workflow.WorkflowContext.{ - DEFAULT_EXECUTION_ID, - DEFAULT_WORKFLOW_ID -} +import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, OperatorIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.WorkflowContext.{DEFAULT_EXECUTION_ID, DEFAULT_WORKFLOW_ID} import org.apache.texera.amber.core.workflow.{PhysicalOp, PhysicalPlan, PortIdentity} import org.apache.texera.amber.operator.aggregate.AggregateOpDesc import org.apache.texera.amber.operator.cartesianProduct.CartesianProductOpDesc @@ -42,22 +35,14 @@ import org.apache.texera.amber.operator.distinct.DistinctOpDesc import org.apache.texera.amber.operator.dummy.DummyOpDesc import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc -import org.apache.texera.amber.operator.huggingFace.{ - HuggingFaceIrisLogisticRegressionOpDesc, - HuggingFaceSentimentAnalysisOpDesc, - HuggingFaceSpamSMSDetectionOpDesc, - HuggingFaceTextSummarizationOpDesc -} +import org.apache.texera.amber.operator.huggingFace.{HuggingFaceIrisLogisticRegressionOpDesc, HuggingFaceSentimentAnalysisOpDesc, HuggingFaceSpamSMSDetectionOpDesc, HuggingFaceTextSummarizationOpDesc} import org.apache.texera.amber.operator.ifStatement.IfOpDesc import org.apache.texera.amber.operator.intersect.IntersectOpDesc import org.apache.texera.amber.operator.intervalJoin.IntervalJoinOpDesc import org.apache.texera.amber.operator.keywordSearch.KeywordSearchOpDesc import org.apache.texera.amber.operator.limit.LimitOpDesc import org.apache.texera.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc -import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{ - SklearnAdvancedKNNClassifierTrainerOpDesc, - SklearnAdvancedKNNRegressorTrainerOpDesc -} +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{SklearnAdvancedKNNClassifierTrainerOpDesc, SklearnAdvancedKNNRegressorTrainerOpDesc} import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVRTrainer.SklearnAdvancedSVRTrainerOpDesc import org.apache.texera.amber.operator.metadata.{OPVersion, OperatorInfo, PropertyNameConstants} @@ -71,10 +56,7 @@ import org.apache.texera.amber.operator.sleep.SleepOpDesc import org.apache.texera.amber.operator.sort.{SortOpDesc, StableMergeSortOpDesc} import org.apache.texera.amber.operator.sortPartitions.SortPartitionsOpDesc import org.apache.texera.amber.operator.source.apis.reddit.RedditSearchSourceOpDesc -import org.apache.texera.amber.operator.source.apis.twitter.v2.{ - TwitterFullArchiveSearchSourceOpDesc, - TwitterSearchSourceOpDesc -} +import org.apache.texera.amber.operator.source.apis.twitter.v2.{TwitterFullArchiveSearchSourceOpDesc, TwitterSearchSourceOpDesc} import org.apache.texera.amber.operator.source.fetcher.URLFetcherOpDesc import org.apache.texera.amber.operator.source.scan.FileScanSourceOpDesc import org.apache.texera.amber.operator.source.scan.arrow.ArrowSourceOpDesc @@ -137,6 +119,7 @@ import org.apache.texera.amber.operator.visualization.volcanoPlot.VolcanoPlotOpD import org.apache.texera.amber.operator.visualization.waterfallChart.WaterfallChartOpDesc import org.apache.texera.amber.operator.visualization.wordCloud.WordCloudOpDesc import org.apache.commons.lang3.builder.{EqualsBuilder, HashCodeBuilder, ToStringBuilder} +import org.apache.texera.amber.operator.loop.{LoopEndOpDesc, LoopStartOpDesc} import org.apache.texera.amber.operator.visualization.stripChart.StripChartOpDesc import java.util.UUID @@ -202,6 +185,8 @@ trait StateTransferFunc new Type(value = classOf[TypeCastingOpDesc], name = "TypeCasting"), new Type(value = classOf[LimitOpDesc], name = "Limit"), new Type(value = classOf[SleepOpDesc], name = "Sleep"), + new Type(value = classOf[LoopStartOpDesc], name = "LoopStart"), + new Type(value = classOf[LoopEndOpDesc], name = "LoopEnd"), new Type(value = classOf[RandomKSamplingOpDesc], name = "RandomKSampling"), new Type(value = classOf[ReservoirSamplingOpDesc], name = "ReservoirSampling"), new Type(value = classOf[HashJoinOpDesc[String]], name = "HashJoin"), diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala new file mode 100644 index 00000000000..f56068e9036 --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.loop + +import org.apache.texera.amber.core.executor.OpExecWithClassName +import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} + +class LoopEndOpDesc extends LogicalOp { + override def getPhysicalOp( + workflowId: WorkflowIdentity, + executionId: ExecutionIdentity + ): PhysicalOp = { + PhysicalOp + .oneToOnePhysicalOp( + workflowId, + executionId, + operatorIdentifier, + OpExecWithClassName("org.apache.texera.amber.operator.loop.LoopEndOpExec") + ) + .withInputPorts(operatorInfo.inputPorts) + .withOutputPorts(operatorInfo.outputPorts) + .withSuggestedWorkerNum(1) + } + + override def operatorInfo: OperatorInfo = + OperatorInfo( + "Loop End", + "Loop End", + OperatorGroupConstants.CONTROL_GROUP, + inputPorts = List(InputPort()), + outputPorts = List(OutputPort()) + ) +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala new file mode 100644 index 00000000000..60f18cd5fc9 --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala @@ -0,0 +1,8 @@ +package org.apache.texera.amber.operator.loop + +import org.apache.texera.amber.core.executor.OperatorExecutor +import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} + +class LoopEndOpExec extends OperatorExecutor { + override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = Iterator(tuple) +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala new file mode 100644 index 00000000000..4f3e86eb624 --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.loop + +import com.fasterxml.jackson.annotation.JsonProperty +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +import org.apache.texera.amber.core.executor.OpExecWithClassName +import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.util.JSONUtils.objectMapper + +class LoopStartOpDesc extends LogicalOp { + + override def getPhysicalOp( + workflowId: WorkflowIdentity, + executionId: ExecutionIdentity + ): PhysicalOp = { + PhysicalOp + .oneToOnePhysicalOp( + workflowId, + executionId, + operatorIdentifier, + OpExecWithClassName( + "org.apache.texera.amber.operator.loop.LoopStartOpExec", + objectMapper.writeValueAsString(this) + ) + ) + .withInputPorts(operatorInfo.inputPorts) + .withOutputPorts(operatorInfo.outputPorts) + .withSuggestedWorkerNum(1) + .withParallelizable(false) + } + + override def operatorInfo: OperatorInfo = + OperatorInfo( + "Loop Start", + "Loop Start", + OperatorGroupConstants.CONTROL_GROUP, + inputPorts = List(InputPort()), + outputPorts = List(OutputPort()) + ) + +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala new file mode 100644 index 00000000000..1ff10c650cb --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.loop + +import org.apache.texera.amber.core.executor.OperatorExecutor +import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} +import org.apache.texera.amber.util.JSONUtils.objectMapper + +import scala.collection.mutable + +class LoopStartOpExec(descString: String) extends OperatorExecutor { + private val data = new mutable.ArrayBuffer[Tuple] + private var currentIteration = 0 + + override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = { + data.append(tuple) + Iterator.empty + } + + override def onFinish(port: Int): Iterator[TupleLike] = { + currentIteration += 1 + data.iterator + } + +} + + diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts index b23f92caf32..185b723b778 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts @@ -341,7 +341,6 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy body: { fill: "rgba(158,158,158,0.2)", pointerEvents: "none", - visibility: "hidden", }, }, }, diff --git a/frontend/src/assets/operator_images/LoopEnd.png b/frontend/src/assets/operator_images/LoopEnd.png new file mode 100644 index 0000000000000000000000000000000000000000..ee0f9ab6faccd328c102f214a7547d3e34d8359b GIT binary patch literal 5865 zcmVPy0qe(P?J^Czx6vRgiNy(U0Jp~gW+wJeo>FF6J zv)#V8`#arz@9BT?$35r!zVkc3^qfB5Mles!2yn63hL)Czs8n_X^S=Vv1(x+u03U(x zw}GeyL>8ERFmHopZ36HFlo}CsI${gOizOdGK38`k{#z@ z0LKG37R*P3=tuxoiu(3&tdALqidki^V-aG#x}8IXROixtAl!S_|ebFwduQ`ESz7 zbjWWA!Ukv4O65`j7wZ5mr#j}LP?(eVvu>^l%f;g1u&m1fd@ZLWO2h2?;QMz`x%|g8 zl*iSXOTu!ocp!+b29XmNfTo?#?1K3=DwqG9Ck%2$Se`Ls0yb^B0l<}c0y*Qh-j24m zSu}U<#*Et;n3o&Evg2F~=9?0Ai{@d6bl9le!$dcd>(0}mE{=}O6P9iJSP*>=z$tMS zYUV~lU>TUNAtNe;3TxWfHkYHj3yLDwV#I#U*Bvu(0mQ2kYVYyhm7M68OmOyH1TYUdIDwl7~3I>@WY#2Q?4@75X1sr2X zhye5BAi6jU(P5b&EIZC=0J;F|X-JY;1XL(v2f6Og)3h*k!m{HiPt~1if?#ylguvzG zx_8wyBf-I`5|(XyE{HBka6Lwp4g{F*AgLzA@=ZJefa|EJsK{hR*vTAn) zvr1r8XZ5{7q#gf6nX9h83WX`u)%BM|<&GgtD_}u%716^WQrV|@dR`3@0)bb+yaLQ` zpj=)>ij;LetjJs}?hm4aK{Oe_VPWFqp>>K(XBzV=FrTL@`b!&O*>R2s^P@S51$YU- zQ}F#~FmBwlG=KiPy4b+er|*m{TTX;!oebd906w8hcinpWViC?E*L}Wj84)LHA#CWm zQg?Z~M6@f7VN0Oy8(UkuY5x44Gz=I?Cr_U~5nH#aq)Bx@eO6ZN8fFKxDiCs}v}a75 zu;H8aiHwrl6>_cRw#T%;`Ts|C%qXH zCfrK1XKzk@12q}IGiUCAjT^58aD4{udArGVznJ9G6DMpY+^t0SgXji{KS@b~FDWZH zk}=N$^H(MHRF$2u8#htHW+0G7m8-mp%H>C^0@)-3xL7;~mi2uAs$6JVPhtTxi4iso z5qu&oM0(wxcVokb%jltpDk<8KN!5J*`33B>lZvqVW{L)?>#o?LiHQ(42)+hhPAQ(X zM|~kvSJxvcZbmLtb4SOy!A(gzw+Skq_L#Uq;K>lywwI)mDZsoGt*z6uM_=89)n4R+c?}AMbE&KA_w_8)%o!a4?&vrQmC7R^+D98s zbr$QWi?Ct&pck|)4p^x|>r59FImE**I!md%5|zs7v~b~@ za@%DjD+KtWiw;1cpzhLjIT@Cb>#892$WP6L4XOwN_e9nZXP`O;5Piu3M+uYzy z%ls>i$Ae31shU_5VZ$h_H#Nn#z6;Ab)tp&tBU}=rgEOt~KdI$IOvTTRAAdm9sIJ-w z>o`BqR!o@aG;yXu5>9Cha^-C^)lf%q~72=*S-=8>I zZ``p7Yuh*UW?tSTiiYoh$>d;(BAlr3Fh}fXi3%MmjQK|LyjdgZ7@M$;vqrQlHqrf5 zDw#g9qPw5GaBgqEPdFQQExGQ9kz=D0mK|pbfJMnKHT^Y zMPKzBD26Fh$#oYGoj(>~Ws6}j&mzyeF~)+-)D#h5+g5Ed5Pn*R{b*Koqh_IllnmX_TcsXlA7Gd8GCyKUP#v7*XDUU0(+ zYunF)=me3CV4g#sH&aBm5tOzqzWo0|1dvCo_Auu;Oz(-*2q$45}AkS$UK*6 zmjNaJQiJmKeQlwnp1t~Oa&1po+r9`yWg)Fklk5JwkZdD1Wdzu9{#&e32y>ACT^(h7uwg|uzS-Z)C6@fKtLWQ$zZF1f6 zV8Y5}7PPeNX>zr)Mz7@U33IlpahXP?$S!WN5%!?I-R4*32VR4%94T_&r0 z+mLo~v3NQxt6OAos%}9@*b8A<|0SZNrDYcrJ0OB=CWVHv1@DL$=llQBM_9*E318O> zD0!7!_b>suMr4K&V8?j{z@Y*r-rPr6+kOy4QxjLfd@p(4MTyHb0yB*O+x9{bjak8^ zt_3hJ>L=_=AgZfL!kCIpW3O=k>X`?T{}k@8&eZRfE`DLs;ZFHL?1=i%K+S&sMJ1Trbw{> zsCf}7Rs2{T39f}@-68;liP91PH7_EiqPEm6flSe2Q=Kp`+!C=%5_I;46G)W3 zyop>_$u>_DA|S;p^M-9gGU-^d+#A-MVRh_Z)9H{q!gG@pR>g2Kg~n&eLUL3G%~&99b$COJLi_g6M_Bu`tnEi9?zvfpXDW zCLT6VJjjKuvhlDc;~*C#$di$x83zgowr!Ksl_y4I+*ZLD>-8pUHshP0NAq-K&6>Id zd1OXLZI-V~U@E_4bnEkJmVD(GIp+ajYHQ__b#+>;r?!@zzhRSXiMBdJu9X|E(^v=Lkl0d8+U0?dD? zHZI2Rdy(q~9TvCi<{jS0J`17~VhA67D)SujyqO}hjnLE(VB4MzqDw^#V1AA~@5FxD z!wD-_r$Fi4jFy(&P2Vn&txZK}7(1{Izz!k?`2KZNF5fyjVW&;|7%G)lMU*R{Lktkp zV+3nKK(Lr#pZ$fxp|oJZYoii2JQhf~X_fdMluCP<-UnLvB&sCH-D+=F-#imV6on0u z2g@FIBnSvwEM5i6x=qB0KmlJ#o_B|sd?Q#30%Dm3{nq>b)l@EjKMG+d!Lt4$xaxJ- zamRgW_Uz4qI*e#R1bF7m9q`_JZ}nCU+FeYI@9#(D^6ILD4PJtdvlPG?V#b*Iv0~8b z5G+-laFFfY80wY`R)Y=F;W5xu0@PTs4?G{CN%rWo6 zvQDM0E_K*9k46NzqvI&}{*wT97FWx>9^=O!Ky&AAjH0V1!iH~IslHyr?PXpAqR)}* zz7b_HW=wnp*l`X9^RGZOqH8q1hG7yeA=jN3Rc_6M4N3bHfK#F>i8*d1DwWe|;lej# z$}!WcAQ1K@ctYHh^`RZh$aOzm%f{44SgBS?qg$>%_RcbI+FG`@wz0vR6~)Q=Oh5nL{*m9 zICD>%2$WjyS+}?NWOCi#)w76Lgbm)ZvULo`T9fgJ)wAQ_9Bm$~p*F@$^N&r~@My3E zMEYX@n3tlpb-D?diO$?0kY(%E`C4KFhML9vICw4S@MVZ&u6b746A zVWz-c0KS>l<^VXITsO9>H!;G7q`eHlohgE{625+>ast=4dB5}fo zZ(7+R98uP$>K*_$W5R@6o2kM~P?2Th#%lpwFI+bu$}Wsp3?s_K(ZWvu@e)4RuqJ(%1h8UwOO=g~Defj$~ql4M?kU!H9mTDSkF8 z-#3GK0W8bSbNTw9OuO$pAetstOgJi{gU6}(R_4!F%gw4sw3;JI*zi1iJeZ#V(Ju8G zDLC(G06)Q|O^fJ(2mT?bClS$MYoVzCz5?Kri3m+n2=hB2QrB0{Cn7BXQ5~6jL?%9%S$9hqCjpRS?!#EGZ;7SS)FIz@JxClEbC+dD!BO*Q8zQSaq1XAbdKbkwSR5e2s`LZ`*4On z#G>x;t6^K8SHQdi%x|DvUKQuoA^55<7WW6y!62Fp;4ly!3E%&v|=H`>}R zNcx_&n%Yzd+iw)xcKUAIY6zL%ha-@%yR{aGQzdNpF_w0~Of7hsu|Q&hEcJkyI$=Z7 zs?bDLnkbEU+Inzn#yKjUwIhiT!8qV$g0LZJ_XP92R5BOhAlM9Z`|i*$jvIWP1XP(J z?7+D;y&SP5@N7I8F>|nz1|)NY4d1rPGxo#WUwtLLTRP>bD=VUJNqeqpT~a0qJMgyE zU96*4^cr=1okcEmz>#Uf25;NGvPj?4Sj$TX?pf8b49wTmT0ThUVlq$Ifw!&WT-@t9 zx+y2I2s-yLzOKGlfE#P6A{bw3eG+m**x+s3=OMd6TT$iuFwNVn+tJoGOSi|WUXXG{ z*g-c52ZHEoO+6Rt#WvgXm|Za67NtykmKKsr!VV(t;jk>_DKq_QqqFKh`2JnGqN}2t z6>Vg$2|GaCzUFr-r%6=@H_ylj%tN6thZZb&EzbZo1YrlgarXexbP#m};d|y;3+65` z&mXOKYu+GgD8dc~pB-lkfUh@c2xB*Zd&zYd=Z%on>}^=W4tncO0x%UsUv6m6net3A z{}jL?DwkJR+xWcx-tdIo?yalNx?8uN4I-7)cS2q_ZNP5k=fK>J*4D@Jem0G|7fnUj zAx8!k9rjreoeJPnIXUyHl$_s!`AHD{It$TLQHhq+xTYoSPyoX&D#rsj7R*P734vBp zK-VG14-y}VmIF|oR-QLqbTkywsAHotF@-xjEv#PsaR8HHS%-jGm8(=`m-bE6J*?cK zz+1f^h^s)P%7wlTUk`V)0_`k4SDLq_~R%4M5l?=V1$00000NkvXXu0mjf{#zRE literal 0 HcmV?d00001 diff --git a/frontend/src/assets/operator_images/LoopStart.png b/frontend/src/assets/operator_images/LoopStart.png new file mode 100644 index 0000000000000000000000000000000000000000..7e5be023cdf6b64dd1bc140e5d75980455afeefb GIT binary patch literal 2138 zcmV-g2&MOlP)Px-6iGxuRCr$Pok@1vFbsy}TQ#e+JKw6wt+cDNN^a$bl4voq27m;>LBHSYqAkVY z$0rb+#7kG<@A`TD-M{p=3agGPM=EfMd@k!_*RSip?%sX$Fa2BAilA^a1?mO0>wb6l z?j5k2v68i*x_G;KNw`$Lu656Tc9Eo zuwW)j8SQ}zShIknU1*!20#9EcQB z6cz^>1a(cxTka+6Y@h~k>Vdf?xiG9J0X0-|fOc49G29qr3Tieum%s|bdO}cVHSgWK zRd`?mYB7L@zzV^7YEZqNBe1F-Ru$AD0sZM+llKS!)Id<7mwzo{Qv-F_Gg`;Ajf=V5 zcEPBP*Z_3}D+OT1L9Nzsk<(YzwwE$XlA1;kSW!?b@}lb|h!7T_9?%@2san@xK`cu5 z4NPGHDnVMNrYfLLkh*xZogYJ3fJ!!d(!e#T>u=Ub0F_K1J7MHlAR1IaI-00i9oPgi z9*BT73e;we!|nF0U1^@ zs7;RU1&&!a`tnAT$9|9)ZqY&gy58B9DER@RKu1jgs7DGB{-0nA5~$U$I4!jz2t$w; zqkgJLlynDUghNo9^B97VVtR@~AU#e0tKZ~t7t{tJ75|b-a~jdPl25tgMD{_gXRF?k zXh|dcz09jVg*qRav^@^AC+J29unB4*p2DAq9F9D=pW8RejG4((AV;*f{{Mr`-7oK{ z+EsgPgPL!YKX^Y6rKgkdgTv61*9U7K6zNnkYM3epFlN!gQDWcT~H&}?8PnHV1_x_cm~zVnY^)I>;EKn0{3-V~!vN)Oxq z1Xa7+7l8B!s6zQ=uyYedn!q}B{|`_Nx_vRvhYQil;ar^?Yf$$#9B^gF6_Z@Wbki%x ztAm4^J*sB6k1kr5l+3wnhXBsW4WL>Slvzs@udINoSG{+mM^%k8;7yegDI0JR1JDygFuPXM(N0BWhGCaCVhT#aw=V=z?de(toRjZVSEj!2b;lCwiOAI&NI40$xA|c}btX{twu`E7^XstQe;!h`;?wV2 z%Jo4-_kRvhHM@QHbw`UcQ|SIrz^bA*?VPi-I)#csRl~5VdQ*UDgxEX`@Jqdlp9+)We4}*#tzgGR)hetLR`( zU|H*=iJ9@^;87n?j-wUVwYy&yI3H%)=vntJ?akv=w?Xq4_}$+EM50(-LrH< z#{4!zlKpGYwa)zOG_vem6=HDBqaRts+|g(kR99Km4F_4&$W*8>sEl|tQ-hhs9_Vun z>gTnz5P+#bA;|?nZ4OexEW{zGgI7fekWRZ0s4d!pa$Jc7Dqv0LRAQSWb7U`2X@e|? zo0Q0)PVs8qDJuir)QU*(sEv*6XNeh_us{W@2^ces5|+~3$lf=OsRtia46fp6Wg4dG za4SL!HmH+;m@# zDsgnbrDHOvO~+D)^f3}JKy6IZ+tl4B?Q;+TwJ~8C^f8h$L2X1+(!N)UiejKPA}mF- zxHJ)4oya1?$hF7Y9dSk14)CI8Pk&?3lD+Fpmu$lmB4(g-(a!au8T+XF? z&3-<|(l-l&nh{o0IL%^I3J3n{a?L|445|^V{+yco^kdRID20W+G-h2trJ@70V|d`a zt^4IY5lN{uHH`wO6RauwE@t@doj)}B__y3}7UfZ?b`FmA)(Wr6Db&ar0hK8vsx4#E z73EQ58h~eJ1sL0ssI2 literal 0 HcmV?d00001 From b95465d2e0fe28fe4edc05cf955fe2b18ac28419 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 10 Feb 2026 18:19:19 -0800 Subject: [PATCH 02/55] update --- .../worker/promisehandlers/InitializeExecutorHandler.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala index cc1a32594b6..bf45d8eff9a 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala @@ -39,7 +39,6 @@ trait InitializeExecutorHandler { req: InitializeExecutorRequest, ctx: AsyncRPCContext ): Future[EmptyReturn] = { - println(s"Initializing executor with request: $req") dp.serializationManager.setOpInitialization(req) val workerIdx = VirtualIdentityUtils.getWorkerIndex(actorId) val workerCount = req.totalWorkerCount From d9d0cd94388ef21f1e32289d8f0e5d359553443f Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 10 Feb 2026 18:30:19 -0800 Subject: [PATCH 03/55] update --- .../architecture/controller/execution/RegionExecution.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala index e905c2b0449..d5939c2e3b1 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala @@ -59,6 +59,8 @@ case class RegionExecution(region: Region) { physicalOpId: PhysicalOpIdentity, inheritOperatorExecution: Option[OperatorExecution] = None ): OperatorExecution = { + assert(!operatorExecutions.contains(physicalOpId), "OperatorExecution already exists.") + operatorExecutions.getOrElseUpdate( physicalOpId, inheritOperatorExecution From 144ae29f27dbb693f8f499da10ebb73ea1e6b88b Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 10 Feb 2026 18:37:00 -0800 Subject: [PATCH 04/55] update --- .../architecture/worker/promisehandlers/AssignPortHandler.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala index 1cc725dff83..57a7782cf55 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala @@ -41,7 +41,6 @@ trait AssignPortHandler { this: DataProcessorRPCHandlerInitializer => override def assignPort(msg: AssignPortRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { - println("ergergerge") val schema = Schema.fromRawSchema(msg.schema) if (msg.input) { val inputPortURIStrs = msg.storageUris.toList From 7b13fef2bd720015728fbe4051ad7ff5a4a9cdd5 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 10 Feb 2026 18:39:58 -0800 Subject: [PATCH 05/55] update --- .../engine/architecture/controller/WorkflowScheduler.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala index e2239f99a9b..385779fd38d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala @@ -52,9 +52,5 @@ class WorkflowScheduler( this.physicalPlan = updatedPhysicalPlan } - def getNextRegions: Set[Region] = { - val region : Set[Region] = if (!schedule.hasNext) Set() else schedule.loopNext() - println("current Region: " + region) - region - } + def getNextRegions: Set[Region] = if (!schedule.hasNext) Set() else schedule.loopNext() } From bd5ac3aba6c1449bf7c4e62af64ad3b7bd3163c7 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 10 Feb 2026 18:45:46 -0800 Subject: [PATCH 06/55] update --- .../amber/operator/loop/LoopEndOpExec.scala | 19 +++++++++++++++++++ .../amber/operator/loop/LoopStartOpExec.scala | 18 +----------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala index 60f18cd5fc9..9083c1e77aa 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.texera.amber.operator.loop import org.apache.texera.amber.core.executor.OperatorExecutor diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala index 1ff10c650cb..f15d90a6cca 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala @@ -18,27 +18,11 @@ */ package org.apache.texera.amber.operator.loop - import org.apache.texera.amber.core.executor.OperatorExecutor import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} -import org.apache.texera.amber.util.JSONUtils.objectMapper - -import scala.collection.mutable class LoopStartOpExec(descString: String) extends OperatorExecutor { - private val data = new mutable.ArrayBuffer[Tuple] - private var currentIteration = 0 - - override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = { - data.append(tuple) - Iterator.empty - } - - override def onFinish(port: Int): Iterator[TupleLike] = { - currentIteration += 1 - data.iterator - } - + override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = Iterator(tuple) } From 19be0c1a3b4282f6c1830b254de6b78a6f2c705e Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 10 Feb 2026 19:28:06 -0800 Subject: [PATCH 07/55] update --- .../architecture/worker/promisehandlers/AssignPortHandler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala index 57a7782cf55..fe959733abb 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala @@ -55,7 +55,7 @@ trait AssignPortHandler { // Same as AddInputChannelHandler dp.inputGateway.getChannel(channelId).setPortId(msg.portId) dp.inputManager.getPort(msg.portId).channels.add(channelId) - //dp.stateManager.assertState(READY, RUNNING, PAUSED) + dp.stateManager.assertState(READY, RUNNING, PAUSED) } } else { val storageURIOption: Option[URI] = msg.storageUris.head match { From 706884f0b9e32c60bf7bfb6586fc10ab98b8758c Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 10 Feb 2026 20:32:39 -0800 Subject: [PATCH 08/55] update --- .../amber/engine/architecture/common/AmberProcessor.scala | 4 ++-- .../texera/amber/engine/common/rpc/AsyncRPCClient.scala | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/AmberProcessor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/AmberProcessor.scala index e7763073232..22811b46417 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/AmberProcessor.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/AmberProcessor.scala @@ -43,7 +43,7 @@ abstract class AmberProcessor( with Serializable { /** FIFO & exactly once */ - val inputGateway: InputGateway = new NetworkInputGateway(this.actorId) + val inputGateway: NetworkInputGateway = new NetworkInputGateway(this.actorId) // 1. Unified Output val outputGateway: NetworkOutputGateway = @@ -55,7 +55,7 @@ abstract class AmberProcessor( } ) // 2. RPC Layer - val asyncRPCClient = new AsyncRPCClient(outputGateway, actorId) + val asyncRPCClient = new AsyncRPCClient(inputGateway, outputGateway, actorId) val asyncRPCServer: AsyncRPCServer = new AsyncRPCServer(outputGateway, actorId) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCClient.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCClient.scala index 704ebd7f476..f7e26803b47 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCClient.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCClient.scala @@ -27,7 +27,10 @@ import org.apache.texera.amber.core.virtualidentity.{ EmbeddedControlMessageIdentity } import org.apache.texera.amber.engine.architecture.controller.ClientEvent -import org.apache.texera.amber.engine.architecture.messaginglayer.NetworkOutputGateway +import org.apache.texera.amber.engine.architecture.messaginglayer.{ + NetworkInputGateway, + NetworkOutputGateway +} import org.apache.texera.amber.engine.architecture.rpc.controlcommands._ import org.apache.texera.amber.engine.architecture.rpc.controllerservice.ControllerServiceFs2Grpc import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{ @@ -125,7 +128,8 @@ object AsyncRPCClient { } class AsyncRPCClient( - outputGateway: NetworkOutputGateway, + val inputGateway: NetworkInputGateway, + val outputGateway: NetworkOutputGateway, val actorId: ActorVirtualIdentity ) extends AmberLogging { From 21e6a41bb503d23ab1d67af3c142e6237528ac66 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 10 Feb 2026 20:33:28 -0800 Subject: [PATCH 09/55] update --- .../architecture/messaginglayer/NetworkInputGateway.scala | 4 ++++ .../architecture/messaginglayer/NetworkOutputGateway.scala | 4 ++++ .../architecture/scheduling/RegionExecutionCoordinator.scala | 2 ++ 3 files changed, 10 insertions(+) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/NetworkInputGateway.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/NetworkInputGateway.scala index 5cfd8aabc04..1d3ee3cb72c 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/NetworkInputGateway.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/NetworkInputGateway.scala @@ -86,4 +86,8 @@ class NetworkInputGateway(val actorId: ActorVirtualIdentity) enforcers += enforcer } + def removeControlChannel(from: ActorVirtualIdentity): Unit = { + inputChannels.remove(ChannelIdentity(from, actorId, isControl = true)) + } + } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/NetworkOutputGateway.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/NetworkOutputGateway.scala index 929a30f4efa..e35e819d41f 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/NetworkOutputGateway.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/NetworkOutputGateway.scala @@ -94,4 +94,8 @@ class NetworkOutputGateway( idToSequenceNums.getOrElseUpdate(channelId, new AtomicLong()).getAndIncrement() } + def removeControlChannel(to: ActorVirtualIdentity): Unit = { + idToSequenceNums.remove(ChannelIdentity(actorId, to, isControl = true)) + } + } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index 7e5b228801f..4b861de6574 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@ -166,6 +166,8 @@ class RegionExecutionCoordinator( val actorRef = actorRefService.getActorRef(workerId) // Remove the actorRef so that no other actors can find the worker and send messages. actorRefService.removeActorRef(workerId) + asyncRPCClient.outputGateway.removeControlChannel(workerId) + asyncRPCClient.inputGateway.removeControlChannel(workerId) gracefulStop(actorRef, Duration(5, TimeUnit.SECONDS)).asTwitter() } }.toSeq From 24da3e38ac1498a1f4cc1da0342d2d436cfdcc28 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 11 Feb 2026 00:27:38 -0800 Subject: [PATCH 10/55] update --- .../amber/engine/architecture/scheduling/Schedule.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala index d0ba5268091..c4b9c34c6cb 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala @@ -22,6 +22,7 @@ package org.apache.texera.amber.engine.architecture.scheduling case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterator[Set[Region]] { private var currentLevel = levelSets.keys.minOption.getOrElse(0) private var loopStartLevel = currentLevel + private var i = 0 def getRegions: List[Region] = levelSets.values.flatten.toList @@ -36,7 +37,10 @@ case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterat def loopNext(): Set[Region] = { val regions = levelSets(currentLevel) - if(regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-")))) currentLevel = loopStartLevel + if(regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-"))) && i < 5) { + currentLevel = loopStartLevel + i+=1 + } currentLevel += 1 regions } From 2ba0fa400d9a993352259e97583324c5bb62ef91 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 11 Feb 2026 01:39:46 -0800 Subject: [PATCH 11/55] update --- .../apache/texera/amber/operator/loop/LoopEndOpDesc.scala | 8 ++++++++ .../apache/texera/amber/operator/loop/LoopEndOpExec.scala | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index f56068e9036..72f97f7c898 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -19,6 +19,8 @@ package org.apache.texera.amber.operator.loop +import com.fasterxml.jackson.annotation.JsonProperty +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} @@ -26,6 +28,12 @@ import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} class LoopEndOpDesc extends LogicalOp { + + @JsonProperty(required = true) + @JsonSchemaTitle("Iteration Number") + var iteration: Int = _ + + override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala index 9083c1e77aa..cb9393ed4f8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala @@ -21,7 +21,12 @@ package org.apache.texera.amber.operator.loop import org.apache.texera.amber.core.executor.OperatorExecutor import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} +import org.apache.texera.amber.util.JSONUtils.objectMapper + + +class LoopEndOpExec(descString: String) extends OperatorExecutor { + private val desc: LoopEndOpDesc = objectMapper.readValue(descString, classOf[LoopEndOpDesc]) + val iteration: Int = desc.iteration -class LoopEndOpExec extends OperatorExecutor { override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = Iterator(tuple) } From a05ffd1f780a8c1911df7e16b408206ab3820804 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 11 Feb 2026 02:06:08 -0800 Subject: [PATCH 12/55] update --- .../apache/texera/amber/operator/loop/LoopEndOpDesc.scala | 7 ++++--- .../apache/texera/amber/operator/loop/LoopEndOpExec.scala | 5 +---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index 72f97f7c898..31719229f0b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -19,13 +19,14 @@ package org.apache.texera.amber.operator.loop -import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.util.JSONUtils.objectMapper class LoopEndOpDesc extends LogicalOp { @@ -33,7 +34,6 @@ class LoopEndOpDesc extends LogicalOp { @JsonSchemaTitle("Iteration Number") var iteration: Int = _ - override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity @@ -43,10 +43,11 @@ class LoopEndOpDesc extends LogicalOp { workflowId, executionId, operatorIdentifier, - OpExecWithClassName("org.apache.texera.amber.operator.loop.LoopEndOpExec") + OpExecWithClassName("org.apache.texera.amber.operator.loop.LoopEndOpExec",objectMapper.writeValueAsString(this)) ) .withInputPorts(operatorInfo.inputPorts) .withOutputPorts(operatorInfo.outputPorts) + .withParallelizable(false) .withSuggestedWorkerNum(1) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala index cb9393ed4f8..b1e6deace19 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala @@ -24,9 +24,6 @@ import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} import org.apache.texera.amber.util.JSONUtils.objectMapper -class LoopEndOpExec(descString: String) extends OperatorExecutor { - private val desc: LoopEndOpDesc = objectMapper.readValue(descString, classOf[LoopEndOpDesc]) - val iteration: Int = desc.iteration - +class LoopEndOpExec extends OperatorExecutor { override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = Iterator(tuple) } From 44fc0e795bc02d064c3e517bd0004615bbd7fa6d Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 11 Feb 2026 02:06:12 -0800 Subject: [PATCH 13/55] update --- .../architecture/scheduling/Schedule.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala index c4b9c34c6cb..20f29942218 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala @@ -19,10 +19,14 @@ package org.apache.texera.amber.engine.architecture.scheduling +import org.apache.texera.amber.core.executor.OpExecWithClassName +import org.apache.texera.amber.operator.loop.LoopEndOpDesc +import org.apache.texera.amber.util.JSONUtils.objectMapper + case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterator[Set[Region]] { private var currentLevel = levelSets.keys.minOption.getOrElse(0) private var loopStartLevel = currentLevel - private var i = 0 + private var i = 1 def getRegions: List[Region] = levelSets.values.flatten.toList @@ -37,10 +41,14 @@ case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterat def loopNext(): Set[Region] = { val regions = levelSets(currentLevel) - if(regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-"))) && i < 5) { - currentLevel = loopStartLevel - i+=1 + + if(regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-")))) { + if (i < objectMapper.readValue(regions.head.getOperators.head.opExecInitInfo.asInstanceOf[OpExecWithClassName].descString, classOf[LoopEndOpDesc]).iteration) { + currentLevel = loopStartLevel + i+=1 + } } + currentLevel += 1 regions } From 846aac2187ef8c5b1b9db0488970e29fadd57c0b Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 11 Feb 2026 17:15:05 -0800 Subject: [PATCH 14/55] update --- .../org/apache/texera/amber/operator/loop/LoopEndOpExec.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala index b1e6deace19..d3e736c0e3a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala @@ -21,9 +21,7 @@ package org.apache.texera.amber.operator.loop import org.apache.texera.amber.core.executor.OperatorExecutor import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} -import org.apache.texera.amber.util.JSONUtils.objectMapper - -class LoopEndOpExec extends OperatorExecutor { +class LoopEndOpExec(descString: String) extends OperatorExecutor { override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = Iterator(tuple) } From 6be7dc5bcf3b0aaf6df92e4eff59ddc03e2f9096 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 11 Feb 2026 18:35:07 -0800 Subject: [PATCH 15/55] update --- .../messaginglayer/AmberFIFOChannel.scala | 22 +++++++++---------- .../RegionExecutionCoordinator.scala | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala index d81b4239ba7..7917721c9c8 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala @@ -41,18 +41,18 @@ class AmberFIFOChannel(val channelId: ChannelIdentity) extends AmberLogging { private var portId: Option[PortIdentity] = None def acceptMessage(msg: WorkflowFIFOMessage): Unit = { - val seq = msg.sequenceNumber - val payload = msg.payload - if (isDuplicated(seq)) { - logger.debug( - s"received duplicated message $payload with seq = $seq while current seq = $current" - ) - } else if (isAhead(seq)) { - logger.debug(s"received ahead message $payload with seq = $seq while current seq = $current") - stash(seq, msg) - } else { + //val seq = msg.sequenceNumber + //val payload = msg.payload + //if (isDuplicated(seq)) { + // logger.debug( + // s"received duplicated message $payload with seq = $seq while current seq = $current" + // ) + //} else if (isAhead(seq)) { + // logger.debug(s"received ahead message $payload with seq = $seq while current seq = $current") + // stash(seq, msg) + //} else { enforceFIFO(msg) - } + //} } def getCurrentSeq: Long = current diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index 4b861de6574..2cddb29ba12 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@ -166,8 +166,8 @@ class RegionExecutionCoordinator( val actorRef = actorRefService.getActorRef(workerId) // Remove the actorRef so that no other actors can find the worker and send messages. actorRefService.removeActorRef(workerId) - asyncRPCClient.outputGateway.removeControlChannel(workerId) - asyncRPCClient.inputGateway.removeControlChannel(workerId) + //asyncRPCClient.outputGateway.removeControlChannel(workerId) + //asyncRPCClient.inputGateway.removeControlChannel(workerId) gracefulStop(actorRef, Duration(5, TimeUnit.SECONDS)).asTwitter() } }.toSeq From d8338d124197c1343ecb7b03cb9adf4e307effe3 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 11 Feb 2026 23:08:34 -0800 Subject: [PATCH 16/55] update --- .../messaginglayer/AmberFIFOChannel.scala | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala index 7917721c9c8..2556c55a36d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala @@ -41,18 +41,17 @@ class AmberFIFOChannel(val channelId: ChannelIdentity) extends AmberLogging { private var portId: Option[PortIdentity] = None def acceptMessage(msg: WorkflowFIFOMessage): Unit = { - //val seq = msg.sequenceNumber - //val payload = msg.payload - //if (isDuplicated(seq)) { - // logger.debug( - // s"received duplicated message $payload with seq = $seq while current seq = $current" - // ) - //} else if (isAhead(seq)) { - // logger.debug(s"received ahead message $payload with seq = $seq while current seq = $current") - // stash(seq, msg) - //} else { + //channel remove + val seq = msg.sequenceNumber + val payload = msg.payload + if (isDuplicated(seq)) { + logger.debug(s"received duplicated message $payload with seq = $seq while current seq = $current") + } else if (isAhead(seq)) { + logger.debug(s"received ahead message $payload with seq = $seq while current seq = $current") + stash(seq, msg) + } else { enforceFIFO(msg) - //} + } } def getCurrentSeq: Long = current From 36e517e463f171499cad3753c0fbc5497157cb0b Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Fri, 13 Feb 2026 13:21:28 -0800 Subject: [PATCH 17/55] fix --- .../architecture/scheduling/RegionExecutionCoordinator.scala | 4 ++-- .../worker/promisehandlers/AssignPortHandler.scala | 2 +- .../architecture/worker/promisehandlers/EndHandler.scala | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index 2cddb29ba12..4b861de6574 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@ -166,8 +166,8 @@ class RegionExecutionCoordinator( val actorRef = actorRefService.getActorRef(workerId) // Remove the actorRef so that no other actors can find the worker and send messages. actorRefService.removeActorRef(workerId) - //asyncRPCClient.outputGateway.removeControlChannel(workerId) - //asyncRPCClient.inputGateway.removeControlChannel(workerId) + asyncRPCClient.outputGateway.removeControlChannel(workerId) + asyncRPCClient.inputGateway.removeControlChannel(workerId) gracefulStop(actorRef, Duration(5, TimeUnit.SECONDS)).asTwitter() } }.toSeq diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala index fe959733abb..57a7782cf55 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala @@ -55,7 +55,7 @@ trait AssignPortHandler { // Same as AddInputChannelHandler dp.inputGateway.getChannel(channelId).setPortId(msg.portId) dp.inputManager.getPort(msg.portId).channels.add(channelId) - dp.stateManager.assertState(READY, RUNNING, PAUSED) + //dp.stateManager.assertState(READY, RUNNING, PAUSED) } } else { val storageURIOption: Option[URI] = msg.storageUris.head match { diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandler.scala index 2a6a20b3d3e..472eb09016f 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandler.scala @@ -49,7 +49,7 @@ trait EndHandler { s"${dp.inputManager.inputMessageQueue.peek()}" ) } - assert(dp.inputManager.inputMessageQueue.isEmpty) + //assert(dp.inputManager.inputMessageQueue.isEmpty) // Now we can safely acknowledge that this worker can be terminated. EmptyReturn() } From a4bfbdb2181d87947922ba8be390d023256a6979 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Fri, 13 Feb 2026 13:54:04 -0800 Subject: [PATCH 18/55] fix --- .../scheduling/RegionExecutionCoordinator.scala | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index 4b861de6574..318d3b2c602 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@ -536,7 +536,20 @@ class RegionExecutionCoordinator( region.getOperator(outputPortId.opId).outputPorts(outputPortId.portId)._3 val schema = schemaOptional.getOrElse(throw new IllegalStateException("Schema is missing")) - DocumentFactory.createDocument(storageUriToAdd, schema) + + + if (region.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-"))) { + try { + DocumentFactory.openDocument(storageUriToAdd) + } catch { + case _: Exception => + DocumentFactory.createDocument(storageUriToAdd, schema) + } + } + else { + DocumentFactory.createDocument(storageUriToAdd, schema) + } + WorkflowExecutionsResource.insertOperatorPortResultUri( eid = eid, globalPortId = outputPortId, From 084f602990c43f7cb60c63b13a70b04d0d883314 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 14 Feb 2026 19:46:25 -0800 Subject: [PATCH 19/55] update --- .../messaginglayer/AmberFIFOChannel.scala | 4 ++- .../RegionExecutionCoordinator.scala | 21 +++++++------- .../architecture/scheduling/Schedule.scala | 21 +++++++++++--- .../promisehandlers/AssignPortHandler.scala | 2 +- .../texera/amber/operator/LogicalOp.scala | 28 +++++++++++++++---- .../amber/operator/loop/LoopEndOpDesc.scala | 5 +++- .../amber/operator/loop/LoopStartOpExec.scala | 2 -- 7 files changed, 58 insertions(+), 25 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala index 2556c55a36d..b7611c8f8f9 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/AmberFIFOChannel.scala @@ -45,7 +45,9 @@ class AmberFIFOChannel(val channelId: ChannelIdentity) extends AmberLogging { val seq = msg.sequenceNumber val payload = msg.payload if (isDuplicated(seq)) { - logger.debug(s"received duplicated message $payload with seq = $seq while current seq = $current") + logger.debug( + s"received duplicated message $payload with seq = $seq while current seq = $current" + ) } else if (isAhead(seq)) { logger.debug(s"received ahead message $payload with seq = $seq while current seq = $current") stash(seq, msg) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index 0f105410972..5cfbc593917 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@ -210,14 +210,15 @@ class RegionExecutionCoordinator( regionExecution: RegionExecution, attempt: Int = 1 ): Future[Unit] = { - terminateWorkers(regionExecution).rescue { case err => - logger.warn( - s"Failed to terminate region ${region.id.id} on attempt $attempt. Retrying in ${killRetryDelay.inMilliseconds} ms.", - err - ) - Future - .sleep(killRetryDelay)(killRetryTimer) - .flatMap(_ => terminateWorkersWithRetry(regionExecution, attempt + 1)) + terminateWorkers(regionExecution).rescue { + case err => + logger.warn( + s"Failed to terminate region ${region.id.id} on attempt $attempt. Retrying in ${killRetryDelay.inMilliseconds} ms.", + err + ) + Future + .sleep(killRetryDelay)(killRetryTimer) + .flatMap(_ => terminateWorkersWithRetry(regionExecution, attempt + 1)) } } @@ -572,7 +573,6 @@ class RegionExecutionCoordinator( val schema = schemaOptional.getOrElse(throw new IllegalStateException("Schema is missing")) - if (region.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-"))) { try { DocumentFactory.openDocument(storageUriToAdd) @@ -580,8 +580,7 @@ class RegionExecutionCoordinator( case _: Exception => DocumentFactory.createDocument(storageUriToAdd, schema) } - } - else { + } else { DocumentFactory.createDocument(storageUriToAdd, schema) } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala index 20f29942218..93d96d640ac 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala @@ -34,7 +34,9 @@ case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterat override def next(): Set[Region] = { val regions = levelSets(currentLevel) - if(regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopStart-operator-")))) loopStartLevel = currentLevel + if ( + regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopStart-operator-"))) + ) loopStartLevel = currentLevel currentLevel += 1 regions } @@ -42,10 +44,21 @@ case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterat def loopNext(): Set[Region] = { val regions = levelSets(currentLevel) - if(regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-")))) { - if (i < objectMapper.readValue(regions.head.getOperators.head.opExecInitInfo.asInstanceOf[OpExecWithClassName].descString, classOf[LoopEndOpDesc]).iteration) { + if ( + regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-"))) + ) { + if ( + i < objectMapper + .readValue( + regions.head.getOperators.head.opExecInitInfo + .asInstanceOf[OpExecWithClassName] + .descString, + classOf[LoopEndOpDesc] + ) + .iteration + ) { currentLevel = loopStartLevel - i+=1 + i += 1 } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala index 57a7782cf55..3fb489e1055 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala @@ -55,7 +55,7 @@ trait AssignPortHandler { // Same as AddInputChannelHandler dp.inputGateway.getChannel(channelId).setPortId(msg.portId) dp.inputManager.getPort(msg.portId).channels.add(channelId) - //dp.stateManager.assertState(READY, RUNNING, PAUSED) + //dp.stateManager.assertState(READY, RUNNING, PAUSED) } } else { val storageURIOption: Option[URI] = msg.storageUris.head match { diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala index ee57514212b..1b92674864c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala @@ -24,8 +24,15 @@ import com.fasterxml.jackson.annotation._ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OperatorExecutor import org.apache.texera.amber.core.tuple.Schema -import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, OperatorIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.WorkflowContext.{DEFAULT_EXECUTION_ID, DEFAULT_WORKFLOW_ID} +import org.apache.texera.amber.core.virtualidentity.{ + ExecutionIdentity, + OperatorIdentity, + WorkflowIdentity +} +import org.apache.texera.amber.core.workflow.WorkflowContext.{ + DEFAULT_EXECUTION_ID, + DEFAULT_WORKFLOW_ID +} import org.apache.texera.amber.core.workflow.{PhysicalOp, PhysicalPlan, PortIdentity} import org.apache.texera.amber.operator.aggregate.AggregateOpDesc import org.apache.texera.amber.operator.cartesianProduct.CartesianProductOpDesc @@ -35,14 +42,22 @@ import org.apache.texera.amber.operator.distinct.DistinctOpDesc import org.apache.texera.amber.operator.dummy.DummyOpDesc import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc -import org.apache.texera.amber.operator.huggingFace.{HuggingFaceIrisLogisticRegressionOpDesc, HuggingFaceSentimentAnalysisOpDesc, HuggingFaceSpamSMSDetectionOpDesc, HuggingFaceTextSummarizationOpDesc} +import org.apache.texera.amber.operator.huggingFace.{ + HuggingFaceIrisLogisticRegressionOpDesc, + HuggingFaceSentimentAnalysisOpDesc, + HuggingFaceSpamSMSDetectionOpDesc, + HuggingFaceTextSummarizationOpDesc +} import org.apache.texera.amber.operator.ifStatement.IfOpDesc import org.apache.texera.amber.operator.intersect.IntersectOpDesc import org.apache.texera.amber.operator.intervalJoin.IntervalJoinOpDesc import org.apache.texera.amber.operator.keywordSearch.KeywordSearchOpDesc import org.apache.texera.amber.operator.limit.LimitOpDesc import org.apache.texera.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc -import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{SklearnAdvancedKNNClassifierTrainerOpDesc, SklearnAdvancedKNNRegressorTrainerOpDesc} +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{ + SklearnAdvancedKNNClassifierTrainerOpDesc, + SklearnAdvancedKNNRegressorTrainerOpDesc +} import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVRTrainer.SklearnAdvancedSVRTrainerOpDesc import org.apache.texera.amber.operator.metadata.{OPVersion, OperatorInfo, PropertyNameConstants} @@ -56,7 +71,10 @@ import org.apache.texera.amber.operator.sleep.SleepOpDesc import org.apache.texera.amber.operator.sort.{SortOpDesc, StableMergeSortOpDesc} import org.apache.texera.amber.operator.sortPartitions.SortPartitionsOpDesc import org.apache.texera.amber.operator.source.apis.reddit.RedditSearchSourceOpDesc -import org.apache.texera.amber.operator.source.apis.twitter.v2.{TwitterFullArchiveSearchSourceOpDesc, TwitterSearchSourceOpDesc} +import org.apache.texera.amber.operator.source.apis.twitter.v2.{ + TwitterFullArchiveSearchSourceOpDesc, + TwitterSearchSourceOpDesc +} import org.apache.texera.amber.operator.source.fetcher.URLFetcherOpDesc import org.apache.texera.amber.operator.source.scan.FileScanSourceOpDesc import org.apache.texera.amber.operator.source.scan.arrow.ArrowSourceOpDesc diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index 31719229f0b..8288db6aa7f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -43,7 +43,10 @@ class LoopEndOpDesc extends LogicalOp { workflowId, executionId, operatorIdentifier, - OpExecWithClassName("org.apache.texera.amber.operator.loop.LoopEndOpExec",objectMapper.writeValueAsString(this)) + OpExecWithClassName( + "org.apache.texera.amber.operator.loop.LoopEndOpExec", + objectMapper.writeValueAsString(this) + ) ) .withInputPorts(operatorInfo.inputPorts) .withOutputPorts(operatorInfo.outputPorts) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala index f15d90a6cca..1d0c4cbe9e2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala @@ -24,5 +24,3 @@ import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} class LoopStartOpExec(descString: String) extends OperatorExecutor { override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = Iterator(tuple) } - - From 55d5cec6fb7cb890b87df278ce113d4e2f24016e Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 14 Feb 2026 20:43:38 -0800 Subject: [PATCH 20/55] update --- .../amber/engine/architecture/scheduling/Schedule.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala index 93d96d640ac..435096cda1f 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala @@ -34,16 +34,15 @@ case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterat override def next(): Set[Region] = { val regions = levelSets(currentLevel) - if ( - regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopStart-operator-"))) - ) loopStartLevel = currentLevel currentLevel += 1 regions } def loopNext(): Set[Region] = { val regions = levelSets(currentLevel) - + if ( + regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopStart-operator-"))) + ) loopStartLevel = currentLevel - 1 if ( regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-"))) ) { From b8faf93e5adaff8b4c1aea5b7c681426ab63edaf Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 15 Feb 2026 00:45:20 -0800 Subject: [PATCH 21/55] update --- .../architecture/scheduling/RegionExecutionCoordinator.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index 5cfbc593917..cd1541aaf7e 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@ -180,8 +180,8 @@ class RegionExecutionCoordinator( val actorRef = actorRefService.getActorRef(workerId) // Remove the actorRef so that no other actors can find the worker and send messages. actorRefService.removeActorRef(workerId) - asyncRPCClient.outputGateway.removeControlChannel(workerId) asyncRPCClient.inputGateway.removeControlChannel(workerId) + asyncRPCClient.outputGateway.removeControlChannel(workerId) gracefulStop(actorRef, ScalaDuration(5, TimeUnit.SECONDS)).asTwitter() } }.toSeq @@ -210,8 +210,7 @@ class RegionExecutionCoordinator( regionExecution: RegionExecution, attempt: Int = 1 ): Future[Unit] = { - terminateWorkers(regionExecution).rescue { - case err => + terminateWorkers(regionExecution).rescue { case err => logger.warn( s"Failed to terminate region ${region.id.id} on attempt $attempt. Retrying in ${killRetryDelay.inMilliseconds} ms.", err From a53506afca890849710a601e1df21ba1a54ba6f3 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 15 Feb 2026 00:45:45 -0800 Subject: [PATCH 22/55] update --- .../architecture/worker/promisehandlers/AssignPortHandler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala index 3fb489e1055..fe959733abb 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/AssignPortHandler.scala @@ -55,7 +55,7 @@ trait AssignPortHandler { // Same as AddInputChannelHandler dp.inputGateway.getChannel(channelId).setPortId(msg.portId) dp.inputManager.getPort(msg.portId).channels.add(channelId) - //dp.stateManager.assertState(READY, RUNNING, PAUSED) + dp.stateManager.assertState(READY, RUNNING, PAUSED) } } else { val storageURIOption: Option[URI] = msg.storageUris.head match { From d44a664f9f81448d2f13ded582df0061c6210c3a Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 15 Feb 2026 00:46:33 -0800 Subject: [PATCH 23/55] update --- .../controller/execution/WorkflowExecution.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala index 6071a4dc504..2de29f31fdd 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala @@ -45,6 +45,11 @@ case class WorkflowExecution() { */ def initRegionExecution(region: Region): RegionExecution = { regionExecutions.remove(region.id) + // ensure the region execution hasn't been initialized already. + assert( + !regionExecutions.contains(region.id), + s"RegionExecution of ${region.id} already initialized." + ) regionExecutions.getOrElseUpdate(region.id, RegionExecution(region)) } From 1cd48fd12b88ab599b6262fb4b98c0a0baad487b Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 15 Feb 2026 00:47:24 -0800 Subject: [PATCH 24/55] update --- .../scheduling/RegionExecutionCoordinator.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index cd1541aaf7e..a6b7e173ccb 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@ -211,13 +211,13 @@ class RegionExecutionCoordinator( attempt: Int = 1 ): Future[Unit] = { terminateWorkers(regionExecution).rescue { case err => - logger.warn( - s"Failed to terminate region ${region.id.id} on attempt $attempt. Retrying in ${killRetryDelay.inMilliseconds} ms.", - err - ) - Future - .sleep(killRetryDelay)(killRetryTimer) - .flatMap(_ => terminateWorkersWithRetry(regionExecution, attempt + 1)) + logger.warn( + s"Failed to terminate region ${region.id.id} on attempt $attempt. Retrying in ${killRetryDelay.inMilliseconds} ms.", + err + ) + Future + .sleep(killRetryDelay)(killRetryTimer) + .flatMap(_ => terminateWorkersWithRetry(regionExecution, attempt + 1)) } } From e35a33286c52b28749cda6a8911ca4db47726142 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 15 Feb 2026 16:21:38 -0800 Subject: [PATCH 25/55] update --- .../architecture/scheduling/Schedule.scala | 27 ++++++++++--------- .../amber/operator/loop/LoopEndOpDesc.scala | 7 ----- .../amber/operator/loop/LoopStartOpDesc.scala | 3 +++ 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala index 435096cda1f..2f067437466 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala @@ -20,12 +20,13 @@ package org.apache.texera.amber.engine.architecture.scheduling import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.operator.loop.LoopEndOpDesc +import org.apache.texera.amber.operator.loop.{LoopEndOpDesc, LoopStartOpDesc} import org.apache.texera.amber.util.JSONUtils.objectMapper case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterator[Set[Region]] { private var currentLevel = levelSets.keys.minOption.getOrElse(0) private var loopStartLevel = currentLevel + private var iteration = 1 private var i = 1 def getRegions: List[Region] = levelSets.values.flatten.toList @@ -42,25 +43,25 @@ case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterat val regions = levelSets(currentLevel) if ( regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopStart-operator-"))) - ) loopStartLevel = currentLevel - 1 + ) { + iteration = objectMapper + .readValue( + regions.head.getOperators.head.opExecInitInfo + .asInstanceOf[OpExecWithClassName] + .descString, + classOf[LoopStartOpDesc] + ) + .iteration + loopStartLevel = currentLevel - 1 + } if ( regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopEnd-operator-"))) ) { - if ( - i < objectMapper - .readValue( - regions.head.getOperators.head.opExecInitInfo - .asInstanceOf[OpExecWithClassName] - .descString, - classOf[LoopEndOpDesc] - ) - .iteration - ) { + if (i < iteration) { currentLevel = loopStartLevel i += 1 } } - currentLevel += 1 regions } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index 8288db6aa7f..08330db5841 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -19,8 +19,6 @@ package org.apache.texera.amber.operator.loop -import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} -import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} @@ -29,11 +27,6 @@ import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, Operat import org.apache.texera.amber.util.JSONUtils.objectMapper class LoopEndOpDesc extends LogicalOp { - - @JsonProperty(required = true) - @JsonSchemaTitle("Iteration Number") - var iteration: Int = _ - override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala index 4f3e86eb624..473db52584a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala @@ -29,6 +29,9 @@ import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, Operat import org.apache.texera.amber.util.JSONUtils.objectMapper class LoopStartOpDesc extends LogicalOp { + @JsonProperty(required = true) + @JsonSchemaTitle("Iteration Number") + var iteration: Int = _ override def getPhysicalOp( workflowId: WorkflowIdentity, From 30a85623a7abb8b30411d79a05b790ceaba59d05 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 24 Feb 2026 14:26:48 -0800 Subject: [PATCH 26/55] update --- .../architecture/rpc/controlcommands.proto | 6 +++ .../architecture/rpc/workerservice.proto | 1 + .../messaginglayer/OutputManager.scala | 37 ++++++++++++++----- .../core/storage/result/ResultSchema.scala | 5 +++ 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto index ed17be236dc..f73f8707d6f 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto @@ -57,6 +57,7 @@ message ControlRequest { EmptyRequest emptyRequest = 56; PrepareCheckpointRequest prepareCheckpointRequest = 57; QueryStatisticsRequest queryStatisticsRequest = 58; + EndIterationRequest endIterationRequest = 59; // request for testing Ping ping = 100; @@ -271,4 +272,9 @@ message PrepareCheckpointRequest{ message QueryStatisticsRequest{ repeated core.ActorVirtualIdentity filterByWorkers = 1; +} + +message EndIterationRequest{ + core.ActorVirtualIdentity LoopStartId = 1 [(scalapb.field).no_box = true]; + int32 iteration = 2; } \ No newline at end of file diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto index dbcd6d8a5e0..a2c4d6ef9b7 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto @@ -47,6 +47,7 @@ service WorkerService { rpc EndWorker(EmptyRequest) returns (EmptyReturn); rpc StartChannel(EmptyRequest) returns (EmptyReturn); rpc EndChannel(EmptyRequest) returns (EmptyReturn); + rpc EndIteration(EndIterationRequest) returns (EmptyReturn); rpc DebugCommand(DebugCommandRequest) returns (EmptyReturn); rpc EvaluatePythonExpression(EvaluatePythonExpressionRequest) returns (EvaluatedValue); rpc NoOperation(EmptyRequest) returns (EmptyReturn); diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala index 4ab3d18056f..202d83d1638 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala @@ -22,20 +22,14 @@ package org.apache.texera.amber.engine.architecture.messaginglayer import org.apache.texera.amber.core.state.State import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.storage.model.BufferedItemWriter +import org.apache.texera.amber.core.storage.result.ResultSchema import org.apache.texera.amber.core.tuple._ import org.apache.texera.amber.core.virtualidentity.{ActorVirtualIdentity, ChannelIdentity} import org.apache.texera.amber.core.workflow.{PhysicalLink, PortIdentity} -import org.apache.texera.amber.engine.architecture.messaginglayer.OutputManager.{ - DPOutputIterator, - getBatchSize, - toPartitioner -} +import org.apache.texera.amber.engine.architecture.messaginglayer.OutputManager.{DPOutputIterator, getBatchSize, toPartitioner} import org.apache.texera.amber.engine.architecture.sendsemantics.partitioners._ import org.apache.texera.amber.engine.architecture.sendsemantics.partitionings._ -import org.apache.texera.amber.engine.architecture.worker.managers.{ - OutputPortResultWriterThread, - PortStorageWriterTerminateSignal -} +import org.apache.texera.amber.engine.architecture.worker.managers.{OutputPortResultWriterThread, PortStorageWriterTerminateSignal} import org.apache.texera.amber.engine.common.AmberLogging import org.apache.texera.amber.util.VirtualIdentityUtils @@ -124,6 +118,10 @@ class OutputManager( : mutable.HashMap[PortIdentity, OutputPortResultWriterThread] = mutable.HashMap() + private val ECMWriters + : mutable.HashMap[PortIdentity, BufferedItemWriter[Tuple]] = + mutable.HashMap() + /** * Add down stream operator and its corresponding Partitioner. * @@ -232,6 +230,23 @@ class OutputManager( }) } + def saveECMToStorageIfNeeded( + tuple: Tuple, + outputPortId: Option[PortIdentity] = None + ): Unit = { + (outputPortId match { + case Some(portId) => + this.ECMWriters.get(portId) match { + case Some(_) => this.ECMWriters.filter(_._1 == portId) + case None => Map.empty + } + case None => this.ECMWriters + }).foreach({ + case (portId, writer) => + writer.putOne(new Tuple(ResultSchema.ecmSchema, Array(worker.name))) + }) + } + /** * Singal the port storage writer to flush the remaining buffer and wait for commits to finish so that * the output port is properly completed. If the output port does not need storage, no action will be done. @@ -280,6 +295,10 @@ class OutputManager( } private def setupOutputStorageWriterThread(portId: PortIdentity, storageUri: URI): Unit = { + this.ECMWriters(portId) = DocumentFactory + .createDocument(storageUri.resolve("ecm"), ResultSchema.ecmSchema) + .writer(VirtualIdentityUtils.getWorkerIndex(actorId).toString) + .asInstanceOf[BufferedItemWriter[Tuple]] val bufferedItemWriter = DocumentFactory .openDocument(storageUri) ._1 diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/ResultSchema.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/ResultSchema.scala index ade33283f7f..4be34b856d6 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/ResultSchema.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/ResultSchema.scala @@ -39,4 +39,9 @@ object ResultSchema { val consoleMessagesSchema: Schema = new Schema( new Attribute("message", AttributeType.STRING) ) + + val ecmSchema: Schema = new Schema( + new Attribute("LoopStartId", AttributeType.STRING), + new Attribute("iteration", AttributeType.INTEGER) + ) } From b717fb0d46d6c2b82e49b2c4f356bd9484add129 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 24 Feb 2026 15:36:36 -0800 Subject: [PATCH 27/55] update --- .../storage/result/iceberg/IcebergTableWriter.scala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala index 549cb4b9d17..e5e2a5719d2 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala @@ -29,7 +29,7 @@ import org.apache.iceberg.io.{DataWriter, OutputFile} import org.apache.iceberg.parquet.Parquet import org.apache.iceberg.{Schema, Table} -import java.nio.file.Paths +import java.nio.file.{Files, Path, Paths} import scala.collection.mutable.ArrayBuffer /** @@ -107,9 +107,12 @@ private[storage] class IcebergTableWriter[T]( private def flushBuffer(): Unit = { if (buffer.nonEmpty) { // Create a unique file path using the writer's identifier and the filename index - val filepath = Paths.get(table.location()).resolve(s"${writerIdentifier}_${filenameIdx}") - // Increment the filename index by 1 - filenameIdx += 1 + var filepath: Path = null + do { + filepath = Paths.get(table.location()).resolve(s"${writerIdentifier}_$filenameIdx") + filenameIdx+= 1 + } while (Files.exists(filepath)) + val outputFile: OutputFile = table.io().newOutputFile(filepath.toString) // Create a Parquet data writer to write a new file val dataWriter: DataWriter[Record] = Parquet From 04fe614bf0d8c377357fc6a806220275d0709af2 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 26 Feb 2026 21:11:54 -0800 Subject: [PATCH 28/55] update --- .../messaginglayer/OutputManager.scala | 2 +- .../DataProcessorRPCHandlerInitializer.scala | 1 + .../promisehandlers/EndIterationHandler.scala | 44 +++++++++++ .../operator/loop/LoopStartv2OpDesc.scala | 77 +++++++++++++++++++ 4 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala create mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartv2OpDesc.scala diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala index 202d83d1638..8f0c0f6feef 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala @@ -243,7 +243,7 @@ class OutputManager( case None => this.ECMWriters }).foreach({ case (portId, writer) => - writer.putOne(new Tuple(ResultSchema.ecmSchema, Array(worker.name))) + writer.putOne(new Tuple(ResultSchema.ecmSchema, Array("erge"))) }) } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala index 2abcdf66975..a49860c1e11 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala @@ -45,6 +45,7 @@ class DataProcessorRPCHandlerInitializer(val dp: DataProcessor) with ResumeHandler with StartHandler with EndHandler + with EndIterationHandler with StartChannelHandler with EndChannelHandler with AssignPortHandler diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala new file mode 100644 index 00000000000..e1827c7e949 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.engine.architecture.worker.promisehandlers + +import com.twitter.util.Future +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, EmptyRequest, EndIterationRequest} +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn +import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer +import org.apache.texera.amber.operator.loop.LoopEndOpExec + +trait EndIterationHandler { + this: DataProcessorRPCHandlerInitializer => + + override def endIteration( + request: EndIterationRequest, + ctx: AsyncRPCContext + ): Future[EmptyReturn] = { + dp.executor match { + case _: LoopEndOpExec => + //workerInterface.nextIteration(EmptyRequest(), mkContext(request.worker)) + case _ => + //dp.processOnFinish() + //dp.outputManager.finalizeIteration(request.worker) + } + EmptyReturn() + } +} \ No newline at end of file diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartv2OpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartv2OpDesc.scala new file mode 100644 index 00000000000..5f5050b384a --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartv2OpDesc.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.loop + +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +import org.apache.texera.amber.core.executor.OpExecWithCode +import org.apache.texera.amber.core.tuple.Schema +import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp, PortIdentity, SchemaPropagationFunc} +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} + +class LoopStartv2OpDesc extends LogicalOp { + override def getPhysicalOp( + workflowId: WorkflowIdentity, + executionId: ExecutionIdentity + ): PhysicalOp = { + val pythonCode = + try { + generatePythonCode() + } catch { + case ex: Throwable => + s"#EXCEPTION DURING CODE GENERATION: ${ex.getMessage}" + } + PhysicalOp.oneToOnePhysicalOp( + workflowId, + executionId, + operatorIdentifier, + OpExecWithCode(pythonCode, "python") + ) + .withInputPorts(operatorInfo.inputPorts) + .withOutputPorts(operatorInfo.outputPorts) + .withSuggestedWorkerNum(1) + .withParallelizable(false) + } + + @JsonSchemaTitle("Output columns") + var lambdaAttributeUnits: List[LambdaAttributeUnit] = List() + + + override def operatorInfo: OperatorInfo = + OperatorInfo( + "Python Table Reducer", + "Reduce Table to Tuple", + OperatorGroupConstants.PYTHON_GROUP, + inputPorts = List(InputPort()), + outputPorts = List(OutputPort()) + ) + + def generatePythonCode(): String = { + s""" + |from pytexera import * + |class ProcessTableOperator(UDFTableOperator): + | + | @overrides + | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: + | yield table + |""".stripMargin + } +} From bd270311c15aeb4761dcd3c51d8a1ff223b42643 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Fri, 27 Feb 2026 21:38:45 -0800 Subject: [PATCH 29/55] update --- .../texera/amber/engine/architecture/rpc/controlcommands.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto index 4f72f216bad..efff6948a00 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto @@ -278,10 +278,10 @@ enum StatisticsUpdateTarget { message QueryStatisticsRequest{ repeated core.ActorVirtualIdentity filterByWorkers = 1; + StatisticsUpdateTarget updateTarget = 2; } message EndIterationRequest{ core.ActorVirtualIdentity LoopStartId = 1 [(scalapb.field).no_box = true]; int32 iteration = 2; - StatisticsUpdateTarget updateTarget = 2; } \ No newline at end of file From 99e0f86c2eefc390a7bc2369b9be5ed0bff0b12d Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 28 Feb 2026 00:45:22 -0800 Subject: [PATCH 30/55] update --- .../org/apache/texera/amber/core/storage/DocumentFactory.scala | 1 + .../org/apache/texera/amber/core/storage/VFSURIFactory.scala | 1 + 2 files changed, 2 insertions(+) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala index 15949ef4717..683e65786a8 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala @@ -72,6 +72,7 @@ object DocumentFactory { case RESULT => StorageConfig.icebergTableResultNamespace case CONSOLE_MESSAGES => StorageConfig.icebergTableConsoleMessagesNamespace case RUNTIME_STATISTICS => StorageConfig.icebergTableRuntimeStatisticsNamespace + case ECM => "ECM" case _ => throw new IllegalArgumentException(s"Resource type $resourceType is not supported") } diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/VFSURIFactory.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/VFSURIFactory.scala index 3513ac5ecd8..f4f3a9e4bf1 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/VFSURIFactory.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/VFSURIFactory.scala @@ -34,6 +34,7 @@ object VFSResourceType extends Enumeration { val RESULT: Value = Value("result") val RUNTIME_STATISTICS: Value = Value("runtimeStatistics") val CONSOLE_MESSAGES: Value = Value("consoleMessages") + val ECM: Value = Value("ecm") } object VFSURIFactory { From 53ae08be556fe7752091a8370c7ce71beac5e69f Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 28 Feb 2026 04:06:50 -0800 Subject: [PATCH 31/55] update --- .../amber/operator/loop/LoopEndOpDesc.scala | 45 ++++++++--- .../amber/operator/loop/LoopEndOpExec.scala | 27 ------- .../amber/operator/loop/LoopStartOpDesc.scala | 46 ++++++++--- .../amber/operator/loop/LoopStartOpExec.scala | 26 ------- .../operator/loop/LoopStartv2OpDesc.scala | 77 ------------------- 5 files changed, 71 insertions(+), 150 deletions(-) delete mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala delete mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala delete mode 100644 common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartv2OpDesc.scala diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index 08330db5841..9e1031aa4a5 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -19,32 +19,44 @@ package org.apache.texera.amber.operator.loop -import org.apache.texera.amber.core.executor.OpExecWithClassName +import com.fasterxml.jackson.annotation.JsonProperty +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +import org.apache.texera.amber.core.executor.OpExecWithCode import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.util.JSONUtils.objectMapper class LoopEndOpDesc extends LogicalOp { + @JsonProperty(required = true, defaultValue = "i += 1") + @JsonSchemaTitle("Update") + var update: String = _ + + @JsonProperty(required = true, defaultValue = "i < len(table)") + @JsonSchemaTitle("Condition") + var condition: String = _ + override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity ): PhysicalOp = { - PhysicalOp - .oneToOnePhysicalOp( + val pythonCode = + try { + generatePythonCode() + } catch { + case ex: Throwable => + s"#EXCEPTION DURING CODE GENERATION: ${ex.getMessage}" + } + PhysicalOp.oneToOnePhysicalOp( workflowId, executionId, operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.loop.LoopEndOpExec", - objectMapper.writeValueAsString(this) - ) + OpExecWithCode(pythonCode, "python") ) .withInputPorts(operatorInfo.inputPorts) .withOutputPorts(operatorInfo.outputPorts) - .withParallelizable(false) .withSuggestedWorkerNum(1) + .withParallelizable(false) } override def operatorInfo: OperatorInfo = @@ -55,4 +67,19 @@ class LoopEndOpDesc extends LogicalOp { inputPorts = List(InputPort()), outputPorts = List(OutputPort()) ) + + def generatePythonCode(): String = { + s""" + |from pytexera import * + |class ProcessTableOperator(UDFTableOperator): + | @overrides + | def loop_condition_check(self) -> bool: + | $update + | return $condition + | + | @overrides + | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: + | yield table + |""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala deleted file mode 100644 index d3e736c0e3a..00000000000 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpExec.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.operator.loop - -import org.apache.texera.amber.core.executor.OperatorExecutor -import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} - -class LoopEndOpExec(descString: String) extends OperatorExecutor { - override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = Iterator(tuple) -} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala index 473db52584a..e077e412656 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala @@ -21,31 +21,41 @@ package org.apache.texera.amber.operator.loop import com.fasterxml.jackson.annotation.JsonProperty import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle -import org.apache.texera.amber.core.executor.OpExecWithClassName +import org.apache.texera.amber.core.executor.OpExecWithCode import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.util.JSONUtils.objectMapper class LoopStartOpDesc extends LogicalOp { - @JsonProperty(required = true) - @JsonSchemaTitle("Iteration Number") - var iteration: Int = _ + @JsonProperty(required = true, defaultValue = "i") + @JsonSchemaTitle("Variable") + var variable: String = _ + + @JsonProperty(required = true, defaultValue = "i = 0") + @JsonSchemaTitle("Initialization") + var initialization: String = _ + + @JsonProperty(required = true, defaultValue = "table.iloc[0]") + @JsonSchemaTitle("Output") + var output: String = _ override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity ): PhysicalOp = { - PhysicalOp - .oneToOnePhysicalOp( + val pythonCode = + try { + generatePythonCode() + } catch { + case ex: Throwable => + s"#EXCEPTION DURING CODE GENERATION: ${ex.getMessage}" + } + PhysicalOp.oneToOnePhysicalOp( workflowId, executionId, operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.loop.LoopStartOpExec", - objectMapper.writeValueAsString(this) - ) + OpExecWithCode(pythonCode, "python") ) .withInputPorts(operatorInfo.inputPorts) .withOutputPorts(operatorInfo.outputPorts) @@ -62,4 +72,18 @@ class LoopStartOpDesc extends LogicalOp { outputPorts = List(OutputPort()) ) + def generatePythonCode(): String = { + s""" + |from pytexera import * + |class ProcessLoopStartOperator(LoopStartOperator): + | @overrides + | def loop_initialization(self): + | $initialization + | return "$variable",$variable + | @overrides + | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: + | $initialization + | yield $output + |""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala deleted file mode 100644 index 1d0c4cbe9e2..00000000000 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpExec.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.operator.loop -import org.apache.texera.amber.core.executor.OperatorExecutor -import org.apache.texera.amber.core.tuple.{Tuple, TupleLike} - -class LoopStartOpExec(descString: String) extends OperatorExecutor { - override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = Iterator(tuple) -} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartv2OpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartv2OpDesc.scala deleted file mode 100644 index 5f5050b384a..00000000000 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartv2OpDesc.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.operator.loop - -import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle -import org.apache.texera.amber.core.executor.OpExecWithCode -import org.apache.texera.amber.core.tuple.Schema -import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp, PortIdentity, SchemaPropagationFunc} -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} - -class LoopStartv2OpDesc extends LogicalOp { - override def getPhysicalOp( - workflowId: WorkflowIdentity, - executionId: ExecutionIdentity - ): PhysicalOp = { - val pythonCode = - try { - generatePythonCode() - } catch { - case ex: Throwable => - s"#EXCEPTION DURING CODE GENERATION: ${ex.getMessage}" - } - PhysicalOp.oneToOnePhysicalOp( - workflowId, - executionId, - operatorIdentifier, - OpExecWithCode(pythonCode, "python") - ) - .withInputPorts(operatorInfo.inputPorts) - .withOutputPorts(operatorInfo.outputPorts) - .withSuggestedWorkerNum(1) - .withParallelizable(false) - } - - @JsonSchemaTitle("Output columns") - var lambdaAttributeUnits: List[LambdaAttributeUnit] = List() - - - override def operatorInfo: OperatorInfo = - OperatorInfo( - "Python Table Reducer", - "Reduce Table to Tuple", - OperatorGroupConstants.PYTHON_GROUP, - inputPorts = List(InputPort()), - outputPorts = List(OutputPort()) - ) - - def generatePythonCode(): String = { - s""" - |from pytexera import * - |class ProcessTableOperator(UDFTableOperator): - | - | @overrides - | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | yield table - |""".stripMargin - } -} From 8c7d53cc104be299c8759ce0448596fc5744739d Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 28 Feb 2026 15:59:44 -0800 Subject: [PATCH 32/55] update --- .../apache/texera/amber/operator/loop/LoopEndOpDesc.scala | 6 +----- .../apache/texera/amber/operator/loop/LoopStartOpDesc.scala | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index 9e1031aa4a5..3dc0dc81f90 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -71,15 +71,11 @@ class LoopEndOpDesc extends LogicalOp { def generatePythonCode(): String = { s""" |from pytexera import * - |class ProcessTableOperator(UDFTableOperator): + |class ProcessLoopEndOperator(LoopEndOperator): | @overrides | def loop_condition_check(self) -> bool: | $update | return $condition - | - | @overrides - | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | yield table |""".stripMargin } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala index e077e412656..1c94102682b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala @@ -80,6 +80,7 @@ class LoopStartOpDesc extends LogicalOp { | def loop_initialization(self): | $initialization | return "$variable",$variable + | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | $initialization From 92ab10f56623bab1cda53b03b44ea46db29971d8 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 28 Feb 2026 17:14:24 -0800 Subject: [PATCH 33/55] update --- .../engine/architecture/scheduling/Schedule.scala | 11 ++--------- .../texera/amber/operator/loop/LoopEndOpDesc.scala | 3 ++- .../texera/amber/operator/loop/LoopStartOpDesc.scala | 3 ++- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala index 2f067437466..b891577ff8c 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala @@ -26,7 +26,7 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterator[Set[Region]] { private var currentLevel = levelSets.keys.minOption.getOrElse(0) private var loopStartLevel = currentLevel - private var iteration = 1 + private var iteration = 10 private var i = 1 def getRegions: List[Region] = levelSets.values.flatten.toList @@ -44,14 +44,7 @@ case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterat if ( regions.exists(_.getOperators.exists(_.id.logicalOpId.id.startsWith("LoopStart-operator-"))) ) { - iteration = objectMapper - .readValue( - regions.head.getOperators.head.opExecInitInfo - .asInstanceOf[OpExecWithClassName] - .descString, - classOf[LoopStartOpDesc] - ) - .iteration + //iteration = objectMapper.readValue(regions.head.getOperators.head.opExecInitInfo.asInstanceOf[OpExecWithClassName].descString, classOf[LoopStartOpDesc]).iteration loopStartLevel = currentLevel - 1 } if ( diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index 3dc0dc81f90..294028d2666 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -47,7 +47,8 @@ class LoopEndOpDesc extends LogicalOp { case ex: Throwable => s"#EXCEPTION DURING CODE GENERATION: ${ex.getMessage}" } - PhysicalOp.oneToOnePhysicalOp( + PhysicalOp + .oneToOnePhysicalOp( workflowId, executionId, operatorIdentifier, diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala index 1c94102682b..2b34251bb6d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala @@ -51,7 +51,8 @@ class LoopStartOpDesc extends LogicalOp { case ex: Throwable => s"#EXCEPTION DURING CODE GENERATION: ${ex.getMessage}" } - PhysicalOp.oneToOnePhysicalOp( + PhysicalOp + .oneToOnePhysicalOp( workflowId, executionId, operatorIdentifier, From bc5856643b3903136020a6e2a2ca09426690e495 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 28 Feb 2026 17:14:49 -0800 Subject: [PATCH 34/55] update --- .../amber/core/storage/result/iceberg/IcebergTableWriter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala index e5e2a5719d2..25b6df58001 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/iceberg/IcebergTableWriter.scala @@ -110,7 +110,7 @@ private[storage] class IcebergTableWriter[T]( var filepath: Path = null do { filepath = Paths.get(table.location()).resolve(s"${writerIdentifier}_$filenameIdx") - filenameIdx+= 1 + filenameIdx += 1 } while (Files.exists(filepath)) val outputFile: OutputFile = table.io().newOutputFile(filepath.toString) From c65585650aba5b729de7ab40e7849514ff5fb68a Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 28 Feb 2026 18:10:16 -0800 Subject: [PATCH 35/55] update --- .../promisehandlers/EndIterationHandler.scala | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala index e1827c7e949..b0233bbdcb9 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala @@ -20,25 +20,28 @@ package org.apache.texera.amber.engine.architecture.worker.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, EmptyRequest, EndIterationRequest} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ + AsyncRPCContext, + EmptyRequest, + EndIterationRequest +} import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer -import org.apache.texera.amber.operator.loop.LoopEndOpExec trait EndIterationHandler { this: DataProcessorRPCHandlerInitializer => override def endIteration( - request: EndIterationRequest, - ctx: AsyncRPCContext - ): Future[EmptyReturn] = { + request: EndIterationRequest, + ctx: AsyncRPCContext + ): Future[EmptyReturn] = { dp.executor match { - case _: LoopEndOpExec => - //workerInterface.nextIteration(EmptyRequest(), mkContext(request.worker)) + //case _: LoopEndOpExec => + //workerInterface.nextIteration(EmptyRequest(), mkContext(request.worker)) case _ => - //dp.processOnFinish() - //dp.outputManager.finalizeIteration(request.worker) + //dp.processOnFinish() + //dp.outputManager.finalizeIteration(request.worker) } EmptyReturn() } -} \ No newline at end of file +} From 0970a53a65ef38690d9befd39029b8fb62a83662 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 28 Feb 2026 20:11:01 -0800 Subject: [PATCH 36/55] update --- amber/src/main/python/core/runnables/main_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/main/python/core/runnables/main_loop.py b/amber/src/main/python/core/runnables/main_loop.py index d73c655734f..d86a2a05eb7 100644 --- a/amber/src/main/python/core/runnables/main_loop.py +++ b/amber/src/main/python/core/runnables/main_loop.py @@ -329,7 +329,7 @@ def _process_ecm(self, ecm_element: ECMElement): if ecm.ecm_type != EmbeddedControlMessageType.NO_ALIGNMENT: self.context.pause_manager.resume(PauseType.ECM_PAUSE) - + self._switch_context() if self.context.tuple_processing_manager.current_internal_marker: { StartChannel: self._process_start_channel, From 2b78e2ce01499e829cb7af1d6d2976586296acf3 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 28 Feb 2026 20:11:26 -0800 Subject: [PATCH 37/55] update --- .../messaginglayer/OutputManager.scala | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala index 8f0c0f6feef..13f516b6bf3 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala @@ -26,10 +26,17 @@ import org.apache.texera.amber.core.storage.result.ResultSchema import org.apache.texera.amber.core.tuple._ import org.apache.texera.amber.core.virtualidentity.{ActorVirtualIdentity, ChannelIdentity} import org.apache.texera.amber.core.workflow.{PhysicalLink, PortIdentity} -import org.apache.texera.amber.engine.architecture.messaginglayer.OutputManager.{DPOutputIterator, getBatchSize, toPartitioner} +import org.apache.texera.amber.engine.architecture.messaginglayer.OutputManager.{ + DPOutputIterator, + getBatchSize, + toPartitioner +} import org.apache.texera.amber.engine.architecture.sendsemantics.partitioners._ import org.apache.texera.amber.engine.architecture.sendsemantics.partitionings._ -import org.apache.texera.amber.engine.architecture.worker.managers.{OutputPortResultWriterThread, PortStorageWriterTerminateSignal} +import org.apache.texera.amber.engine.architecture.worker.managers.{ + OutputPortResultWriterThread, + PortStorageWriterTerminateSignal +} import org.apache.texera.amber.engine.common.AmberLogging import org.apache.texera.amber.util.VirtualIdentityUtils @@ -118,8 +125,7 @@ class OutputManager( : mutable.HashMap[PortIdentity, OutputPortResultWriterThread] = mutable.HashMap() - private val ECMWriters - : mutable.HashMap[PortIdentity, BufferedItemWriter[Tuple]] = + private val ECMWriters: mutable.HashMap[PortIdentity, BufferedItemWriter[Tuple]] = mutable.HashMap() /** @@ -230,21 +236,8 @@ class OutputManager( }) } - def saveECMToStorageIfNeeded( - tuple: Tuple, - outputPortId: Option[PortIdentity] = None - ): Unit = { - (outputPortId match { - case Some(portId) => - this.ECMWriters.get(portId) match { - case Some(_) => this.ECMWriters.filter(_._1 == portId) - case None => Map.empty - } - case None => this.ECMWriters - }).foreach({ - case (portId, writer) => - writer.putOne(new Tuple(ResultSchema.ecmSchema, Array("erge"))) - }) + def saveECMToStorageIfNeeded(tuple: Tuple, outputPortId: PortIdentity): Unit = { + this.ECMWriters(outputPortId).putOne(new Tuple(ResultSchema.ecmSchema, Array("erge"))) } /** From 55f288fd06a5461bc7de5ae8869d105f0d827b0b Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 04:30:55 -0800 Subject: [PATCH 38/55] update --- .../scheduling/RegionExecutionCoordinator.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index 2965fe5f137..6793248d8a3 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@ -211,14 +211,15 @@ class RegionExecutionCoordinator( regionExecution: RegionExecution, attempt: Int = 1 ): Future[Unit] = { - terminateWorkers(regionExecution).rescue { case err => - logger.warn( - s"Failed to terminate region ${region.id.id} on attempt $attempt. Retrying in ${killRetryDelay.inMilliseconds} ms.", - err - ) - Future - .sleep(killRetryDelay)(killRetryTimer) - .flatMap(_ => terminateWorkersWithRetry(regionExecution, attempt + 1)) + terminateWorkers(regionExecution).rescue { + case err => + logger.warn( + s"Failed to terminate region ${region.id.id} on attempt $attempt. Retrying in ${killRetryDelay.inMilliseconds} ms.", + err + ) + Future + .sleep(killRetryDelay)(killRetryTimer) + .flatMap(_ => terminateWorkersWithRetry(regionExecution, attempt + 1)) } } From 2ccde1e0dffbe4c154e04e733ba3df4ac5bb3202 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 04:57:53 -0800 Subject: [PATCH 39/55] update --- amber/src/main/python/core/runnables/data_processor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/amber/src/main/python/core/runnables/data_processor.py b/amber/src/main/python/core/runnables/data_processor.py index 4399b1a3a2f..815e85a6446 100644 --- a/amber/src/main/python/core/runnables/data_processor.py +++ b/amber/src/main/python/core/runnables/data_processor.py @@ -100,6 +100,7 @@ def process_state(self, state: State) -> None: self._context.worker_id, self._context.console_message_manager.print_buf, ): + self._switch_context() self._set_output_state(executor.process_state(state, port_id)) except Exception as err: From 9b0d14dea303ff31f2acea06921b0eac0d319e20 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 05:03:22 -0800 Subject: [PATCH 40/55] update --- amber/src/main/python/core/models/state.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/amber/src/main/python/core/models/state.py b/amber/src/main/python/core/models/state.py index 2c8a268dfb7..ca39acc3335 100644 --- a/amber/src/main/python/core/models/state.py +++ b/amber/src/main/python/core/models/state.py @@ -35,6 +35,13 @@ def __init__( self.__dict__.update(table.to_pandas().iloc[0].to_dict()) self.schema = Schema(table.schema) + @classmethod + def from_tuple(cls, tuple, schema): + obj = cls() + obj.__dict__.update(tuple.as_dict()) + obj.schema = schema + return obj + def add( self, key: str, value: any, value_type: Optional[AttributeType] = None ) -> None: From 3a2d0b9abeae838a8b3aaac2110eeba1f4d188aa Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 05:03:41 -0800 Subject: [PATCH 41/55] update --- amber/src/main/python/core/storage/vfs_uri_factory.py | 1 + .../org/apache/texera/amber/core/storage/VFSURIFactory.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/amber/src/main/python/core/storage/vfs_uri_factory.py b/amber/src/main/python/core/storage/vfs_uri_factory.py index de0c5db56ec..0e23e607055 100644 --- a/amber/src/main/python/core/storage/vfs_uri_factory.py +++ b/amber/src/main/python/core/storage/vfs_uri_factory.py @@ -34,6 +34,7 @@ class VFSResourceType(str, Enum): RESULT = "result" RUNTIME_STATISTICS = "runtimeStatistics" CONSOLE_MESSAGES = "consoleMessages" + STATE = "state" class VFSURIFactory: diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/VFSURIFactory.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/VFSURIFactory.scala index f4f3a9e4bf1..990776a69f0 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/VFSURIFactory.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/VFSURIFactory.scala @@ -34,7 +34,7 @@ object VFSResourceType extends Enumeration { val RESULT: Value = Value("result") val RUNTIME_STATISTICS: Value = Value("runtimeStatistics") val CONSOLE_MESSAGES: Value = Value("consoleMessages") - val ECM: Value = Value("ecm") + val STATE: Value = Value("state") } object VFSURIFactory { From 0cfcf2fb3da3582f798046e554d41f76750acb57 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 05:04:07 -0800 Subject: [PATCH 42/55] update --- amber/src/main/python/core/storage/document_factory.py | 4 ++-- .../apache/texera/amber/core/storage/DocumentFactory.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/amber/src/main/python/core/storage/document_factory.py b/amber/src/main/python/core/storage/document_factory.py index 9b686ab66b6..a939c4302ff 100644 --- a/amber/src/main/python/core/storage/document_factory.py +++ b/amber/src/main/python/core/storage/document_factory.py @@ -61,7 +61,7 @@ def create_document(uri: str, schema: Schema) -> VirtualDocument: if parsed_uri.scheme == VFSURIFactory.VFS_FILE_URI_SCHEME: _, _, _, resource_type = VFSURIFactory.decode_uri(uri) - if resource_type in {VFSResourceType.RESULT}: + if resource_type in {VFSResourceType.RESULT, VFSResourceType.STATE}: storage_key = DocumentFactory.sanitize_uri_path(parsed_uri) # Convert Amber Schema to Iceberg Schema with LARGE_BINARY @@ -96,7 +96,7 @@ def open_document(uri: str) -> typing.Tuple[VirtualDocument, Optional[Schema]]: if parsed_uri.scheme == "vfs": _, _, _, resource_type = VFSURIFactory.decode_uri(uri) - if resource_type in {VFSResourceType.RESULT}: + if resource_type in {VFSResourceType.RESULT, VFSResourceType.STATE}: storage_key = DocumentFactory.sanitize_uri_path(parsed_uri) table = load_table_metadata( diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala index 683e65786a8..6912472b69e 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala @@ -72,7 +72,7 @@ object DocumentFactory { case RESULT => StorageConfig.icebergTableResultNamespace case CONSOLE_MESSAGES => StorageConfig.icebergTableConsoleMessagesNamespace case RUNTIME_STATISTICS => StorageConfig.icebergTableRuntimeStatisticsNamespace - case ECM => "ECM" + case STATE => "STATE" case _ => throw new IllegalArgumentException(s"Resource type $resourceType is not supported") } From 00e49a56e4b63d2a4bc89ae3f934753ee1338478 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 05:49:08 -0800 Subject: [PATCH 43/55] update --- .../org/apache/texera/amber/core/storage/DocumentFactory.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala index 6912472b69e..4782d96f7df 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala @@ -72,7 +72,7 @@ object DocumentFactory { case RESULT => StorageConfig.icebergTableResultNamespace case CONSOLE_MESSAGES => StorageConfig.icebergTableConsoleMessagesNamespace case RUNTIME_STATISTICS => StorageConfig.icebergTableRuntimeStatisticsNamespace - case STATE => "STATE" + case STATE => "STATE" case _ => throw new IllegalArgumentException(s"Resource type $resourceType is not supported") } From f71dbec1101e8cd0bed138c0f1bc2b3ce6c8b0fb Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 05:49:11 -0800 Subject: [PATCH 44/55] update --- .../texera/amber/core/storage/result/ResultSchema.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/ResultSchema.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/ResultSchema.scala index 4be34b856d6..ade33283f7f 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/ResultSchema.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/result/ResultSchema.scala @@ -39,9 +39,4 @@ object ResultSchema { val consoleMessagesSchema: Schema = new Schema( new Attribute("message", AttributeType.STRING) ) - - val ecmSchema: Schema = new Schema( - new Attribute("LoopStartId", AttributeType.STRING), - new Attribute("iteration", AttributeType.INTEGER) - ) } From 565ee717c81a4dd63e32e8ac481ec907e6542abf Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 05:49:22 -0800 Subject: [PATCH 45/55] update --- ...ut_port_materialization_reader_runnable.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/amber/src/main/python/core/storage/runnables/input_port_materialization_reader_runnable.py b/amber/src/main/python/core/storage/runnables/input_port_materialization_reader_runnable.py index e49c0316cc7..3d3242df9bf 100644 --- a/amber/src/main/python/core/storage/runnables/input_port_materialization_reader_runnable.py +++ b/amber/src/main/python/core/storage/runnables/input_port_materialization_reader_runnable.py @@ -17,8 +17,8 @@ import typing from loguru import logger -from pyarrow import Table from typing import Union +from pyarrow import Table from core.architecture.sendsemantics.broad_cast_partitioner import ( BroadcastPartitioner, @@ -34,7 +34,7 @@ from core.architecture.sendsemantics.round_robin_partitioner import ( RoundRobinPartitioner, ) -from core.models import Tuple, InternalQueue, DataFrame, DataPayload +from core.models import Tuple, InternalQueue, DataFrame, DataPayload, State, StateFrame from core.models.internal_queue import DataElement, ECMElement from core.storage.document_factory import DocumentFactory from core.util import Stoppable, get_one_of @@ -125,6 +125,15 @@ def tuple_to_batch_with_filter(self, tuple_: Tuple) -> typing.Iterator[DataFrame if receiver == self.worker_actor_id: yield self.tuples_to_data_frame(tuples) + def emit_state_with_filter(self, state: State) -> typing.Iterator[StateFrame]: + for receiver, payload in self.partitioner.flush_state(state): + if receiver == self.worker_actor_id: + yield ( + StateFrame(payload) + if isinstance(payload, State) + else self.tuples_to_data_frame(payload) + ) + def run(self) -> None: """ Main execution logic that reads tuples from the materialized storage and @@ -149,6 +158,19 @@ def run(self) -> None: tup.cast_to_schema(self.tuple_schema) for data_frame in self.tuple_to_batch_with_filter(tup): self.emit_payload(data_frame) + try: + state_document, state_schema = DocumentFactory.open_document( + f"{self.uri}/state" + ) + state_iterator = state_document.get() + for state in state_iterator: + for state_frame in self.emit_state_with_filter( + State.from_tuple(state, state_schema) + ): + self.emit_payload(state_frame) + except ValueError: + pass + self.emit_ecm("EndChannel", EmbeddedControlMessageType.PORT_ALIGNMENT) self._finished = True except Exception as err: From aa444a0093d3c8968debab144171a90e25f8aed9 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 16:59:51 -0800 Subject: [PATCH 46/55] update --- .../org/apache/texera/amber/core/storage/DocumentFactory.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala index 4782d96f7df..ae37def667e 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/DocumentFactory.scala @@ -72,7 +72,7 @@ object DocumentFactory { case RESULT => StorageConfig.icebergTableResultNamespace case CONSOLE_MESSAGES => StorageConfig.icebergTableConsoleMessagesNamespace case RUNTIME_STATISTICS => StorageConfig.icebergTableRuntimeStatisticsNamespace - case STATE => "STATE" + case STATE => "state" case _ => throw new IllegalArgumentException(s"Resource type $resourceType is not supported") } @@ -120,6 +120,7 @@ object DocumentFactory { case RESULT => StorageConfig.icebergTableResultNamespace case CONSOLE_MESSAGES => StorageConfig.icebergTableConsoleMessagesNamespace case RUNTIME_STATISTICS => StorageConfig.icebergTableRuntimeStatisticsNamespace + case STATE => "state" case _ => throw new IllegalArgumentException(s"Resource type $resourceType is not supported") } From 2e7c72ac4ec2289543ada0a17f907e2e59f66953 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 20:52:40 -0800 Subject: [PATCH 47/55] update --- .../python/core/storage/document_factory.py | 107 ++++++++++-------- .../core/storage/iceberg/iceberg_utils.py | 3 +- 2 files changed, 60 insertions(+), 50 deletions(-) diff --git a/amber/src/main/python/core/storage/document_factory.py b/amber/src/main/python/core/storage/document_factory.py index a939c4302ff..8a4d6fe3c5f 100644 --- a/amber/src/main/python/core/storage/document_factory.py +++ b/amber/src/main/python/core/storage/document_factory.py @@ -61,30 +61,35 @@ def create_document(uri: str, schema: Schema) -> VirtualDocument: if parsed_uri.scheme == VFSURIFactory.VFS_FILE_URI_SCHEME: _, _, _, resource_type = VFSURIFactory.decode_uri(uri) - if resource_type in {VFSResourceType.RESULT, VFSResourceType.STATE}: - storage_key = DocumentFactory.sanitize_uri_path(parsed_uri) - - # Convert Amber Schema to Iceberg Schema with LARGE_BINARY - # field name encoding - iceberg_schema = amber_schema_to_iceberg_schema(schema) - - create_table( - IcebergCatalogInstance.get_instance(), - StorageConfig.ICEBERG_TABLE_RESULT_NAMESPACE, - storage_key, - iceberg_schema, - override_if_exists=True, - ) - - return IcebergDocument[Tuple]( - StorageConfig.ICEBERG_TABLE_RESULT_NAMESPACE, - storage_key, - iceberg_schema, - amber_tuples_to_arrow_table, - arrow_table_to_amber_tuples, - ) - else: - raise ValueError(f"Resource type {resource_type} is not supported") + match resource_type: + case VFSResourceType.RESULT: + namespace = StorageConfig.ICEBERG_TABLE_RESULT_NAMESPACE + case VFSResourceType.STATE: + namespace = "state" + case _: + raise ValueError(f"Resource type {resource_type} is not supported") + + storage_key = DocumentFactory.sanitize_uri_path(parsed_uri) + # Convert Amber Schema to Iceberg Schema with LARGE_BINARY + # field name encoding + iceberg_schema = amber_schema_to_iceberg_schema(schema) + + create_table( + IcebergCatalogInstance.get_instance(), + namespace, + storage_key, + iceberg_schema, + override_if_exists=True, + ) + + return IcebergDocument[Tuple]( + namespace, + storage_key, + iceberg_schema, + amber_tuples_to_arrow_table, + arrow_table_to_amber_tuples, + ) + else: raise NotImplementedError( f"Unsupported URI scheme: {parsed_uri.scheme} for creating the document" @@ -96,30 +101,36 @@ def open_document(uri: str) -> typing.Tuple[VirtualDocument, Optional[Schema]]: if parsed_uri.scheme == "vfs": _, _, _, resource_type = VFSURIFactory.decode_uri(uri) - if resource_type in {VFSResourceType.RESULT, VFSResourceType.STATE}: - storage_key = DocumentFactory.sanitize_uri_path(parsed_uri) - - table = load_table_metadata( - IcebergCatalogInstance.get_instance(), - StorageConfig.ICEBERG_TABLE_RESULT_NAMESPACE, - storage_key, - ) - - if table is None: - raise ValueError("No storage is found for the given URI") - - amber_schema = Schema(table.schema().as_arrow()) - - document = IcebergDocument( - StorageConfig.ICEBERG_TABLE_RESULT_NAMESPACE, - storage_key, - table.schema(), - amber_tuples_to_arrow_table, - arrow_table_to_amber_tuples, - ) - return document, amber_schema - else: - raise ValueError(f"Resource type {resource_type} is not supported") + match resource_type: + case VFSResourceType.RESULT: + namespace = StorageConfig.ICEBERG_TABLE_RESULT_NAMESPACE + case VFSResourceType.STATE: + namespace = "state" + case _: + raise ValueError(f"Resource type {resource_type} is not supported") + + storage_key = DocumentFactory.sanitize_uri_path(parsed_uri) + + table = load_table_metadata( + IcebergCatalogInstance.get_instance(), + namespace, + storage_key, + ) + + if table is None: + raise ValueError("No storage is found for the given URI") + + amber_schema = Schema(table.schema().as_arrow()) + + document = IcebergDocument( + namespace, + storage_key, + table.schema(), + amber_tuples_to_arrow_table, + arrow_table_to_amber_tuples, + ) + return document, amber_schema + else: raise NotImplementedError( f"Unsupported URI scheme: {parsed_uri.scheme} for opening the document" diff --git a/amber/src/main/python/core/storage/iceberg/iceberg_utils.py b/amber/src/main/python/core/storage/iceberg/iceberg_utils.py index 9e17b2e0e82..cca785beb69 100644 --- a/amber/src/main/python/core/storage/iceberg/iceberg_utils.py +++ b/amber/src/main/python/core/storage/iceberg/iceberg_utils.py @@ -148,7 +148,7 @@ def create_postgres_catalog( catalog_name, **{ "uri": f"postgresql+pg8000://{username}:{password}@{uri_without_scheme}", - "warehouse": f"file://{warehouse_path}", + "warehouse": warehouse_path, }, ) @@ -180,7 +180,6 @@ def create_table( if catalog.table_exists(identifier) and override_if_exists: catalog.drop_table(identifier) - table = catalog.create_table( identifier=identifier, schema=table_schema, From f8ce99fec14bb831a8a5a26531f1e69055a8e16b Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 20:53:19 -0800 Subject: [PATCH 48/55] update --- .../architecture/packaging/output_manager.py | 18 +++++++++++ .../main/python/core/runnables/main_loop.py | 1 + ...ut_port_materialization_reader_runnable.py | 2 +- .../messaginglayer/OutputManager.scala | 17 +++++----- .../architecture/worker/DataProcessor.scala | 1 + ...InputPortMaterializationReaderThread.scala | 32 ++++++++----------- .../texera/amber/core/state/State.scala | 13 ++++---- 7 files changed, 51 insertions(+), 33 deletions(-) diff --git a/amber/src/main/python/core/architecture/packaging/output_manager.py b/amber/src/main/python/core/architecture/packaging/output_manager.py index bf4afbf396f..d92e8dcc987 100644 --- a/amber/src/main/python/core/architecture/packaging/output_manager.py +++ b/amber/src/main/python/core/architecture/packaging/output_manager.py @@ -87,6 +87,8 @@ def __init__(self, worker_id: str): PortIdentity, typing.Tuple[Queue, PortStorageWriter, Thread] ] = dict() + self._storage_uris: typing.Dict[PortIdentity, str] = dict() + def is_missing_output_ports(self): """ This method is only used for ensuring correct region execution. @@ -126,6 +128,7 @@ def set_up_port_storage_writer(self, port_id: PortIdentity, storage_uri: str): Create a separate thread for saving output tuples of a port to storage in batch. """ + self._storage_uris[port_id] = storage_uri document, _ = DocumentFactory.open_document(storage_uri) buffered_item_writer = document.writer(str(get_worker_index(self.worker_id))) writer_queue = Queue() @@ -171,6 +174,21 @@ def save_tuple_to_storage_if_needed(self, tuple_: Tuple, port_id=None) -> None: PortStorageWriterElement(data_tuple=tuple_) ) + def save_state_to_storage_if_needed(self, state: State, port_id=None) -> None: + if port_id is None: + uris = self._storage_uris.values() + elif port_id in self._storage_uris: + uris = [self._storage_uris[port_id]] + else: + return + + for uri in uris: + writer = DocumentFactory.create_document( + uri.replace("/result", "/state"), state.schema + ).writer(str(get_worker_index(self.worker_id))) + writer.put_one(Tuple(vars(state))) + writer.close() + def close_port_storage_writers(self) -> None: """ Flush the buffers of port storage writers and wait for all the diff --git a/amber/src/main/python/core/runnables/main_loop.py b/amber/src/main/python/core/runnables/main_loop.py index d86a2a05eb7..11adfd2fcad 100644 --- a/amber/src/main/python/core/runnables/main_loop.py +++ b/amber/src/main/python/core/runnables/main_loop.py @@ -197,6 +197,7 @@ def process_input_state(self) -> None: payload=batch, ) ) + self.context.output_manager.save_state_to_storage_if_needed(output_state) def process_tuple_with_udf(self) -> Iterator[Optional[Tuple]]: """ diff --git a/amber/src/main/python/core/storage/runnables/input_port_materialization_reader_runnable.py b/amber/src/main/python/core/storage/runnables/input_port_materialization_reader_runnable.py index 3d3242df9bf..12aa8991263 100644 --- a/amber/src/main/python/core/storage/runnables/input_port_materialization_reader_runnable.py +++ b/amber/src/main/python/core/storage/runnables/input_port_materialization_reader_runnable.py @@ -160,7 +160,7 @@ def run(self) -> None: self.emit_payload(data_frame) try: state_document, state_schema = DocumentFactory.open_document( - f"{self.uri}/state" + self.uri.replace("/result", "/state") ) state_iterator = state_document.get() for state in state_iterator: diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala index 13f516b6bf3..a018158f369 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala @@ -125,8 +125,7 @@ class OutputManager( : mutable.HashMap[PortIdentity, OutputPortResultWriterThread] = mutable.HashMap() - private val ECMWriters: mutable.HashMap[PortIdentity, BufferedItemWriter[Tuple]] = - mutable.HashMap() + private val storageUris: mutable.HashMap[Int, URI] = mutable.HashMap() /** * Add down stream operator and its corresponding Partitioner. @@ -236,8 +235,13 @@ class OutputManager( }) } - def saveECMToStorageIfNeeded(tuple: Tuple, outputPortId: PortIdentity): Unit = { - this.ECMWriters(outputPortId).putOne(new Tuple(ResultSchema.ecmSchema, Array("erge"))) + def saveStateToStorageIfNeeded(state: State, outputPortId: Int): Unit = { + val writer = DocumentFactory + .createDocument(this.storageUris(outputPortId).resolve("state"), state.schema) + .writer(VirtualIdentityUtils.getWorkerIndex(actorId).toString) + .asInstanceOf[BufferedItemWriter[Tuple]] + writer.putOne(state.toTuple) + writer.close() } /** @@ -288,10 +292,7 @@ class OutputManager( } private def setupOutputStorageWriterThread(portId: PortIdentity, storageUri: URI): Unit = { - this.ECMWriters(portId) = DocumentFactory - .createDocument(storageUri.resolve("ecm"), ResultSchema.ecmSchema) - .writer(VirtualIdentityUtils.getWorkerIndex(actorId).toString) - .asInstanceOf[BufferedItemWriter[Tuple]] + this.storageUris(portId.id) = storageUri val bufferedItemWriter = DocumentFactory .openDocument(storageUri) ._1 diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala index 3aa5fa90a46..bc89f26a754 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala @@ -126,6 +126,7 @@ class DataProcessor( val outputState = executor.processState(state, port) if (outputState.isDefined) { outputManager.emitState(outputState.get) + outputManager.saveStateToStorageIfNeeded(state, port) } } catch safely { case e => diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/InputPortMaterializationReaderThread.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/InputPortMaterializationReaderThread.scala index 10fbbc44a2c..91b22a98d9e 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/InputPortMaterializationReaderThread.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/InputPortMaterializationReaderThread.scala @@ -21,31 +21,19 @@ package org.apache.texera.amber.engine.architecture.worker.managers import io.grpc.MethodDescriptor import org.apache.texera.amber.config.ApplicationConfig +import org.apache.texera.amber.core.state.State import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.storage.model.VirtualDocument import org.apache.texera.amber.core.tuple.Tuple -import org.apache.texera.amber.core.virtualidentity.{ - ActorVirtualIdentity, - ChannelIdentity, - EmbeddedControlMessageIdentity -} +import org.apache.texera.amber.core.virtualidentity.{ActorVirtualIdentity, ChannelIdentity, EmbeddedControlMessageIdentity} import org.apache.texera.amber.engine.architecture.messaginglayer.OutputManager.toPartitioner -import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.{ - NO_ALIGNMENT, - PORT_ALIGNMENT -} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.{NO_ALIGNMENT, PORT_ALIGNMENT} import org.apache.texera.amber.engine.architecture.rpc.controlcommands._ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn -import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceGrpc.{ - METHOD_END_CHANNEL, - METHOD_START_CHANNEL -} +import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceGrpc.{METHOD_END_CHANNEL, METHOD_START_CHANNEL} import org.apache.texera.amber.engine.architecture.sendsemantics.partitionings.Partitioning -import org.apache.texera.amber.engine.architecture.worker.WorkflowWorker.{ - DPInputQueueElement, - FIFOMessageElement -} -import org.apache.texera.amber.engine.common.ambermessage.{DataFrame, WorkflowFIFOMessage} +import org.apache.texera.amber.engine.architecture.worker.WorkflowWorker.{DPInputQueueElement, FIFOMessageElement} +import org.apache.texera.amber.engine.common.ambermessage.{DataFrame, StateFrame, WorkflowFIFOMessage} import org.apache.texera.amber.util.VirtualIdentityUtils.getFromActorIdForInputPortStorage import java.net.URI @@ -106,6 +94,14 @@ class InputPortMaterializationReaderThread( } // Flush any remaining tuples in the buffer. if (buffer.nonEmpty) flush() + val state_document = DocumentFactory.openDocument(uri.resolve("state"))._1.asInstanceOf[VirtualDocument[Tuple]] + val stateReadIterator = state_document.get() + + if (stateReadIterator.hasNext) { + val state = State(Option(stateReadIterator.next())) + inputMessageQueue.put(FIFOMessageElement(WorkflowFIFOMessage(channelId, getSequenceNumber, StateFrame(state)))) + } + emitECM(METHOD_END_CHANNEL, PORT_ALIGNMENT) isFinished.set(true) } catch { diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/state/State.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/state/State.scala index 3226c9d2fe7..92abd9041c6 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/state/State.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/state/State.scala @@ -41,14 +41,15 @@ final case class State(tuple: Option[Tuple] = None, passToAllDownstream: Boolean def apply(key: String): Any = get(key) + def schema: Schema = + Schema(data.map { + case (name, (attrType, _)) => + new Attribute(name, attrType) + }.toList) + def toTuple: Tuple = Tuple - .builder( - Schema(data.map { - case (name, (attrType, _)) => - new Attribute(name, attrType) - }.toList) - ) + .builder(schema) .addSequentially(data.values.map(_._2).toArray) .build() From fe7e07146c77a69265078cbbc36a4e3a69e6665b Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sun, 1 Mar 2026 20:57:54 -0800 Subject: [PATCH 49/55] update --- .../texera/amber/operator/loop/LoopStartOpDesc.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala index 2b34251bb6d..a6dbc6d5de3 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala @@ -76,11 +76,14 @@ class LoopStartOpDesc extends LogicalOp { def generatePythonCode(): String = { s""" |from pytexera import * - |class ProcessLoopStartOperator(LoopStartOperator): + |class ProcessTableOperator(UDFTableOperator): | @overrides - | def loop_initialization(self): + | def produce_state_on_finish(self, port: int) -> State: + | state = State(pass_to_all_downstream = True) | $initialization - | return "$variable",$variable + | state["variable"] = "$variable" + | state["iteration"] = $variable + | return state | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: From 08679f003fae0f0fa6568cea06b979dbb78047f5 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 4 Mar 2026 01:25:21 -0800 Subject: [PATCH 50/55] update --- amber/src/main/python/core/models/operator.py | 24 +++++++++++++++++++ amber/src/main/python/pytexera/__init__.py | 3 +++ .../amber/operator/loop/LoopEndOpDesc.scala | 8 ++++--- .../amber/operator/loop/LoopStartOpDesc.scala | 7 +----- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/amber/src/main/python/core/models/operator.py b/amber/src/main/python/core/models/operator.py index 79050839958..e05bd45b6c7 100644 --- a/amber/src/main/python/core/models/operator.py +++ b/amber/src/main/python/core/models/operator.py @@ -293,3 +293,27 @@ def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike] time, or None. """ yield + + +class LoopStartOperator(TableOperator): + def open(self) -> None: + pass + + @abstractmethod + def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: + yield + + def close(self) -> None: + pass + + +class LoopEndOperator(TableOperator): + def open(self) -> None: + pass + + @overrides.final + def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: + yield table + + def close(self) -> None: + pass diff --git a/amber/src/main/python/pytexera/__init__.py b/amber/src/main/python/pytexera/__init__.py index e40d1a43fe0..c6001667380 100644 --- a/amber/src/main/python/pytexera/__init__.py +++ b/amber/src/main/python/pytexera/__init__.py @@ -19,6 +19,7 @@ from overrides import overrides from typing import Iterator, Optional, Union +from core.models.operator import LoopStartOperator, LoopEndOperator from pyamber import * from .storage.dataset_file_document import DatasetFileDocument from .storage.large_binary_input_stream import LargeBinaryInputStream @@ -43,6 +44,8 @@ "UDFTableOperator", "UDFBatchOperator", "UDFSourceOperator", + "LoopStartOperator", + "LoopEndOperator", "DatasetFileDocument", "largebinary", "LargeBinaryInputStream", diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index 294028d2666..719b14afe2f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -74,9 +74,11 @@ class LoopEndOpDesc extends LogicalOp { |from pytexera import * |class ProcessLoopEndOperator(LoopEndOperator): | @overrides - | def loop_condition_check(self) -> bool: - | $update - | return $condition + | def process_state(self, state: State, port: int) -> Optional[State]: + | i = state['i'] + | $update + | print(i) + | return state |""".stripMargin } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala index a6dbc6d5de3..8abc3ad0dc8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala @@ -28,10 +28,6 @@ import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} class LoopStartOpDesc extends LogicalOp { - @JsonProperty(required = true, defaultValue = "i") - @JsonSchemaTitle("Variable") - var variable: String = _ - @JsonProperty(required = true, defaultValue = "i = 0") @JsonSchemaTitle("Initialization") var initialization: String = _ @@ -81,8 +77,7 @@ class LoopStartOpDesc extends LogicalOp { | def produce_state_on_finish(self, port: int) -> State: | state = State(pass_to_all_downstream = True) | $initialization - | state["variable"] = "$variable" - | state["iteration"] = $variable + | state["i"] = i | return state | | @overrides From b18d9dbc523378052189c7b187a310c05329ef35 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 4 Mar 2026 01:37:39 -0800 Subject: [PATCH 51/55] update --- amber/src/main/python/core/models/operator.py | 4 ++-- .../org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/amber/src/main/python/core/models/operator.py b/amber/src/main/python/core/models/operator.py index e05bd45b6c7..7532fae1bd7 100644 --- a/amber/src/main/python/core/models/operator.py +++ b/amber/src/main/python/core/models/operator.py @@ -311,9 +311,9 @@ class LoopEndOperator(TableOperator): def open(self) -> None: pass - @overrides.final + @abstractmethod def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - yield table + yield def close(self) -> None: pass diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index 719b14afe2f..d5aa4a0a721 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -79,6 +79,10 @@ class LoopEndOpDesc extends LogicalOp { | $update | print(i) | return state + | + | @overrides + | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: + | yield table |""".stripMargin } } From dac211aff4c7969f1cc6d1de496e5998f0703fb9 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 4 Mar 2026 21:25:46 -0800 Subject: [PATCH 52/55] update --- amber/src/main/python/core/runnables/main_loop.py | 3 ++- .../texera/amber/operator/loop/LoopEndOpDesc.scala | 10 +--------- .../texera/amber/operator/loop/LoopStartOpDesc.scala | 11 +++++++++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/amber/src/main/python/core/runnables/main_loop.py b/amber/src/main/python/core/runnables/main_loop.py index 11adfd2fcad..0cfb89f7d27 100644 --- a/amber/src/main/python/core/runnables/main_loop.py +++ b/amber/src/main/python/core/runnables/main_loop.py @@ -94,12 +94,13 @@ def complete(self) -> None: """ # flush the buffered console prints self._check_and_report_console_messages(force_flush=True) + controller_interface = self._async_rpc_client.controller_stub() + #controller_interface.iteration_completed(EmptyRequest()) self.context.executor_manager.executor.close() # stop the data processing thread self.data_processor.stop() self.context.state_manager.transit_to(WorkerState.COMPLETED) self.context.statistics_manager.update_total_execution_time(time.time_ns()) - controller_interface = self._async_rpc_client.controller_stub() controller_interface.worker_execution_completed(EmptyRequest()) self.context.close() diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala index d5aa4a0a721..f5498fad1cd 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopEndOpDesc.scala @@ -28,13 +28,7 @@ import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} class LoopEndOpDesc extends LogicalOp { - @JsonProperty(required = true, defaultValue = "i += 1") - @JsonSchemaTitle("Update") - var update: String = _ - @JsonProperty(required = true, defaultValue = "i < len(table)") - @JsonSchemaTitle("Condition") - var condition: String = _ override def getPhysicalOp( workflowId: WorkflowIdentity, @@ -75,9 +69,7 @@ class LoopEndOpDesc extends LogicalOp { |class ProcessLoopEndOperator(LoopEndOperator): | @overrides | def process_state(self, state: State, port: int) -> Optional[State]: - | i = state['i'] - | $update - | print(i) + | print(state) | return state | | @overrides diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala index 8abc3ad0dc8..f2550d2c00f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/loop/LoopStartOpDesc.scala @@ -32,6 +32,14 @@ class LoopStartOpDesc extends LogicalOp { @JsonSchemaTitle("Initialization") var initialization: String = _ + @JsonProperty(required = true, defaultValue = "i += 1") + @JsonSchemaTitle("Update") + var update: String = _ + + @JsonProperty(required = true, defaultValue = "i < len(table)") + @JsonSchemaTitle("Condition") + var condition: String = _ + @JsonProperty(required = true, defaultValue = "table.iloc[0]") @JsonSchemaTitle("Output") var output: String = _ @@ -75,8 +83,11 @@ class LoopStartOpDesc extends LogicalOp { |class ProcessTableOperator(UDFTableOperator): | @overrides | def produce_state_on_finish(self, port: int) -> State: + | table = Table(self._TableOperator__table_data[port]) | state = State(pass_to_all_downstream = True) | $initialization + | state["condition"] = $condition + | $update | state["i"] = i | return state | From 43f2ca68c853e76bf002b1962f205138538d94b3 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 5 Mar 2026 00:12:09 -0800 Subject: [PATCH 53/55] update --- .../texera/amber/engine/architecture/scheduling/Schedule.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala index b891577ff8c..36ccd4a5ad7 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala @@ -26,7 +26,7 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterator[Set[Region]] { private var currentLevel = levelSets.keys.minOption.getOrElse(0) private var loopStartLevel = currentLevel - private var iteration = 10 + private var iteration = 3 private var i = 1 def getRegions: List[Region] = levelSets.values.flatten.toList From 3ada4c23138d6a405e5efe559b6d7b6f58e16b2f Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 7 Mar 2026 21:19:44 -0800 Subject: [PATCH 54/55] update --- .../architecture/rpc/controlcommands.proto | 9 +- .../architecture/rpc/controllerservice.proto | 1 + .../architecture/rpc/workerservice.proto | 1 - amber/src/main/python/core/models/operator.py | 6 + .../amber/engine/architecture/rpc/__init__.py | 1489 +++++++++-------- ...ControllerAsyncRPCHandlerInitializer.scala | 1 + .../IterationCompletedHandler.scala} | 34 +- 7 files changed, 799 insertions(+), 742 deletions(-) rename amber/src/main/scala/org/apache/texera/amber/engine/architecture/{worker/promisehandlers/EndIterationHandler.scala => controller/promisehandlers/IterationCompletedHandler.scala} (56%) diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto index efff6948a00..a6676cc4a23 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto @@ -46,6 +46,7 @@ message ControlRequest { PortCompletedRequest portCompletedRequest = 9; WorkerStateUpdatedRequest workerStateUpdatedRequest = 10; LinkWorkersRequest linkWorkersRequest = 11; + IterationCompletedRequest iterationCompletedRequest = 12; // request for worker AddInputChannelRequest addInputChannelRequest = 50; @@ -57,7 +58,7 @@ message ControlRequest { EmptyRequest emptyRequest = 56; PrepareCheckpointRequest prepareCheckpointRequest = 57; QueryStatisticsRequest queryStatisticsRequest = 58; - EndIterationRequest endIterationRequest = 59; + // request for testing Ping ping = 100; @@ -281,7 +282,7 @@ message QueryStatisticsRequest{ StatisticsUpdateTarget updateTarget = 2; } -message EndIterationRequest{ - core.ActorVirtualIdentity LoopStartId = 1 [(scalapb.field).no_box = true]; - int32 iteration = 2; +message IterationCompletedRequest{ + core.OperatorIdentity LoopStartId = 1 [(scalapb.field).no_box = true]; + bool next = 2; } \ No newline at end of file diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto index 70d189a3411..2f6066c5713 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto @@ -42,6 +42,7 @@ service ControllerService { rpc PauseWorkflow(EmptyRequest) returns (EmptyReturn); rpc WorkerStateUpdated(WorkerStateUpdatedRequest) returns (EmptyReturn); rpc WorkerExecutionCompleted(EmptyRequest) returns (EmptyReturn); + rpc IterationCompleted(IterationCompletedRequest) returns (EmptyReturn); rpc LinkWorkers(LinkWorkersRequest) returns (EmptyReturn); rpc ControllerInitiateQueryStatistics(QueryStatisticsRequest) returns (EmptyReturn); rpc RetryWorkflow(RetryWorkflowRequest) returns (EmptyReturn); diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto index a2c4d6ef9b7..dbcd6d8a5e0 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto @@ -47,7 +47,6 @@ service WorkerService { rpc EndWorker(EmptyRequest) returns (EmptyReturn); rpc StartChannel(EmptyRequest) returns (EmptyReturn); rpc EndChannel(EmptyRequest) returns (EmptyReturn); - rpc EndIteration(EndIterationRequest) returns (EmptyReturn); rpc DebugCommand(DebugCommandRequest) returns (EmptyReturn); rpc EvaluatePythonExpression(EvaluatePythonExpressionRequest) returns (EvaluatedValue); rpc NoOperation(EmptyRequest) returns (EmptyReturn); diff --git a/amber/src/main/python/core/models/operator.py b/amber/src/main/python/core/models/operator.py index 7532fae1bd7..3e36a0ff359 100644 --- a/amber/src/main/python/core/models/operator.py +++ b/amber/src/main/python/core/models/operator.py @@ -317,3 +317,9 @@ def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike] def close(self) -> None: pass + + def condition(self): + return self.state["condition"] + + def loop_start_id(self): + return self.state["LoopStartId"] diff --git a/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py b/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py index ea6ddc5e43f..245f157ab2d 100644 --- a/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py +++ b/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py @@ -43,6 +43,12 @@ class ConsoleMessageType(betterproto.Enum): DEBUGGER = 3 +class StatisticsUpdateTarget(betterproto.Enum): + BOTH_UI_AND_PERSISTENCE = 0 + UI_ONLY = 1 + PERSISTENCE_ONLY = 2 + + class ErrorLanguage(betterproto.Enum): PYTHON = 0 SCALA = 1 @@ -96,6 +102,9 @@ class ControlRequest(betterproto.Message): link_workers_request: "LinkWorkersRequest" = betterproto.message_field( 11, group="sealed_value" ) + iteration_completed_request: "IterationCompletedRequest" = ( + betterproto.message_field(12, group="sealed_value") + ) add_input_channel_request: "AddInputChannelRequest" = betterproto.message_field( 50, group="sealed_value" ) @@ -385,6 +394,13 @@ class QueryStatisticsRequest(betterproto.Message): filter_by_workers: List["___core__.ActorVirtualIdentity"] = ( betterproto.message_field(1) ) + update_target: "StatisticsUpdateTarget" = betterproto.enum_field(2) + + +@dataclass(eq=False, repr=False) +class IterationCompletedRequest(betterproto.Message): + loop_start_id: "___core__.OperatorIdentity" = betterproto.message_field(1) + next: bool = betterproto.bool_field(2) @dataclass(eq=False, repr=False) @@ -517,503 +533,522 @@ class WorkerMetricsResponse(betterproto.Message): metrics: "_worker__.WorkerMetrics" = betterproto.message_field(1) -class RpcTesterStub(betterproto.ServiceStub): - async def send_ping( +class ControllerServiceStub(betterproto.ServiceStub): + async def retrieve_workflow_state( self, - ping: "Ping", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "IntResponse": + ) -> "RetrieveWorkflowStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing", - ping, - IntResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState", + empty_request, + RetrieveWorkflowStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_pong( + async def propagate_embedded_control_message( self, - pong: "Pong", + propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "IntResponse": + ) -> "PropagateEmbeddedControlMessageResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong", - pong, - IntResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage", + propagate_embedded_control_message_request, + PropagateEmbeddedControlMessageResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_nested( + async def take_global_checkpoint( self, - nested: "Nested", + take_global_checkpoint_request: "TakeGlobalCheckpointRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "TakeGlobalCheckpointResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested", - nested, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint", + take_global_checkpoint_request, + TakeGlobalCheckpointResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_pass( + async def debug_command( self, - pass_: "Pass", + debug_command_request: "DebugCommandRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass", - pass_, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand", + debug_command_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_error_command( + async def evaluate_python_expression( self, - error_command: "ErrorCommand", + evaluate_python_expression_request: "EvaluatePythonExpressionRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EvaluatePythonExpressionResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand", - error_command, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression", + evaluate_python_expression_request, + EvaluatePythonExpressionResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_recursion( + async def console_message_triggered( self, - recursion: "Recursion", + console_message_triggered_request: "ConsoleMessageTriggeredRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion", - recursion, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered", + console_message_triggered_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_collect( + async def port_completed( self, - collect: "Collect", + port_completed_request: "PortCompletedRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect", - collect, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted", + port_completed_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_generate_number( + async def start_workflow( self, - generate_number: "GenerateNumber", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "IntResponse": + ) -> "StartWorkflowResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber", - generate_number, - IntResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow", + empty_request, + StartWorkflowResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_multi_call( + async def resume_workflow( self, - multi_call: "MultiCall", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall", - multi_call, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_chain( + async def pause_workflow( self, - chain: "Chain", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain", - chain, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - -class WorkerServiceStub(betterproto.ServiceStub): - async def add_input_channel( + async def worker_state_updated( self, - add_input_channel_request: "AddInputChannelRequest", + worker_state_updated_request: "WorkerStateUpdatedRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel", - add_input_channel_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated", + worker_state_updated_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def add_partitioning( + async def worker_execution_completed( self, - add_partitioning_request: "AddPartitioningRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning", - add_partitioning_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def assign_port( + async def iteration_completed( self, - assign_port_request: "AssignPortRequest", + iteration_completed_request: "IterationCompletedRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort", - assign_port_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/IterationCompleted", + iteration_completed_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def finalize_checkpoint( + async def link_workers( self, - finalize_checkpoint_request: "FinalizeCheckpointRequest", + link_workers_request: "LinkWorkersRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "FinalizeCheckpointResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint", - finalize_checkpoint_request, - FinalizeCheckpointResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers", + link_workers_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def flush_network_buffer( + async def controller_initiate_query_statistics( self, - empty_request: "EmptyRequest", + query_statistics_request: "QueryStatisticsRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics", + query_statistics_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def initialize_executor( + async def retry_workflow( self, - initialize_executor_request: "InitializeExecutorRequest", + retry_workflow_request: "RetryWorkflowRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor", - initialize_executor_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow", + retry_workflow_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def open_executor( + +class RpcTesterStub(betterproto.ServiceStub): + async def send_ping( self, - empty_request: "EmptyRequest", + ping: "Ping", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "IntResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing", + ping, + IntResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def pause_worker( + async def send_pong( self, - empty_request: "EmptyRequest", + pong: "Pong", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerStateResponse": + ) -> "IntResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker", - empty_request, - WorkerStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong", + pong, + IntResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def prepare_checkpoint( + async def send_nested( self, - prepare_checkpoint_request: "PrepareCheckpointRequest", + nested: "Nested", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint", - prepare_checkpoint_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested", + nested, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def query_statistics( + async def send_pass( self, - empty_request: "EmptyRequest", + pass_: "Pass", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerMetricsResponse": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics", - empty_request, - WorkerMetricsResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass", + pass_, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def resume_worker( + async def send_error_command( self, - empty_request: "EmptyRequest", + error_command: "ErrorCommand", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerStateResponse": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker", - empty_request, - WorkerStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand", + error_command, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def retrieve_state( + async def send_recursion( self, - empty_request: "EmptyRequest", + recursion: "Recursion", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion", + recursion, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def retry_current_tuple( + async def send_collect( self, - empty_request: "EmptyRequest", + collect: "Collect", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect", + collect, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def start_worker( + async def send_generate_number( self, - empty_request: "EmptyRequest", + generate_number: "GenerateNumber", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerStateResponse": + ) -> "IntResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker", - empty_request, - WorkerStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber", + generate_number, + IntResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def end_worker( + async def send_multi_call( self, - empty_request: "EmptyRequest", + multi_call: "MultiCall", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall", + multi_call, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def start_channel( + async def send_chain( self, - empty_request: "EmptyRequest", + chain: "Chain", + *, + timeout: Optional[float] = None, + deadline: Optional["Deadline"] = None, + metadata: Optional["MetadataLike"] = None + ) -> "StringResponse": + return await self._unary_unary( + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain", + chain, + StringResponse, + timeout=timeout, + deadline=deadline, + metadata=metadata, + ) + + +class WorkerServiceStub(betterproto.ServiceStub): + async def add_input_channel( + self, + add_input_channel_request: "AddInputChannelRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel", + add_input_channel_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def end_channel( + async def add_partitioning( self, - empty_request: "EmptyRequest", + add_partitioning_request: "AddPartitioningRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning", + add_partitioning_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def debug_command( + async def assign_port( self, - debug_command_request: "DebugCommandRequest", + assign_port_request: "AssignPortRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand", - debug_command_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort", + assign_port_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def evaluate_python_expression( + async def finalize_checkpoint( self, - evaluate_python_expression_request: "EvaluatePythonExpressionRequest", + finalize_checkpoint_request: "FinalizeCheckpointRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EvaluatedValue": + ) -> "FinalizeCheckpointResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression", - evaluate_python_expression_request, - EvaluatedValue, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint", + finalize_checkpoint_request, + FinalizeCheckpointResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def no_operation( + async def flush_network_buffer( self, empty_request: "EmptyRequest", *, @@ -1022,7 +1057,7 @@ async def no_operation( metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer", empty_request, EmptyReturn, timeout=timeout, @@ -1030,162 +1065,160 @@ async def no_operation( metadata=metadata, ) - -class ControllerServiceStub(betterproto.ServiceStub): - async def retrieve_workflow_state( + async def initialize_executor( self, - empty_request: "EmptyRequest", + initialize_executor_request: "InitializeExecutorRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "RetrieveWorkflowStateResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState", - empty_request, - RetrieveWorkflowStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor", + initialize_executor_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def propagate_embedded_control_message( + async def open_executor( self, - propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "PropagateEmbeddedControlMessageResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage", - propagate_embedded_control_message_request, - PropagateEmbeddedControlMessageResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def take_global_checkpoint( + async def pause_worker( self, - take_global_checkpoint_request: "TakeGlobalCheckpointRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "TakeGlobalCheckpointResponse": + ) -> "WorkerStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint", - take_global_checkpoint_request, - TakeGlobalCheckpointResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker", + empty_request, + WorkerStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def debug_command( + async def prepare_checkpoint( self, - debug_command_request: "DebugCommandRequest", + prepare_checkpoint_request: "PrepareCheckpointRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand", - debug_command_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint", + prepare_checkpoint_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def evaluate_python_expression( + async def query_statistics( self, - evaluate_python_expression_request: "EvaluatePythonExpressionRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EvaluatePythonExpressionResponse": + ) -> "WorkerMetricsResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression", - evaluate_python_expression_request, - EvaluatePythonExpressionResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics", + empty_request, + WorkerMetricsResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def console_message_triggered( + async def resume_worker( self, - console_message_triggered_request: "ConsoleMessageTriggeredRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "WorkerStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered", - console_message_triggered_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker", + empty_request, + WorkerStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def port_completed( + async def retrieve_state( self, - port_completed_request: "PortCompletedRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted", - port_completed_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def start_workflow( + async def retry_current_tuple( self, empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StartWorkflowResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple", empty_request, - StartWorkflowResponse, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def resume_workflow( + async def start_worker( self, empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "WorkerStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker", empty_request, - EmptyReturn, + WorkerStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def pause_workflow( + async def end_worker( self, empty_request: "EmptyRequest", *, @@ -1194,7 +1227,7 @@ async def pause_workflow( metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker", empty_request, EmptyReturn, timeout=timeout, @@ -1202,24 +1235,24 @@ async def pause_workflow( metadata=metadata, ) - async def worker_state_updated( + async def start_channel( self, - worker_state_updated_request: "WorkerStateUpdatedRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated", - worker_state_updated_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def worker_execution_completed( + async def end_channel( self, empty_request: "EmptyRequest", *, @@ -1228,7 +1261,7 @@ async def worker_execution_completed( metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel", empty_request, EmptyReturn, timeout=timeout, @@ -1236,51 +1269,51 @@ async def worker_execution_completed( metadata=metadata, ) - async def link_workers( + async def debug_command( self, - link_workers_request: "LinkWorkersRequest", + debug_command_request: "DebugCommandRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers", - link_workers_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand", + debug_command_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def controller_initiate_query_statistics( + async def evaluate_python_expression( self, - query_statistics_request: "QueryStatisticsRequest", + evaluate_python_expression_request: "EvaluatePythonExpressionRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "EvaluatedValue": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics", - query_statistics_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression", + evaluate_python_expression_request, + EvaluatedValue, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def retry_workflow( + async def no_operation( self, - retry_workflow_request: "RetryWorkflowRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow", - retry_workflow_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, @@ -1288,806 +1321,824 @@ async def retry_workflow( ) -class RpcTesterBase(ServiceBase): +class ControllerServiceBase(ServiceBase): - async def send_ping(self, ping: "Ping") -> "IntResponse": + async def retrieve_workflow_state( + self, empty_request: "EmptyRequest" + ) -> "RetrieveWorkflowStateResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_pong(self, pong: "Pong") -> "IntResponse": + async def propagate_embedded_control_message( + self, + propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", + ) -> "PropagateEmbeddedControlMessageResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_nested(self, nested: "Nested") -> "StringResponse": + async def take_global_checkpoint( + self, take_global_checkpoint_request: "TakeGlobalCheckpointRequest" + ) -> "TakeGlobalCheckpointResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_pass(self, pass_: "Pass") -> "StringResponse": + async def debug_command( + self, debug_command_request: "DebugCommandRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_error_command( - self, error_command: "ErrorCommand" - ) -> "StringResponse": + async def evaluate_python_expression( + self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" + ) -> "EvaluatePythonExpressionResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_recursion(self, recursion: "Recursion") -> "StringResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + async def console_message_triggered( + self, console_message_triggered_request: "ConsoleMessageTriggeredRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_collect(self, collect: "Collect") -> "StringResponse": + async def port_completed( + self, port_completed_request: "PortCompletedRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_generate_number( - self, generate_number: "GenerateNumber" - ) -> "IntResponse": + async def start_workflow( + self, empty_request: "EmptyRequest" + ) -> "StartWorkflowResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_multi_call(self, multi_call: "MultiCall") -> "StringResponse": + async def resume_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_chain(self, chain: "Chain") -> "StringResponse": + async def pause_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_send_ping( - self, stream: "grpclib.server.Stream[Ping, IntResponse]" + async def worker_state_updated( + self, worker_state_updated_request: "WorkerStateUpdatedRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def worker_execution_completed( + self, empty_request: "EmptyRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def iteration_completed( + self, iteration_completed_request: "IterationCompletedRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def link_workers( + self, link_workers_request: "LinkWorkersRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def controller_initiate_query_statistics( + self, query_statistics_request: "QueryStatisticsRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def retry_workflow( + self, retry_workflow_request: "RetryWorkflowRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def __rpc_retrieve_workflow_state( + self, + stream: "grpclib.server.Stream[EmptyRequest, RetrieveWorkflowStateResponse]", ) -> None: request = await stream.recv_message() - response = await self.send_ping(request) + response = await self.retrieve_workflow_state(request) await stream.send_message(response) - async def __rpc_send_pong( - self, stream: "grpclib.server.Stream[Pong, IntResponse]" + async def __rpc_propagate_embedded_control_message( + self, + stream: "grpclib.server.Stream[PropagateEmbeddedControlMessageRequest, PropagateEmbeddedControlMessageResponse]", ) -> None: request = await stream.recv_message() - response = await self.send_pong(request) + response = await self.propagate_embedded_control_message(request) await stream.send_message(response) - async def __rpc_send_nested( - self, stream: "grpclib.server.Stream[Nested, StringResponse]" + async def __rpc_take_global_checkpoint( + self, + stream: "grpclib.server.Stream[TakeGlobalCheckpointRequest, TakeGlobalCheckpointResponse]", ) -> None: request = await stream.recv_message() - response = await self.send_nested(request) + response = await self.take_global_checkpoint(request) await stream.send_message(response) - async def __rpc_send_pass( - self, stream: "grpclib.server.Stream[Pass, StringResponse]" + async def __rpc_debug_command( + self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_pass(request) + response = await self.debug_command(request) await stream.send_message(response) - async def __rpc_send_error_command( - self, stream: "grpclib.server.Stream[ErrorCommand, StringResponse]" + async def __rpc_evaluate_python_expression( + self, + stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatePythonExpressionResponse]", ) -> None: request = await stream.recv_message() - response = await self.send_error_command(request) + response = await self.evaluate_python_expression(request) await stream.send_message(response) - async def __rpc_send_recursion( - self, stream: "grpclib.server.Stream[Recursion, StringResponse]" + async def __rpc_console_message_triggered( + self, + stream: "grpclib.server.Stream[ConsoleMessageTriggeredRequest, EmptyReturn]", ) -> None: request = await stream.recv_message() - response = await self.send_recursion(request) + response = await self.console_message_triggered(request) await stream.send_message(response) - async def __rpc_send_collect( - self, stream: "grpclib.server.Stream[Collect, StringResponse]" + async def __rpc_port_completed( + self, stream: "grpclib.server.Stream[PortCompletedRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_collect(request) + response = await self.port_completed(request) await stream.send_message(response) - async def __rpc_send_generate_number( - self, stream: "grpclib.server.Stream[GenerateNumber, IntResponse]" + async def __rpc_start_workflow( + self, stream: "grpclib.server.Stream[EmptyRequest, StartWorkflowResponse]" ) -> None: request = await stream.recv_message() - response = await self.send_generate_number(request) + response = await self.start_workflow(request) await stream.send_message(response) - async def __rpc_send_multi_call( - self, stream: "grpclib.server.Stream[MultiCall, StringResponse]" + async def __rpc_resume_workflow( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_multi_call(request) + response = await self.resume_workflow(request) await stream.send_message(response) - async def __rpc_send_chain( - self, stream: "grpclib.server.Stream[Chain, StringResponse]" + async def __rpc_pause_workflow( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_chain(request) + response = await self.pause_workflow(request) + await stream.send_message(response) + + async def __rpc_worker_state_updated( + self, stream: "grpclib.server.Stream[WorkerStateUpdatedRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.worker_state_updated(request) + await stream.send_message(response) + + async def __rpc_worker_execution_completed( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.worker_execution_completed(request) + await stream.send_message(response) + + async def __rpc_iteration_completed( + self, stream: "grpclib.server.Stream[IterationCompletedRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.iteration_completed(request) + await stream.send_message(response) + + async def __rpc_link_workers( + self, stream: "grpclib.server.Stream[LinkWorkersRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.link_workers(request) + await stream.send_message(response) + + async def __rpc_controller_initiate_query_statistics( + self, stream: "grpclib.server.Stream[QueryStatisticsRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.controller_initiate_query_statistics(request) + await stream.send_message(response) + + async def __rpc_retry_workflow( + self, stream: "grpclib.server.Stream[RetryWorkflowRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.retry_workflow(request) await stream.send_message(response) def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing": grpclib.const.Handler( - self.__rpc_send_ping, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState": grpclib.const.Handler( + self.__rpc_retrieve_workflow_state, grpclib.const.Cardinality.UNARY_UNARY, - Ping, - IntResponse, + EmptyRequest, + RetrieveWorkflowStateResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong": grpclib.const.Handler( - self.__rpc_send_pong, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage": grpclib.const.Handler( + self.__rpc_propagate_embedded_control_message, grpclib.const.Cardinality.UNARY_UNARY, - Pong, - IntResponse, + PropagateEmbeddedControlMessageRequest, + PropagateEmbeddedControlMessageResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested": grpclib.const.Handler( - self.__rpc_send_nested, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint": grpclib.const.Handler( + self.__rpc_take_global_checkpoint, grpclib.const.Cardinality.UNARY_UNARY, - Nested, - StringResponse, + TakeGlobalCheckpointRequest, + TakeGlobalCheckpointResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass": grpclib.const.Handler( - self.__rpc_send_pass, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand": grpclib.const.Handler( + self.__rpc_debug_command, grpclib.const.Cardinality.UNARY_UNARY, - Pass, - StringResponse, + DebugCommandRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand": grpclib.const.Handler( - self.__rpc_send_error_command, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression": grpclib.const.Handler( + self.__rpc_evaluate_python_expression, grpclib.const.Cardinality.UNARY_UNARY, - ErrorCommand, - StringResponse, + EvaluatePythonExpressionRequest, + EvaluatePythonExpressionResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion": grpclib.const.Handler( - self.__rpc_send_recursion, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered": grpclib.const.Handler( + self.__rpc_console_message_triggered, grpclib.const.Cardinality.UNARY_UNARY, - Recursion, - StringResponse, + ConsoleMessageTriggeredRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect": grpclib.const.Handler( - self.__rpc_send_collect, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted": grpclib.const.Handler( + self.__rpc_port_completed, grpclib.const.Cardinality.UNARY_UNARY, - Collect, - StringResponse, + PortCompletedRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber": grpclib.const.Handler( - self.__rpc_send_generate_number, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow": grpclib.const.Handler( + self.__rpc_start_workflow, grpclib.const.Cardinality.UNARY_UNARY, - GenerateNumber, - IntResponse, + EmptyRequest, + StartWorkflowResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall": grpclib.const.Handler( - self.__rpc_send_multi_call, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow": grpclib.const.Handler( + self.__rpc_resume_workflow, grpclib.const.Cardinality.UNARY_UNARY, - MultiCall, - StringResponse, + EmptyRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain": grpclib.const.Handler( - self.__rpc_send_chain, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow": grpclib.const.Handler( + self.__rpc_pause_workflow, grpclib.const.Cardinality.UNARY_UNARY, - Chain, - StringResponse, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated": grpclib.const.Handler( + self.__rpc_worker_state_updated, + grpclib.const.Cardinality.UNARY_UNARY, + WorkerStateUpdatedRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted": grpclib.const.Handler( + self.__rpc_worker_execution_completed, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/IterationCompleted": grpclib.const.Handler( + self.__rpc_iteration_completed, + grpclib.const.Cardinality.UNARY_UNARY, + IterationCompletedRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers": grpclib.const.Handler( + self.__rpc_link_workers, + grpclib.const.Cardinality.UNARY_UNARY, + LinkWorkersRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics": grpclib.const.Handler( + self.__rpc_controller_initiate_query_statistics, + grpclib.const.Cardinality.UNARY_UNARY, + QueryStatisticsRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow": grpclib.const.Handler( + self.__rpc_retry_workflow, + grpclib.const.Cardinality.UNARY_UNARY, + RetryWorkflowRequest, + EmptyReturn, ), } -class WorkerServiceBase(ServiceBase): - - async def add_input_channel( - self, add_input_channel_request: "AddInputChannelRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def add_partitioning( - self, add_partitioning_request: "AddPartitioningRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def assign_port( - self, assign_port_request: "AssignPortRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def finalize_checkpoint( - self, finalize_checkpoint_request: "FinalizeCheckpointRequest" - ) -> "FinalizeCheckpointResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def flush_network_buffer( - self, empty_request: "EmptyRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def initialize_executor( - self, initialize_executor_request: "InitializeExecutorRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def open_executor(self, empty_request: "EmptyRequest") -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def pause_worker( - self, empty_request: "EmptyRequest" - ) -> "WorkerStateResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def prepare_checkpoint( - self, prepare_checkpoint_request: "PrepareCheckpointRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def query_statistics( - self, empty_request: "EmptyRequest" - ) -> "WorkerMetricsResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) +class RpcTesterBase(ServiceBase): - async def resume_worker( - self, empty_request: "EmptyRequest" - ) -> "WorkerStateResponse": + async def send_ping(self, ping: "Ping") -> "IntResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def retrieve_state(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_pong(self, pong: "Pong") -> "IntResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def retry_current_tuple(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_nested(self, nested: "Nested") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def start_worker( - self, empty_request: "EmptyRequest" - ) -> "WorkerStateResponse": + async def send_pass(self, pass_: "Pass") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def end_worker(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_error_command( + self, error_command: "ErrorCommand" + ) -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def start_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_recursion(self, recursion: "Recursion") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def end_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_collect(self, collect: "Collect") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def debug_command( - self, debug_command_request: "DebugCommandRequest" - ) -> "EmptyReturn": + async def send_generate_number( + self, generate_number: "GenerateNumber" + ) -> "IntResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def evaluate_python_expression( - self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" - ) -> "EvaluatedValue": + async def send_multi_call(self, multi_call: "MultiCall") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def no_operation(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_chain(self, chain: "Chain") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_add_input_channel( - self, stream: "grpclib.server.Stream[AddInputChannelRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.add_input_channel(request) - await stream.send_message(response) - - async def __rpc_add_partitioning( - self, stream: "grpclib.server.Stream[AddPartitioningRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.add_partitioning(request) - await stream.send_message(response) - - async def __rpc_assign_port( - self, stream: "grpclib.server.Stream[AssignPortRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.assign_port(request) - await stream.send_message(response) - - async def __rpc_finalize_checkpoint( - self, - stream: "grpclib.server.Stream[FinalizeCheckpointRequest, FinalizeCheckpointResponse]", - ) -> None: - request = await stream.recv_message() - response = await self.finalize_checkpoint(request) - await stream.send_message(response) - - async def __rpc_flush_network_buffer( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.flush_network_buffer(request) - await stream.send_message(response) - - async def __rpc_initialize_executor( - self, stream: "grpclib.server.Stream[InitializeExecutorRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.initialize_executor(request) - await stream.send_message(response) - - async def __rpc_open_executor( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.open_executor(request) - await stream.send_message(response) - - async def __rpc_pause_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" - ) -> None: - request = await stream.recv_message() - response = await self.pause_worker(request) - await stream.send_message(response) - - async def __rpc_prepare_checkpoint( - self, stream: "grpclib.server.Stream[PrepareCheckpointRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.prepare_checkpoint(request) - await stream.send_message(response) - - async def __rpc_query_statistics( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerMetricsResponse]" - ) -> None: - request = await stream.recv_message() - response = await self.query_statistics(request) - await stream.send_message(response) - - async def __rpc_resume_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + async def __rpc_send_ping( + self, stream: "grpclib.server.Stream[Ping, IntResponse]" ) -> None: request = await stream.recv_message() - response = await self.resume_worker(request) + response = await self.send_ping(request) await stream.send_message(response) - async def __rpc_retrieve_state( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_pong( + self, stream: "grpclib.server.Stream[Pong, IntResponse]" ) -> None: request = await stream.recv_message() - response = await self.retrieve_state(request) + response = await self.send_pong(request) await stream.send_message(response) - async def __rpc_retry_current_tuple( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_nested( + self, stream: "grpclib.server.Stream[Nested, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.retry_current_tuple(request) + response = await self.send_nested(request) await stream.send_message(response) - async def __rpc_start_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + async def __rpc_send_pass( + self, stream: "grpclib.server.Stream[Pass, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.start_worker(request) + response = await self.send_pass(request) await stream.send_message(response) - - async def __rpc_end_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + + async def __rpc_send_error_command( + self, stream: "grpclib.server.Stream[ErrorCommand, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.end_worker(request) + response = await self.send_error_command(request) await stream.send_message(response) - async def __rpc_start_channel( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_recursion( + self, stream: "grpclib.server.Stream[Recursion, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.start_channel(request) + response = await self.send_recursion(request) await stream.send_message(response) - async def __rpc_end_channel( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_collect( + self, stream: "grpclib.server.Stream[Collect, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.end_channel(request) + response = await self.send_collect(request) await stream.send_message(response) - async def __rpc_debug_command( - self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" + async def __rpc_send_generate_number( + self, stream: "grpclib.server.Stream[GenerateNumber, IntResponse]" ) -> None: request = await stream.recv_message() - response = await self.debug_command(request) + response = await self.send_generate_number(request) await stream.send_message(response) - async def __rpc_evaluate_python_expression( - self, - stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatedValue]", + async def __rpc_send_multi_call( + self, stream: "grpclib.server.Stream[MultiCall, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.evaluate_python_expression(request) + response = await self.send_multi_call(request) await stream.send_message(response) - async def __rpc_no_operation( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_chain( + self, stream: "grpclib.server.Stream[Chain, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.no_operation(request) + response = await self.send_chain(request) await stream.send_message(response) def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel": grpclib.const.Handler( - self.__rpc_add_input_channel, - grpclib.const.Cardinality.UNARY_UNARY, - AddInputChannelRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning": grpclib.const.Handler( - self.__rpc_add_partitioning, - grpclib.const.Cardinality.UNARY_UNARY, - AddPartitioningRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort": grpclib.const.Handler( - self.__rpc_assign_port, - grpclib.const.Cardinality.UNARY_UNARY, - AssignPortRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint": grpclib.const.Handler( - self.__rpc_finalize_checkpoint, - grpclib.const.Cardinality.UNARY_UNARY, - FinalizeCheckpointRequest, - FinalizeCheckpointResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer": grpclib.const.Handler( - self.__rpc_flush_network_buffer, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor": grpclib.const.Handler( - self.__rpc_initialize_executor, - grpclib.const.Cardinality.UNARY_UNARY, - InitializeExecutorRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor": grpclib.const.Handler( - self.__rpc_open_executor, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker": grpclib.const.Handler( - self.__rpc_pause_worker, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerStateResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint": grpclib.const.Handler( - self.__rpc_prepare_checkpoint, - grpclib.const.Cardinality.UNARY_UNARY, - PrepareCheckpointRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics": grpclib.const.Handler( - self.__rpc_query_statistics, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerMetricsResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker": grpclib.const.Handler( - self.__rpc_resume_worker, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing": grpclib.const.Handler( + self.__rpc_send_ping, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerStateResponse, + Ping, + IntResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState": grpclib.const.Handler( - self.__rpc_retrieve_state, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong": grpclib.const.Handler( + self.__rpc_send_pong, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Pong, + IntResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple": grpclib.const.Handler( - self.__rpc_retry_current_tuple, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested": grpclib.const.Handler( + self.__rpc_send_nested, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Nested, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker": grpclib.const.Handler( - self.__rpc_start_worker, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass": grpclib.const.Handler( + self.__rpc_send_pass, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerStateResponse, + Pass, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker": grpclib.const.Handler( - self.__rpc_end_worker, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand": grpclib.const.Handler( + self.__rpc_send_error_command, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + ErrorCommand, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel": grpclib.const.Handler( - self.__rpc_start_channel, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion": grpclib.const.Handler( + self.__rpc_send_recursion, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Recursion, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel": grpclib.const.Handler( - self.__rpc_end_channel, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect": grpclib.const.Handler( + self.__rpc_send_collect, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Collect, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand": grpclib.const.Handler( - self.__rpc_debug_command, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber": grpclib.const.Handler( + self.__rpc_send_generate_number, grpclib.const.Cardinality.UNARY_UNARY, - DebugCommandRequest, - EmptyReturn, + GenerateNumber, + IntResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression": grpclib.const.Handler( - self.__rpc_evaluate_python_expression, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall": grpclib.const.Handler( + self.__rpc_send_multi_call, grpclib.const.Cardinality.UNARY_UNARY, - EvaluatePythonExpressionRequest, - EvaluatedValue, + MultiCall, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation": grpclib.const.Handler( - self.__rpc_no_operation, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain": grpclib.const.Handler( + self.__rpc_send_chain, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Chain, + StringResponse, ), } -class ControllerServiceBase(ServiceBase): +class WorkerServiceBase(ServiceBase): - async def retrieve_workflow_state( - self, empty_request: "EmptyRequest" - ) -> "RetrieveWorkflowStateResponse": + async def add_input_channel( + self, add_input_channel_request: "AddInputChannelRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def propagate_embedded_control_message( - self, - propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", - ) -> "PropagateEmbeddedControlMessageResponse": + async def add_partitioning( + self, add_partitioning_request: "AddPartitioningRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def take_global_checkpoint( - self, take_global_checkpoint_request: "TakeGlobalCheckpointRequest" - ) -> "TakeGlobalCheckpointResponse": + async def assign_port( + self, assign_port_request: "AssignPortRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def debug_command( - self, debug_command_request: "DebugCommandRequest" - ) -> "EmptyReturn": + async def finalize_checkpoint( + self, finalize_checkpoint_request: "FinalizeCheckpointRequest" + ) -> "FinalizeCheckpointResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def evaluate_python_expression( - self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" - ) -> "EvaluatePythonExpressionResponse": + async def flush_network_buffer( + self, empty_request: "EmptyRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def console_message_triggered( - self, console_message_triggered_request: "ConsoleMessageTriggeredRequest" + async def initialize_executor( + self, initialize_executor_request: "InitializeExecutorRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def port_completed( - self, port_completed_request: "PortCompletedRequest" + async def open_executor(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def pause_worker( + self, empty_request: "EmptyRequest" + ) -> "WorkerStateResponse": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def prepare_checkpoint( + self, prepare_checkpoint_request: "PrepareCheckpointRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def start_workflow( + async def query_statistics( self, empty_request: "EmptyRequest" - ) -> "StartWorkflowResponse": + ) -> "WorkerMetricsResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def resume_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def resume_worker( + self, empty_request: "EmptyRequest" + ) -> "WorkerStateResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def pause_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def retrieve_state(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def worker_state_updated( - self, worker_state_updated_request: "WorkerStateUpdatedRequest" - ) -> "EmptyReturn": + async def retry_current_tuple(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def worker_execution_completed( + async def start_worker( self, empty_request: "EmptyRequest" - ) -> "EmptyReturn": + ) -> "WorkerStateResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def link_workers( - self, link_workers_request: "LinkWorkersRequest" + async def end_worker(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def start_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def end_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def debug_command( + self, debug_command_request: "DebugCommandRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def controller_initiate_query_statistics( - self, query_statistics_request: "QueryStatisticsRequest" - ) -> "EmptyReturn": + async def evaluate_python_expression( + self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" + ) -> "EvaluatedValue": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def retry_workflow( - self, retry_workflow_request: "RetryWorkflowRequest" - ) -> "EmptyReturn": + async def no_operation(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_retrieve_workflow_state( - self, - stream: "grpclib.server.Stream[EmptyRequest, RetrieveWorkflowStateResponse]", + async def __rpc_add_input_channel( + self, stream: "grpclib.server.Stream[AddInputChannelRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.retrieve_workflow_state(request) + response = await self.add_input_channel(request) await stream.send_message(response) - async def __rpc_propagate_embedded_control_message( - self, - stream: "grpclib.server.Stream[PropagateEmbeddedControlMessageRequest, PropagateEmbeddedControlMessageResponse]", + async def __rpc_add_partitioning( + self, stream: "grpclib.server.Stream[AddPartitioningRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.propagate_embedded_control_message(request) + response = await self.add_partitioning(request) await stream.send_message(response) - async def __rpc_take_global_checkpoint( + async def __rpc_assign_port( + self, stream: "grpclib.server.Stream[AssignPortRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.assign_port(request) + await stream.send_message(response) + + async def __rpc_finalize_checkpoint( self, - stream: "grpclib.server.Stream[TakeGlobalCheckpointRequest, TakeGlobalCheckpointResponse]", + stream: "grpclib.server.Stream[FinalizeCheckpointRequest, FinalizeCheckpointResponse]", ) -> None: request = await stream.recv_message() - response = await self.take_global_checkpoint(request) + response = await self.finalize_checkpoint(request) await stream.send_message(response) - async def __rpc_debug_command( - self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" + async def __rpc_flush_network_buffer( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.debug_command(request) + response = await self.flush_network_buffer(request) await stream.send_message(response) - async def __rpc_evaluate_python_expression( - self, - stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatePythonExpressionResponse]", + async def __rpc_initialize_executor( + self, stream: "grpclib.server.Stream[InitializeExecutorRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.evaluate_python_expression(request) + response = await self.initialize_executor(request) await stream.send_message(response) - async def __rpc_console_message_triggered( - self, - stream: "grpclib.server.Stream[ConsoleMessageTriggeredRequest, EmptyReturn]", + async def __rpc_open_executor( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.console_message_triggered(request) + response = await self.open_executor(request) await stream.send_message(response) - async def __rpc_port_completed( - self, stream: "grpclib.server.Stream[PortCompletedRequest, EmptyReturn]" + async def __rpc_pause_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" ) -> None: request = await stream.recv_message() - response = await self.port_completed(request) + response = await self.pause_worker(request) await stream.send_message(response) - async def __rpc_start_workflow( - self, stream: "grpclib.server.Stream[EmptyRequest, StartWorkflowResponse]" + async def __rpc_prepare_checkpoint( + self, stream: "grpclib.server.Stream[PrepareCheckpointRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.start_workflow(request) + response = await self.prepare_checkpoint(request) await stream.send_message(response) - async def __rpc_resume_workflow( + async def __rpc_query_statistics( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerMetricsResponse]" + ) -> None: + request = await stream.recv_message() + response = await self.query_statistics(request) + await stream.send_message(response) + + async def __rpc_resume_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + ) -> None: + request = await stream.recv_message() + response = await self.resume_worker(request) + await stream.send_message(response) + + async def __rpc_retrieve_state( self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.resume_workflow(request) + response = await self.retrieve_state(request) await stream.send_message(response) - async def __rpc_pause_workflow( + async def __rpc_retry_current_tuple( self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.pause_workflow(request) + response = await self.retry_current_tuple(request) await stream.send_message(response) - async def __rpc_worker_state_updated( - self, stream: "grpclib.server.Stream[WorkerStateUpdatedRequest, EmptyReturn]" + async def __rpc_start_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" ) -> None: request = await stream.recv_message() - response = await self.worker_state_updated(request) + response = await self.start_worker(request) await stream.send_message(response) - async def __rpc_worker_execution_completed( + async def __rpc_end_worker( self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.worker_execution_completed(request) + response = await self.end_worker(request) await stream.send_message(response) - async def __rpc_link_workers( - self, stream: "grpclib.server.Stream[LinkWorkersRequest, EmptyReturn]" + async def __rpc_start_channel( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.link_workers(request) + response = await self.start_channel(request) await stream.send_message(response) - async def __rpc_controller_initiate_query_statistics( - self, stream: "grpclib.server.Stream[QueryStatisticsRequest, EmptyReturn]" + async def __rpc_end_channel( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.controller_initiate_query_statistics(request) + response = await self.end_channel(request) await stream.send_message(response) - async def __rpc_retry_workflow( - self, stream: "grpclib.server.Stream[RetryWorkflowRequest, EmptyReturn]" + async def __rpc_debug_command( + self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.retry_workflow(request) + response = await self.debug_command(request) + await stream.send_message(response) + + async def __rpc_evaluate_python_expression( + self, + stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatedValue]", + ) -> None: + request = await stream.recv_message() + response = await self.evaluate_python_expression(request) + await stream.send_message(response) + + async def __rpc_no_operation( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.no_operation(request) await stream.send_message(response) def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState": grpclib.const.Handler( - self.__rpc_retrieve_workflow_state, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel": grpclib.const.Handler( + self.__rpc_add_input_channel, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - RetrieveWorkflowStateResponse, + AddInputChannelRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage": grpclib.const.Handler( - self.__rpc_propagate_embedded_control_message, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning": grpclib.const.Handler( + self.__rpc_add_partitioning, grpclib.const.Cardinality.UNARY_UNARY, - PropagateEmbeddedControlMessageRequest, - PropagateEmbeddedControlMessageResponse, + AddPartitioningRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint": grpclib.const.Handler( - self.__rpc_take_global_checkpoint, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort": grpclib.const.Handler( + self.__rpc_assign_port, grpclib.const.Cardinality.UNARY_UNARY, - TakeGlobalCheckpointRequest, - TakeGlobalCheckpointResponse, + AssignPortRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand": grpclib.const.Handler( - self.__rpc_debug_command, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint": grpclib.const.Handler( + self.__rpc_finalize_checkpoint, grpclib.const.Cardinality.UNARY_UNARY, - DebugCommandRequest, + FinalizeCheckpointRequest, + FinalizeCheckpointResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer": grpclib.const.Handler( + self.__rpc_flush_network_buffer, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression": grpclib.const.Handler( - self.__rpc_evaluate_python_expression, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor": grpclib.const.Handler( + self.__rpc_initialize_executor, grpclib.const.Cardinality.UNARY_UNARY, - EvaluatePythonExpressionRequest, - EvaluatePythonExpressionResponse, + InitializeExecutorRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered": grpclib.const.Handler( - self.__rpc_console_message_triggered, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor": grpclib.const.Handler( + self.__rpc_open_executor, grpclib.const.Cardinality.UNARY_UNARY, - ConsoleMessageTriggeredRequest, + EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted": grpclib.const.Handler( - self.__rpc_port_completed, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker": grpclib.const.Handler( + self.__rpc_pause_worker, grpclib.const.Cardinality.UNARY_UNARY, - PortCompletedRequest, + EmptyRequest, + WorkerStateResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint": grpclib.const.Handler( + self.__rpc_prepare_checkpoint, + grpclib.const.Cardinality.UNARY_UNARY, + PrepareCheckpointRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow": grpclib.const.Handler( - self.__rpc_start_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics": grpclib.const.Handler( + self.__rpc_query_statistics, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, - StartWorkflowResponse, + WorkerMetricsResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow": grpclib.const.Handler( - self.__rpc_resume_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker": grpclib.const.Handler( + self.__rpc_resume_worker, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + WorkerStateResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState": grpclib.const.Handler( + self.__rpc_retrieve_state, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow": grpclib.const.Handler( - self.__rpc_pause_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple": grpclib.const.Handler( + self.__rpc_retry_current_tuple, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated": grpclib.const.Handler( - self.__rpc_worker_state_updated, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker": grpclib.const.Handler( + self.__rpc_start_worker, grpclib.const.Cardinality.UNARY_UNARY, - WorkerStateUpdatedRequest, + EmptyRequest, + WorkerStateResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker": grpclib.const.Handler( + self.__rpc_end_worker, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted": grpclib.const.Handler( - self.__rpc_worker_execution_completed, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel": grpclib.const.Handler( + self.__rpc_start_channel, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers": grpclib.const.Handler( - self.__rpc_link_workers, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel": grpclib.const.Handler( + self.__rpc_end_channel, grpclib.const.Cardinality.UNARY_UNARY, - LinkWorkersRequest, + EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics": grpclib.const.Handler( - self.__rpc_controller_initiate_query_statistics, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand": grpclib.const.Handler( + self.__rpc_debug_command, grpclib.const.Cardinality.UNARY_UNARY, - QueryStatisticsRequest, + DebugCommandRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow": grpclib.const.Handler( - self.__rpc_retry_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression": grpclib.const.Handler( + self.__rpc_evaluate_python_expression, grpclib.const.Cardinality.UNARY_UNARY, - RetryWorkflowRequest, + EvaluatePythonExpressionRequest, + EvaluatedValue, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation": grpclib.const.Handler( + self.__rpc_no_operation, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, EmptyReturn, ), } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala index 4d9a36bab43..ae069db0786 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala @@ -34,6 +34,7 @@ class ControllerAsyncRPCHandlerInitializer( with AmberLogging with LinkWorkersHandler with WorkerExecutionCompletedHandler + with IterationCompletedHandler with WorkerStateUpdatedHandler with PauseHandler with QueryWorkerStatisticsHandler diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/IterationCompletedHandler.scala similarity index 56% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/IterationCompletedHandler.scala index b0233bbdcb9..92f3f6cd902 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndIterationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/IterationCompletedHandler.scala @@ -17,31 +17,29 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.worker.promisehandlers +package org.apache.texera.amber.engine.architecture.controller.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ - AsyncRPCContext, - EmptyRequest, - EndIterationRequest -} +import org.apache.texera.amber.engine.architecture.controller.{ControllerAsyncRPCHandlerInitializer, ExecutionStateUpdate} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, EmptyRequest, IterationCompletedRequest, QueryStatisticsRequest, StatisticsUpdateTarget} import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn -import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer +import org.apache.texera.amber.engine.common.virtualidentity.util.SELF -trait EndIterationHandler { - this: DataProcessorRPCHandlerInitializer => +/** indicate a worker has completed its execution + * i.e. received and processed all data from upstreams + * note that this doesn't mean all the output of this worker + * has been received by the downstream workers. + * + * possible sender: worker + */ +trait IterationCompletedHandler { + this: ControllerAsyncRPCHandlerInitializer => - override def endIteration( - request: EndIterationRequest, + override def iterationCompleted( + msg: IterationCompletedRequest, ctx: AsyncRPCContext ): Future[EmptyReturn] = { - dp.executor match { - //case _: LoopEndOpExec => - //workerInterface.nextIteration(EmptyRequest(), mkContext(request.worker)) - case _ => - //dp.processOnFinish() - //dp.outputManager.finalizeIteration(request.worker) - } + println("ergergergerg", msg) EmptyReturn() } } From be4360829b8c7a416bfbf8d9847d2a1aa8b42175 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Sat, 7 Mar 2026 21:19:55 -0800 Subject: [PATCH 55/55] update --- .../architecture/worker/DataProcessorRPCHandlerInitializer.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala index a49860c1e11..2abcdf66975 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala @@ -45,7 +45,6 @@ class DataProcessorRPCHandlerInitializer(val dp: DataProcessor) with ResumeHandler with StartHandler with EndHandler - with EndIterationHandler with StartChannelHandler with EndChannelHandler with AssignPortHandler