diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto index 1f55927e4ae..9b13f24c4af 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto @@ -34,7 +34,7 @@ option (scalapb.options) = { message ControlRequest { oneof sealed_value { - // request for controller + // request for coordinator PropagateEmbeddedControlMessageRequest propagateEmbeddedControlMessageRequest = 1; TakeGlobalCheckpointRequest takeGlobalCheckpointRequest = 2; DebugCommandRequest debugCommandRequest = 3; @@ -146,7 +146,7 @@ enum ConsoleMessageType{ } message ConsoleMessage { - option (scalapb.message).extends = "org.apache.texera.amber.engine.architecture.controller.ClientEvent"; + option (scalapb.message).extends = "org.apache.texera.amber.engine.architecture.coordinator.ClientEvent"; string worker_id = 1; google.protobuf.Timestamp timestamp = 2 [(scalapb.field).no_box = true]; ConsoleMessageType msg_type = 3; diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlreturns.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlreturns.proto index 43613b5cfdc..064a27b5723 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlreturns.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlreturns.proto @@ -32,7 +32,7 @@ option (scalapb.options) = { message ControlReturn { // Oneof block for various return types oneof sealed_value { - // controller responses + // coordinator responses RetrieveWorkflowStateResponse retrieveWorkflowStateResponse = 1; PropagateEmbeddedControlMessageResponse propagateEmbeddedControlMessageResponse = 2; TakeGlobalCheckpointResponse takeGlobalCheckpointResponse = 3; diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/coordinatorservice.proto similarity index 95% rename from amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto rename to amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/coordinatorservice.proto index 0932a7b914a..b782486c086 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/coordinatorservice.proto @@ -29,7 +29,7 @@ option (scalapb.options) = { }; -service ControllerService { +service CoordinatorService { rpc RetrieveWorkflowState(EmptyRequest) returns (RetrieveWorkflowStateResponse); rpc PropagateEmbeddedControlMessage(PropagateEmbeddedControlMessageRequest) returns (PropagateEmbeddedControlMessageResponse); rpc TakeGlobalCheckpoint(TakeGlobalCheckpointRequest) returns (TakeGlobalCheckpointResponse); @@ -44,7 +44,7 @@ service ControllerService { rpc WorkerExecutionCompleted(EmptyRequest) returns (EmptyReturn); rpc JumpToOperatorRegion(JumpToOperatorRegionRequest) returns (EmptyReturn); rpc LinkWorkers(LinkWorkersRequest) returns (EmptyReturn); - rpc ControllerInitiateQueryStatistics(QueryStatisticsRequest) returns (EmptyReturn); + rpc CoordinatorInitiateQueryStatistics(QueryStatisticsRequest) returns (EmptyReturn); rpc RetryWorkflow(RetryWorkflowRequest) returns (EmptyReturn); rpc ReconfigureWorkflow(WorkflowReconfigureRequest) returns (EmptyReturn); } diff --git a/amber/src/main/python/core/architecture/handlers/control/end_worker_handler.py b/amber/src/main/python/core/architecture/handlers/control/end_worker_handler.py index 434225a5f0a..56baf1f345c 100644 --- a/amber/src/main/python/core/architecture/handlers/control/end_worker_handler.py +++ b/amber/src/main/python/core/architecture/handlers/control/end_worker_handler.py @@ -33,7 +33,7 @@ class EndWorkerHandler(ControlHandler): async def end_worker(self, req: EmptyRequest) -> EmptyReturn: """ - The response of EndWorker to the controller indicates that this worker + The response of EndWorker to the coordinator indicates that this worker has finished not only the data processing logic, but also the processing of all the control messages. """ diff --git a/amber/src/main/python/core/architecture/rpc/async_rpc_client.py b/amber/src/main/python/core/architecture/rpc/async_rpc_client.py index f11c5afdea5..017bc35c566 100644 --- a/amber/src/main/python/core/architecture/rpc/async_rpc_client.py +++ b/amber/src/main/python/core/architecture/rpc/async_rpc_client.py @@ -32,7 +32,7 @@ ReturnInvocation, ControlReturn, ControlInvocation, - ControllerServiceStub, + CoordinatorServiceStub, WorkerServiceStub, ControlRequest, ) @@ -62,16 +62,16 @@ def __init__(self, output_queue: InternalQueue, context: Context): self._send_sequences: Dict[ActorVirtualIdentity, int] = defaultdict(int) self._unfulfilled_promises: Dict[(ActorVirtualIdentity, int), Future] = dict() # TODO: is this correct? - self._controller_service_stub = ControllerServiceStub("") + self._coordinator_service_stub = CoordinatorServiceStub("") rpc_context = AsyncRpcContext( ActorVirtualIdentity(self._context.worker_id), - ActorVirtualIdentity(name="CONTROLLER"), + ActorVirtualIdentity(name="COORDINATOR"), ) - self._controller_service_stub._unary_unary = AsyncRPCClient._assign_context( + self._coordinator_service_stub._unary_unary = AsyncRPCClient._assign_context( self, rpc_context ) - # Apply async_run to all async methods of the controller service stub - self._wrap_all_async_methods_with_async_run(self._controller_service_stub) + # Apply async_run to all async methods of the coordinator service stub + self._wrap_all_async_methods_with_async_run(self._coordinator_service_stub) def _assign_context( self, rpc_context: AsyncRpcContext @@ -108,11 +108,11 @@ def _wrap_all_async_methods_with_async_run(self, instance: Any) -> None: if inspect.iscoroutinefunction(attr): setattr(instance, attr_name, async_run(attr)) - def controller_stub(self) -> ControllerServiceStub: + def coordinator_stub(self) -> CoordinatorServiceStub: """ - Returns a proxy for interacting with the controller interface. + Returns a proxy for interacting with the coordinator interface. """ - return self._controller_service_stub + return self._coordinator_service_stub def get_worker_interface(self, target_worker) -> WorkerServiceStub: """ diff --git a/amber/src/main/python/core/runnables/data_processor.py b/amber/src/main/python/core/runnables/data_processor.py index cf792fd6811..992c276be4f 100644 --- a/amber/src/main/python/core/runnables/data_processor.py +++ b/amber/src/main/python/core/runnables/data_processor.py @@ -106,7 +106,7 @@ def _executor_session(self): back to MainLoop on exit. Reporting must happen *before* the switch: MainLoop's post-switch hook flushes console messages and then enters EXCEPTION_PAUSE, so anything queued after the switch - would arrive at the controller only after the worker resumes. + would arrive at the coordinator only after the worker resumes. """ try: executor = self._context.executor_manager.executor diff --git a/amber/src/main/python/core/runnables/main_loop.py b/amber/src/main/python/core/runnables/main_loop.py index 1334af12bfe..c873d48252d 100644 --- a/amber/src/main/python/core/runnables/main_loop.py +++ b/amber/src/main/python/core/runnables/main_loop.py @@ -90,7 +90,7 @@ def __init__( def complete(self) -> None: """ Complete the DataProcessor, marking state to COMPLETED, and notify the - controller. + coordinator. """ # flush the buffered console prints self._check_and_report_console_messages(force_flush=True) @@ -99,8 +99,8 @@ def complete(self) -> None: self.data_processor.stop() self.context.state_manager.transit_to(WorkerState.COMPLETED) self.context.statistics_manager.update_total_execution_time(time.time_ns()) - controller_interface = self._async_rpc_client.controller_stub() - controller_interface.worker_execution_completed(EmptyRequest()) + coordinator_interface = self._async_rpc_client.coordinator_stub() + coordinator_interface.worker_execution_completed(EmptyRequest()) self.context.close() def _check_and_process_control(self) -> None: @@ -265,7 +265,7 @@ def _process_end_channel(self) -> None: ) if input_port_id is not None: - self._async_rpc_client.controller_stub().port_completed( + self._async_rpc_client.coordinator_stub().port_completed( PortCompletedRequest( port_id=input_port_id, input=True, @@ -285,7 +285,7 @@ def _process_end_channel(self) -> None: # Need to send port completed even if there is no downstream link for port_id in self.context.output_manager.get_port_ids(): - self._async_rpc_client.controller_stub().port_completed( + self._async_rpc_client.coordinator_stub().port_completed( PortCompletedRequest(port_id=port_id, input=False) ) self.complete() @@ -423,7 +423,7 @@ def _process_data_element(self, data_element: DataElement) -> None: logger.exception(err) def _send_console_message(self, console_message: ConsoleMessage): - self._async_rpc_client.controller_stub().console_message_triggered( + self._async_rpc_client.coordinator_stub().console_message_triggered( ConsoleMessageTriggeredRequest(console_message=console_message) ) diff --git a/amber/src/main/python/pytexera/storage/large_binary_manager.py b/amber/src/main/python/pytexera/storage/large_binary_manager.py index b323c676d55..1edd25595ca 100644 --- a/amber/src/main/python/pytexera/storage/large_binary_manager.py +++ b/amber/src/main/python/pytexera/storage/large_binary_manager.py @@ -32,7 +32,7 @@ class LargeBinaryManager: """Manages large binaries in S3 for a worker process. A singleton, so the cached S3 client is shared process-wide. create() appends a - unique suffix to an execution-scoped base URI handed down by the controller as + unique suffix to an execution-scoped base URI handed down by the coordinator as process config (``StorageConfig.S3_LARGE_BINARIES_BASE_URI``); the worker never holds an execution id. This is the Python counterpart of the JVM ``LargeBinaryManager``, which uses a thread-local instead because one JVM process @@ -90,7 +90,7 @@ def _ensure_bucket_exists(self, bucket: str): logger.info(f"Created bucket: {bucket}") def create(self) -> str: - """Append a unique suffix to the controller-provided base URI. + """Append a unique suffix to the coordinator-provided base URI. Pure string construction (no S3 round-trip); the bucket is created on demand at upload time. Returns e.g. ``s3://bucket/objects/{execution_id}/{uuid}``. diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/ExecutorDeployment.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/ExecutorDeployment.scala index 39261432872..04cd7ff0dd9 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/ExecutorDeployment.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/ExecutorDeployment.scala @@ -21,8 +21,8 @@ package org.apache.texera.amber.engine.architecture.common import org.apache.pekko.actor.{Address, Deploy} import org.apache.pekko.remote.RemoteScope -import org.apache.texera.amber.core.workflow.{PhysicalOp, PreferController, RoundRobinPreference} -import org.apache.texera.amber.engine.architecture.controller.execution.OperatorExecution +import org.apache.texera.amber.core.workflow.{PhysicalOp, PreferCoordinator, RoundRobinPreference} +import org.apache.texera.amber.engine.architecture.coordinator.execution.OperatorExecution import org.apache.texera.amber.engine.architecture.deploysemantics.AddressInfo import org.apache.texera.amber.engine.architecture.pythonworker.PythonWorkflowWorker import org.apache.texera.amber.engine.architecture.scheduling.config.OperatorConfig @@ -38,7 +38,7 @@ object ExecutorDeployment { def createWorkers( op: PhysicalOp, - controllerActorService: PekkoActorService, + coordinatorActorService: PekkoActorService, operatorExecution: OperatorExecution, operatorConfig: OperatorConfig, stateRestoreConfig: Option[StateRestoreConfig], @@ -46,8 +46,8 @@ object ExecutorDeployment { ): Unit = { val addressInfo = AddressInfo( - controllerActorService.getClusterNodeAddresses, - controllerActorService.self.path.address + coordinatorActorService.getClusterNodeAddresses, + coordinatorActorService.self.path.address ) operatorConfig.workerConfigs.foreach(workerConfig => { @@ -61,8 +61,8 @@ object ExecutorDeployment { ) val locationPreference = op.locationPreference.getOrElse(RoundRobinPreference) val preferredAddress: Address = locationPreference match { - case PreferController => - addressInfo.controllerAddress + case PreferCoordinator => + addressInfo.coordinatorAddress case RoundRobinPreference => assert( addressInfo.allAddresses.nonEmpty, @@ -83,8 +83,8 @@ object ExecutorDeployment { ) } // Note: At this point, we don't know if the actor is fully initialized. - // Thus, the ActorRef returned from `controllerActorService.actorOf` is ignored. - controllerActorService.actorOf( + // Thus, the ActorRef returned from `coordinatorActorService.actorOf` is ignored. + coordinatorActorService.actorOf( workflowWorker.withDeploy(Deploy(scope = RemoteScope(preferredAddress))) ) operatorExecution.initWorkerExecution(workerId) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/PekkoActorRefMappingService.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/PekkoActorRefMappingService.scala index 323435891ee..2c085b9dc89 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/PekkoActorRefMappingService.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/common/PekkoActorRefMappingService.scala @@ -28,7 +28,7 @@ import org.apache.texera.amber.engine.architecture.common.WorkflowActor.{ RegisterActorRef } import org.apache.texera.amber.engine.common.AmberLogging -import org.apache.texera.amber.engine.common.virtualidentity.util.{CONTROLLER, SELF} +import org.apache.texera.amber.engine.common.virtualidentity.util.{COORDINATOR, SELF} import org.apache.texera.amber.util.VirtualIdentityUtils import scala.collection.mutable @@ -104,8 +104,8 @@ class PekkoActorRefMappingService(actorService: PekkoActorService) extends Amber replyTo.foreach { actor => actor ! RegisterActorRef(id, actorRefMapping(id)) } - } else if (actorId != CONTROLLER) { - // propagation stops at controller + } else if (actorId != COORDINATOR) { + // propagation stops at coordinator if (!queriedActorVirtualIdentities.contains(id)) { try { actorService.parent ! GetActorRef(id, replyTo + actorService.self) @@ -118,7 +118,7 @@ class PekkoActorRefMappingService(actorService: PekkoActorService) extends Amber } } } else { - // on controller, wait for actor ref registration. + // on coordinator, wait for actor ref registration. logger.warn(s"unknown identifier: ${VirtualIdentityUtils.toShorterString(id)}") val toNotifySet = toNotifyOnRegistration.getOrElseUpdate(id, mutable.HashSet[ActorRef]()) replyTo.foreach(toNotifySet.add) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ClientEvent.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/ClientEvent.scala similarity index 97% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ClientEvent.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/ClientEvent.scala index 1092af15e77..be7dce2779f 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ClientEvent.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/ClientEvent.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.texera.amber.core.tuple.Tuple import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/Controller.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Coordinator.scala similarity index 82% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/Controller.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Coordinator.scala index 2bbc8ea1227..167602113e0 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/Controller.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Coordinator.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.pekko.actor.SupervisorStrategy.Stop import org.apache.pekko.actor.{AllForOneStrategy, Props, SupervisorStrategy} @@ -27,7 +27,7 @@ import org.apache.texera.amber.core.virtualidentity.ChannelIdentity import org.apache.texera.amber.core.workflow.{PhysicalPlan, WorkflowContext} import org.apache.texera.amber.engine.architecture.common.WorkflowActor.NetworkAck import org.apache.texera.amber.engine.architecture.common.{ExecutorDeployment, WorkflowActor} -import org.apache.texera.amber.engine.architecture.controller.execution.OperatorExecution +import org.apache.texera.amber.engine.architecture.coordinator.execution.OperatorExecution import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ ControlInvocation, EmbeddedControlMessage @@ -41,15 +41,15 @@ import org.apache.texera.amber.engine.common.ambermessage.{ DirectControlMessagePayload, WorkflowFIFOMessage } -import org.apache.texera.amber.engine.common.virtualidentity.util.{CLIENT, CONTROLLER, SELF} +import org.apache.texera.amber.engine.common.virtualidentity.util.{CLIENT, COORDINATOR, SELF} import org.apache.texera.amber.engine.common.{CheckpointState, SerializedState} import org.apache.texera.web.SessionState import scala.concurrent.duration.DurationInt -object ControllerConfig { - def default: ControllerConfig = - ControllerConfig( +object CoordinatorConfig { + def default: CoordinatorConfig = + CoordinatorConfig( statusUpdateIntervalMs = Option(ApplicationConfig.getStatusUpdateIntervalInMs), runtimeStatisticsPersistenceIntervalMs = Option(ApplicationConfig.getRuntimeStatisticsPersistenceIntervalInMs), @@ -58,50 +58,50 @@ object ControllerConfig { ) } -final case class ControllerConfig( +final case class CoordinatorConfig( statusUpdateIntervalMs: Option[Long], runtimeStatisticsPersistenceIntervalMs: Option[Long], stateRestoreConfOpt: Option[StateRestoreConfig], faultToleranceConfOpt: Option[FaultToleranceConfig] ) -object Controller { +object Coordinator { def props( workflowContext: WorkflowContext, physicalPlan: PhysicalPlan, - controllerConfig: ControllerConfig = ControllerConfig.default + coordinatorConfig: CoordinatorConfig = CoordinatorConfig.default ): Props = Props( - new Controller( + new Coordinator( workflowContext, physicalPlan, - controllerConfig + coordinatorConfig ) ) } -class Controller( +class Coordinator( workflowContext: WorkflowContext, physicalPlan: PhysicalPlan, - controllerConfig: ControllerConfig + coordinatorConfig: CoordinatorConfig ) extends WorkflowActor( - controllerConfig.faultToleranceConfOpt, - CONTROLLER + coordinatorConfig.faultToleranceConfOpt, + COORDINATOR ) { actorRefMappingService.registerActorRef(CLIENT, context.parent) - val controllerTimerService = new ControllerTimerService(controllerConfig, actorService) - var cp = new ControllerProcessor( + val coordinatorTimerService = new CoordinatorTimerService(coordinatorConfig, actorService) + var cp = new CoordinatorProcessor( workflowContext, - controllerConfig, + coordinatorConfig, actorId, logManager.sendCommitted ) // manages the lifecycle of entire replay process - // triggers onStart callback when the first worker/controller marks itself as recovering. - // triggers onComplete callback when all worker/controller finishes recovering. + // triggers onStart callback when the first worker/coordinator marks itself as recovering. + // triggers onComplete callback when all worker/coordinator finishes recovering. private val globalReplayManager = new GlobalReplayManager( () => { //onStart @@ -127,14 +127,14 @@ class Controller( state.send(RegionUpdateEvent(regions)) } - val controllerRestoreConf = controllerConfig.stateRestoreConfOpt - if (controllerRestoreConf.isDefined) { - globalReplayManager.markRecoveryStatus(CONTROLLER, isRecovering = true) + val coordinatorRestoreConf = coordinatorConfig.stateRestoreConfOpt + if (coordinatorRestoreConf.isDefined) { + globalReplayManager.markRecoveryStatus(COORDINATOR, isRecovering = true) setupReplay( cp, - controllerRestoreConf.get, + coordinatorRestoreConf.get, () => { - globalReplayManager.markRecoveryStatus(CONTROLLER, isRecovering = false) + globalReplayManager.markRecoveryStatus(COORDINATOR, isRecovering = false) } ) processMessages() @@ -159,7 +159,7 @@ class Controller( msg.payload match { case payload: DirectControlMessagePayload => cp.processDCM(msg.channelId, payload) case _: EmbeddedControlMessage => // skip ECM - case p => throw new RuntimeException(s"controller cannot handle $p") + case p => throw new RuntimeException(s"coordinator cannot handle $p") } } case None => @@ -195,7 +195,7 @@ class Controller( /** flow-control */ override def getQueuedCredit(channelId: ChannelIdentity): Long = { - 0 // no queued credit for controller + 0 // no queued credit for coordinator } override def handleBackpressure(isBackpressured: Boolean): Unit = {} @@ -214,14 +214,14 @@ class Controller( private def attachRuntimeServicesToCPState(): Unit = { cp.setupActorService(actorService) - cp.setupTimerService(controllerTimerService) + cp.setupTimerService(coordinatorTimerService) cp.setupActorRefService(actorRefMappingService) cp.setupLogManager(logManager) cp.setupTransferService(transferService) } override def loadFromCheckpoint(chkpt: CheckpointState): Unit = { - val cpState: ControllerProcessor = chkpt.load(SerializedState.CP_STATE_KEY) + val cpState: CoordinatorProcessor = chkpt.load(SerializedState.CP_STATE_KEY) val outputMessages: Array[WorkflowFIFOMessage] = chkpt.load(SerializedState.OUTPUT_MSG_KEY) cp = cpState cp.outputHandler = logManager.sendCommitted @@ -236,8 +236,8 @@ class Controller( actorService, OperatorExecution(), //use dummy value here regionExecution.region.resourceConfig.get.operatorConfigs(opId), - controllerConfig.stateRestoreConfOpt, - controllerConfig.faultToleranceConfOpt + coordinatorConfig.stateRestoreConfOpt, + coordinatorConfig.faultToleranceConfOpt ) } } @@ -247,6 +247,6 @@ class Controller( cp.workflowExecution.getAllRegionExecutionsStats ) ) - globalReplayManager.markRecoveryStatus(CONTROLLER, isRecovering = false) + globalReplayManager.markRecoveryStatus(COORDINATOR, isRecovering = false) } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorAsyncRPCHandlerInitializer.scala similarity index 82% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorAsyncRPCHandlerInitializer.scala index 7e5a904716c..0f8f9c8bfc6 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorAsyncRPCHandlerInitializer.scala @@ -17,20 +17,20 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import com.twitter.util.Future import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity -import org.apache.texera.amber.engine.architecture.controller.promisehandlers._ +import org.apache.texera.amber.engine.architecture.coordinator.promisehandlers._ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.AsyncRPCContext -import org.apache.texera.amber.engine.architecture.rpc.controllerservice.ControllerServiceFs2Grpc +import org.apache.texera.amber.engine.architecture.rpc.coordinatorservice.CoordinatorServiceFs2Grpc import org.apache.texera.amber.engine.common.AmberLogging import org.apache.texera.amber.engine.common.rpc.AsyncRPCHandlerInitializer -class ControllerAsyncRPCHandlerInitializer( - val cp: ControllerProcessor +class CoordinatorAsyncRPCHandlerInitializer( + val cp: CoordinatorProcessor ) extends AsyncRPCHandlerInitializer(cp.asyncRPCClient, cp.asyncRPCServer) - with ControllerServiceFs2Grpc[Future, AsyncRPCContext] + with CoordinatorServiceFs2Grpc[Future, AsyncRPCContext] with AmberLogging with LinkWorkersHandler with WorkerExecutionCompletedHandler diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerProcessor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorProcessor.scala similarity index 83% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerProcessor.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorProcessor.scala index 883e250238a..1d75a0ef376 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerProcessor.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorProcessor.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity import org.apache.texera.amber.core.workflow.WorkflowContext @@ -27,15 +27,15 @@ import org.apache.texera.amber.engine.architecture.common.{ PekkoMessageTransferService, AmberProcessor } -import org.apache.texera.amber.engine.architecture.controller.execution.WorkflowExecution +import org.apache.texera.amber.engine.architecture.coordinator.execution.WorkflowExecution import org.apache.texera.amber.engine.architecture.logreplay.ReplayLogManager import org.apache.texera.amber.engine.architecture.scheduling.WorkflowExecutionManager import org.apache.texera.amber.engine.architecture.worker.WorkflowWorker.MainThreadDelegateMessage import org.apache.texera.amber.engine.common.ambermessage.WorkflowFIFOMessage -class ControllerProcessor( +class CoordinatorProcessor( workflowContext: WorkflowContext, - controllerConfig: ControllerConfig, + coordinatorConfig: CoordinatorConfig, actorId: ActorVirtualIdentity, outputHandler: Either[MainThreadDelegateMessage, WorkflowFIFOMessage] => Unit ) extends AmberProcessor(actorId, outputHandler) { @@ -45,16 +45,16 @@ class ControllerProcessor( new WorkflowScheduler(workflowContext, actorId) val workflowExecutionManager: WorkflowExecutionManager = new WorkflowExecutionManager( workflowExecution, - controllerConfig, + coordinatorConfig, asyncRPCClient ) - private val initializer = new ControllerAsyncRPCHandlerInitializer(this) + private val initializer = new CoordinatorAsyncRPCHandlerInitializer(this) - @transient var controllerTimerService: ControllerTimerService = _ + @transient var coordinatorTimerService: CoordinatorTimerService = _ - def setupTimerService(controllerTimerService: ControllerTimerService): Unit = { - this.controllerTimerService = controllerTimerService + def setupTimerService(coordinatorTimerService: CoordinatorTimerService): Unit = { + this.coordinatorTimerService = coordinatorTimerService } @transient var transferService: PekkoMessageTransferService = _ diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerTimerService.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorTimerService.scala similarity index 86% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerTimerService.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorTimerService.scala index a4ad0898c9b..8acf3bbda13 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerTimerService.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorTimerService.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.pekko.actor.Cancellable import org.apache.texera.amber.engine.architecture.common.PekkoActorService @@ -26,14 +26,14 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ QueryStatisticsRequest, StatisticsUpdateTarget } -import org.apache.texera.amber.engine.architecture.rpc.controllerservice.ControllerServiceGrpc.METHOD_CONTROLLER_INITIATE_QUERY_STATISTICS +import org.apache.texera.amber.engine.architecture.rpc.coordinatorservice.CoordinatorServiceGrpc.METHOD_COORDINATOR_INITIATE_QUERY_STATISTICS import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient.ControlInvocation import org.apache.texera.amber.engine.common.virtualidentity.util.SELF import scala.concurrent.duration.{DurationInt, FiniteDuration, MILLISECONDS} -class ControllerTimerService( - controllerConfig: ControllerConfig, +class CoordinatorTimerService( + coordinatorConfig: CoordinatorConfig, pekkoActorService: PekkoActorService ) { var statusUpdateAskHandle: Option[Cancellable] = None @@ -50,7 +50,7 @@ class ControllerTimerService( 0.milliseconds, FiniteDuration.apply(intervalMs.get, MILLISECONDS), ControlInvocation( - METHOD_CONTROLLER_INITIATE_QUERY_STATISTICS, + METHOD_COORDINATOR_INITIATE_QUERY_STATISTICS, QueryStatisticsRequest(Seq.empty, updateTarget), AsyncRPCContext(SELF, SELF), 0 @@ -73,7 +73,7 @@ class ControllerTimerService( def enableStatusUpdate(): Unit = { statusUpdateAskHandle = enableTimer( - controllerConfig.statusUpdateIntervalMs, + coordinatorConfig.statusUpdateIntervalMs, StatisticsUpdateTarget.UI_ONLY, statusUpdateAskHandle ) @@ -81,7 +81,7 @@ class ControllerTimerService( def enableRuntimeStatisticsCollection(): Unit = { runtimeStatisticsAskHandle = enableTimer( - controllerConfig.runtimeStatisticsPersistenceIntervalMs, + coordinatorConfig.runtimeStatisticsPersistenceIntervalMs, StatisticsUpdateTarget.PERSISTENCE_ONLY, runtimeStatisticsAskHandle ) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/GlobalReplayManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/GlobalReplayManager.scala similarity index 95% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/GlobalReplayManager.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/GlobalReplayManager.scala index b8dc3fb6072..8d80782b409 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/GlobalReplayManager.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/GlobalReplayManager.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/Workflow.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Workflow.scala similarity index 94% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/Workflow.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Workflow.scala index 4c2220ae9aa..6ec92deab83 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/Workflow.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Workflow.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.texera.amber.core.workflow.{PhysicalPlan, WorkflowContext} import org.apache.texera.workflow.LogicalPlan diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowScheduler.scala similarity index 96% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowScheduler.scala index b1acb3c0650..1409cee344e 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowScheduler.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity import org.apache.texera.amber.core.workflow.{PhysicalPlan, WorkflowContext} diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/ChannelExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ChannelExecution.scala similarity index 91% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/ChannelExecution.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ChannelExecution.scala index 960cad507ea..e04df0b5f56 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/ChannelExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ChannelExecution.scala @@ -17,6 +17,6 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution case class ChannelExecution() diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/ExecutionUtils.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtils.scala similarity index 98% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/ExecutionUtils.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtils.scala index 7ee9bc04735..e89da69d433 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/ExecutionUtils.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtils.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState import org.apache.texera.amber.engine.architecture.worker.statistics.{ diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/LinkExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/LinkExecution.scala similarity index 94% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/LinkExecution.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/LinkExecution.scala index 434636e16f6..83d2271a22d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/LinkExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/LinkExecution.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.apache.texera.amber.core.virtualidentity.ChannelIdentity diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/OperatorExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/OperatorExecution.scala similarity index 96% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/OperatorExecution.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/OperatorExecution.scala index 8b698af8f63..6315fb06fe9 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/OperatorExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/OperatorExecution.scala @@ -17,11 +17,11 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.engine.architecture.controller.execution.ExecutionUtils.aggregateStates +import org.apache.texera.amber.engine.architecture.coordinator.execution.ExecutionUtils.aggregateStates import org.apache.texera.amber.engine.architecture.deploysemantics.layer.WorkerExecution import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState import org.apache.texera.amber.engine.architecture.worker.statistics.{ diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/RegionExecution.scala similarity index 98% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/RegionExecution.scala index cca66cfc048..0d585ee7dbe 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/RegionExecution.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import com.rits.cloning.Cloner import org.apache.texera.amber.core.virtualidentity.PhysicalOpIdentity diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkerPortExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkerPortExecution.scala similarity index 92% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkerPortExecution.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkerPortExecution.scala index 0b7c3a39397..9ad0e859aa7 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkerPortExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkerPortExecution.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution case class WorkerPortExecution() { var completed: Boolean = false diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkflowExecution.scala similarity index 97% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkflowExecution.scala index 01a8fee466d..8d0396b4fca 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkflowExecution.scala @@ -17,10 +17,10 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.apache.texera.amber.core.virtualidentity.PhysicalOpIdentity -import org.apache.texera.amber.engine.architecture.controller.execution.ExecutionUtils.aggregateMetrics +import org.apache.texera.amber.engine.architecture.coordinator.execution.ExecutionUtils.aggregateMetrics import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState._ import org.apache.texera.amber.engine.architecture.scheduling.{Region, RegionIdentity} diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ConsoleMessageHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/ConsoleMessageHandler.scala similarity index 85% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ConsoleMessageHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/ConsoleMessageHandler.scala index 4030ad7d3ee..14bbfc3952c 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ConsoleMessageHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/ConsoleMessageHandler.scala @@ -17,10 +17,10 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, ConsoleMessageTriggeredRequest @@ -28,7 +28,7 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn trait ConsoleMessageHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def consoleMessageTriggered( msg: ConsoleMessageTriggeredRequest, diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/DebugCommandHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/DebugCommandHandler.scala similarity index 85% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/DebugCommandHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/DebugCommandHandler.scala index 2e18ec6ba14..28cd8dca938 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/DebugCommandHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/DebugCommandHandler.scala @@ -17,11 +17,11 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, DebugCommandRequest @@ -29,7 +29,7 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn trait DebugCommandHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def debugCommand(msg: DebugCommandRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { workerInterface.debugCommand(msg, mkContext(ActorVirtualIdentity(msg.workerId))) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/EmbeddedControlMessageHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/EmbeddedControlMessageHandler.scala similarity index 91% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/EmbeddedControlMessageHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/EmbeddedControlMessageHandler.scala index 45a4491ad95..9e696fb9287 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/EmbeddedControlMessageHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/EmbeddedControlMessageHandler.scala @@ -17,11 +17,11 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future import org.apache.texera.amber.core.virtualidentity.{ActorVirtualIdentity, ChannelIdentity} -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, ControlInvocation, @@ -31,11 +31,11 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{ ControlReturn, PropagateEmbeddedControlMessageResponse } -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.util.VirtualIdentityUtils trait EmbeddedControlMessageHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def propagateEmbeddedControlMessage( msg: PropagateEmbeddedControlMessageRequest, @@ -73,8 +73,8 @@ trait EmbeddedControlMessageHandler { val controlChannels = msg.sourceOpToStartProp.flatMap { source => cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.flatMap { worker => Seq( - ChannelIdentity(CONTROLLER, worker, isControl = true), - ChannelIdentity(worker, CONTROLLER, isControl = true) + ChannelIdentity(COORDINATOR, worker, isControl = true), + ChannelIdentity(worker, COORDINATOR, isControl = true) ) } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/EvaluatePythonExpressionHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/EvaluatePythonExpressionHandler.scala similarity index 90% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/EvaluatePythonExpressionHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/EvaluatePythonExpressionHandler.scala index 6a235b2e77f..3d9865b3523 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/EvaluatePythonExpressionHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/EvaluatePythonExpressionHandler.scala @@ -17,11 +17,11 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future import org.apache.texera.amber.core.virtualidentity.OperatorIdentity -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, EvaluatePythonExpressionRequest @@ -29,7 +29,7 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EvaluatePythonExpressionResponse trait EvaluatePythonExpressionHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def evaluatePythonExpression( msg: EvaluatePythonExpressionRequest, diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/JumpToOperatorRegionHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/JumpToOperatorRegionHandler.scala similarity index 87% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/JumpToOperatorRegionHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/JumpToOperatorRegionHandler.scala index 9fbc1e48afe..2697e21ee5b 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/JumpToOperatorRegionHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/JumpToOperatorRegionHandler.scala @@ -17,10 +17,10 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, JumpToOperatorRegionRequest @@ -28,7 +28,7 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn trait JumpToOperatorRegionHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def jumpToOperatorRegion( msg: JumpToOperatorRegionRequest, diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/LinkWorkersHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/LinkWorkersHandler.scala similarity index 89% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/LinkWorkersHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/LinkWorkersHandler.scala index 1049fc0ed19..4df3c495224 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/LinkWorkersHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/LinkWorkersHandler.scala @@ -17,10 +17,10 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AddInputChannelRequest, AddPartitioningRequest, @@ -32,10 +32,10 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyRetur /** add a data transfer partitioning to the sender workers and update input linking * for the receiver workers of a link strategy. * - * possible sender: controller, client + * possible sender: coordinator, client */ trait LinkWorkersHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def linkWorkers(msg: LinkWorkersRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { val region = cp.workflowExecutionManager.getRegionOfLink(msg.link) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PauseHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/PauseHandler.scala similarity index 88% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PauseHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/PauseHandler.scala index 35a85f56ae9..5a1b8a5cd84 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PauseHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/PauseHandler.scala @@ -17,13 +17,13 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future import org.apache.texera.amber.core.tuple.Tuple import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerAsyncRPCHandlerInitializer, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorAsyncRPCHandlerInitializer, ExecutionStateUpdate, ExecutionStatsUpdate, RuntimeStatisticsPersist @@ -41,14 +41,14 @@ import scala.collection.mutable /** pause the entire workflow * - * possible sender: client, controller + * possible sender: client, coordinator */ trait PauseHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def pauseWorkflow(request: EmptyRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { - cp.controllerTimerService.disableStatusUpdate() // to be enabled in resume - cp.controllerTimerService.disableRuntimeStatisticsCollection() // to be enabled in resume + cp.coordinatorTimerService.disableStatusUpdate() // to be enabled in resume + cp.coordinatorTimerService.disableRuntimeStatisticsCollection() // to be enabled in resume Future .collect( cp.workflowExecution.getRunningRegionExecutions diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PortCompletedHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/PortCompletedHandler.scala similarity index 92% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PortCompletedHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/PortCompletedHandler.scala index b961ee28042..4815f3a140b 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PortCompletedHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/PortCompletedHandler.scala @@ -17,13 +17,13 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future import org.apache.texera.amber.core.WorkflowRuntimeException import org.apache.texera.amber.core.workflow.GlobalPortIdentity -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerAsyncRPCHandlerInitializer, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorAsyncRPCHandlerInitializer, FatalError } import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ @@ -33,7 +33,7 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ StatisticsUpdateTarget } import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.util.VirtualIdentityUtils /** Notify the completion of a port: @@ -44,19 +44,19 @@ import org.apache.texera.amber.util.VirtualIdentityUtils * possible sender: worker */ trait PortCompletedHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def portCompleted( msg: PortCompletedRequest, ctx: AsyncRPCContext ): Future[EmptyReturn] = { - controllerInterface - .controllerInitiateQueryStatistics( + coordinatorInterface + .coordinatorInitiateQueryStatistics( QueryStatisticsRequest( scala.Seq(ctx.sender), StatisticsUpdateTarget.BOTH_UI_AND_PERSISTENCE ), - CONTROLLER + COORDINATOR ) .map { _ => val globalPortId = GlobalPortIdentity( diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/QueryWorkerStatisticsHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/QueryWorkerStatisticsHandler.scala similarity index 95% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/QueryWorkerStatisticsHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/QueryWorkerStatisticsHandler.scala index 1f22fa5b368..58cf25b6536 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/QueryWorkerStatisticsHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/QueryWorkerStatisticsHandler.scala @@ -17,13 +17,13 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future import org.apache.texera.common.config.ApplicationConfig import org.apache.texera.amber.core.virtualidentity.PhysicalOpIdentity -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerAsyncRPCHandlerInitializer, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorAsyncRPCHandlerInitializer, ExecutionStatsUpdate, RuntimeStatisticsPersist } @@ -43,10 +43,10 @@ import org.apache.texera.amber.util.VirtualIdentityUtils /** Get statistics from all the workers * - * possible sender: controller(by statusUpdateAskHandle) + * possible sender: coordinator(by statusUpdateAskHandle) */ trait QueryWorkerStatisticsHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => private var globalQueryStatsOngoing = false @@ -77,7 +77,7 @@ trait QueryWorkerStatisticsHandler { } } - override def controllerInitiateQueryStatistics( + override def coordinatorInitiateQueryStatistics( msg: QueryStatisticsRequest, ctx: AsyncRPCContext ): Future[EmptyReturn] = { diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/ReconfigurationHandler.scala similarity index 93% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/ReconfigurationHandler.scala index 6bedad07640..cf6d22e29a1 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/ReconfigurationHandler.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future import org.apache.texera.amber.core.virtualidentity.{ @@ -25,8 +25,8 @@ import org.apache.texera.amber.core.virtualidentity.{ ChannelIdentity, EmbeddedControlMessageIdentity } -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerAsyncRPCHandlerInitializer, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorAsyncRPCHandlerInitializer, UpdateExecutorCompleted } import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.ALL_ALIGNMENT @@ -37,14 +37,14 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ } import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{ControlReturn, EmptyReturn} import org.apache.texera.amber.engine.common.FriesReconfigurationAlgorithm -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.util.VirtualIdentityUtils import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceGrpc.METHOD_UPDATE_EXECUTOR import scala.collection.mutable trait ReconfigurationHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def reconfigureWorkflow( msg: WorkflowReconfigureRequest, @@ -92,8 +92,8 @@ trait ReconfigurationHandler { val controlChannels = friesComponent.sources.flatMap { source => cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.flatMap { worker => Seq( - ChannelIdentity(CONTROLLER, worker, isControl = true), - ChannelIdentity(worker, CONTROLLER, isControl = true) + ChannelIdentity(COORDINATOR, worker, isControl = true), + ChannelIdentity(worker, COORDINATOR, isControl = true) ) } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ResumeHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/ResumeHandler.scala similarity index 87% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ResumeHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/ResumeHandler.scala index c94ba91c205..78436595492 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ResumeHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/ResumeHandler.scala @@ -17,11 +17,11 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerAsyncRPCHandlerInitializer, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorAsyncRPCHandlerInitializer, ExecutionStatsUpdate, RuntimeStatisticsPersist } @@ -34,10 +34,10 @@ import org.apache.texera.amber.util.VirtualIdentityUtils /** resume the entire workflow * - * possible sender: controller, client + * possible sender: coordinator, client */ trait ResumeHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def resumeWorkflow(msg: EmptyRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { // send all workers resume @@ -62,9 +62,9 @@ trait ResumeHandler { val stats = cp.workflowExecution.getAllRegionExecutionsStats sendToClient(ExecutionStatsUpdate(stats)) sendToClient(RuntimeStatisticsPersist(stats)) - cp.controllerTimerService + cp.coordinatorTimerService .enableStatusUpdate() //re-enabled it since it is disabled in pause - cp.controllerTimerService + cp.coordinatorTimerService .enableRuntimeStatisticsCollection() //re-enabled it since it is disabled in pause EmptyReturn() } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/RetrieveWorkflowStateHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/RetrieveWorkflowStateHandler.scala similarity index 90% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/RetrieveWorkflowStateHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/RetrieveWorkflowStateHandler.scala index b6e75cd823b..26c51bbd5d9 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/RetrieveWorkflowStateHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/RetrieveWorkflowStateHandler.scala @@ -17,11 +17,11 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future import org.apache.texera.amber.core.virtualidentity.EmbeddedControlMessageIdentity -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.NO_ALIGNMENT import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, @@ -38,7 +38,7 @@ import org.apache.texera.amber.engine.common.virtualidentity.util.SELF import java.time.Instant trait RetrieveWorkflowStateHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def retrieveWorkflowState( request: EmptyRequest, @@ -56,7 +56,7 @@ trait RetrieveWorkflowStateHandler { EmptyRequest(), METHOD_RETRIEVE_STATE.getBareMethodName ) - controllerInterface + coordinatorInterface .propagateEmbeddedControlMessage( msg, mkContext(SELF) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/RetryWorkflowHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/RetryWorkflowHandler.scala similarity index 81% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/RetryWorkflowHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/RetryWorkflowHandler.scala index a2e1e257f8a..16088b7c598 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/RetryWorkflowHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/RetryWorkflowHandler.scala @@ -17,24 +17,24 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, EmptyRequest, RetryWorkflowRequest } import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR /** retry the execution of the entire workflow * - * possible sender: controller, client + * possible sender: coordinator, client */ trait RetryWorkflowHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def retryWorkflow( msg: RetryWorkflowRequest, @@ -50,7 +50,7 @@ trait RetryWorkflowHandler { .unit // resume all workers - controllerInterface.resumeWorkflow(EmptyRequest(), mkContext(CONTROLLER)) + coordinatorInterface.resumeWorkflow(EmptyRequest(), mkContext(COORDINATOR)) } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/StartWorkflowHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/StartWorkflowHandler.scala similarity index 82% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/StartWorkflowHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/StartWorkflowHandler.scala index 2acb8ea6316..4b15c837e1d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/StartWorkflowHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/StartWorkflowHandler.scala @@ -17,10 +17,10 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, EmptyRequest @@ -34,7 +34,7 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAg * possible sender: client */ trait StartWorkflowHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def startWorkflow( request: EmptyRequest, @@ -44,8 +44,8 @@ trait StartWorkflowHandler { cp.workflowExecutionManager .advanceRegionExecutions(cp.actorService) .map(_ => { - cp.controllerTimerService.enableStatusUpdate() - cp.controllerTimerService.enableRuntimeStatisticsCollection() + cp.coordinatorTimerService.enableStatusUpdate() + cp.coordinatorTimerService.enableRuntimeStatisticsCollection() StartWorkflowResponse(RUNNING) }) } else { diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/TakeGlobalCheckpointHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/TakeGlobalCheckpointHandler.scala similarity index 94% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/TakeGlobalCheckpointHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/TakeGlobalCheckpointHandler.scala index 128c00fc562..d80b55145d8 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/TakeGlobalCheckpointHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/TakeGlobalCheckpointHandler.scala @@ -17,11 +17,11 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity -import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.NO_ALIGNMENT import org.apache.texera.amber.engine.architecture.rpc.controlcommands._ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.TakeGlobalCheckpointResponse @@ -33,7 +33,7 @@ import org.apache.texera.amber.engine.common.{CheckpointState, SerializedState} import java.net.URI trait TakeGlobalCheckpointHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def takeGlobalCheckpoint( msg: TakeGlobalCheckpointRequest, @@ -50,7 +50,7 @@ trait TakeGlobalCheckpointHandler { val uri = destinationURI.resolve(msg.checkpointId.toString) var totalSize = 0L val physicalOpIdsToTakeCheckpoint = cp.workflowScheduler.physicalPlan.operators.map(_.id) - controllerInterface + coordinatorInterface .propagateEmbeddedControlMessage( PropagateEmbeddedControlMessageRequest( cp.workflowExecution.getAllRegionExecutions diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/WorkerExecutionCompletedHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/WorkerExecutionCompletedHandler.scala similarity index 85% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/WorkerExecutionCompletedHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/WorkerExecutionCompletedHandler.scala index 65926722f24..5be2fc499f0 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/WorkerExecutionCompletedHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/WorkerExecutionCompletedHandler.scala @@ -17,11 +17,11 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerAsyncRPCHandlerInitializer, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorAsyncRPCHandlerInitializer, ExecutionStateUpdate } import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ @@ -41,7 +41,7 @@ import org.apache.texera.amber.engine.common.virtualidentity.util.SELF * possible sender: worker */ trait WorkerExecutionCompletedHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def workerExecutionCompleted( msg: EmptyRequest, @@ -52,7 +52,7 @@ trait WorkerExecutionCompletedHandler { // because the worker might be killed before the next query statistics interval // and the user sees the last update before completion val statsRequest = - controllerInterface.controllerInitiateQueryStatistics( + coordinatorInterface.coordinatorInitiateQueryStatistics( QueryStatisticsRequest(Seq(ctx.sender), StatisticsUpdateTarget.BOTH_UI_AND_PERSISTENCE), mkContext(SELF) ) @@ -68,8 +68,8 @@ trait WorkerExecutionCompletedHandler { if (isWorkflowTerminal) { // after query result come back: send completed event, cleanup ,and kill workflow sendToClient(ExecutionStateUpdate(cp.workflowExecution.getState)) - cp.controllerTimerService.disableStatusUpdate() - cp.controllerTimerService.disableRuntimeStatisticsCollection() + cp.coordinatorTimerService.disableStatusUpdate() + cp.coordinatorTimerService.disableRuntimeStatisticsCollection() } }) EmptyReturn() diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/WorkerStateUpdatedHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/WorkerStateUpdatedHandler.scala similarity index 89% rename from amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/WorkerStateUpdatedHandler.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/WorkerStateUpdatedHandler.scala index 5ee98a4918d..225eeb9b19f 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/WorkerStateUpdatedHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/WorkerStateUpdatedHandler.scala @@ -17,11 +17,11 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.promisehandlers +package org.apache.texera.amber.engine.architecture.coordinator.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerAsyncRPCHandlerInitializer, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorAsyncRPCHandlerInitializer, ExecutionStatsUpdate, RuntimeStatisticsPersist } @@ -37,7 +37,7 @@ import org.apache.texera.amber.util.VirtualIdentityUtils * possible sender: worker */ trait WorkerStateUpdatedHandler { - this: ControllerAsyncRPCHandlerInitializer => + this: CoordinatorAsyncRPCHandlerInitializer => override def workerStateUpdated( msg: WorkerStateUpdatedRequest, diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/deploysemantics/AddressInfo.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/deploysemantics/AddressInfo.scala index f23e54d736d..5fa8d8fb3e2 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/deploysemantics/AddressInfo.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/deploysemantics/AddressInfo.scala @@ -21,8 +21,8 @@ package org.apache.texera.amber.engine.architecture.deploysemantics import org.apache.pekko.actor.Address -// Holds worker and controller node addresses. +// Holds worker and coordinator node addresses. case class AddressInfo( allAddresses: Array[Address], // e.g., Node 1, Node 2, Node 3 - controllerAddress: Address // Controller node + coordinatorAddress: Address // Coordinator node ) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/deploysemantics/layer/WorkerExecution.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/deploysemantics/layer/WorkerExecution.scala index 55e1e309181..59fe98c8f37 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/deploysemantics/layer/WorkerExecution.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/deploysemantics/layer/WorkerExecution.scala @@ -20,7 +20,7 @@ package org.apache.texera.amber.engine.architecture.deploysemantics.layer import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.engine.architecture.controller.execution.WorkerPortExecution +import org.apache.texera.amber.engine.architecture.coordinator.execution.WorkerPortExecution import org.apache.texera.amber.engine.architecture.worker.statistics.WorkerState.UNINITIALIZED import org.apache.texera.amber.engine.architecture.worker.statistics.{WorkerState, WorkerStatistics} diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala index e65a7b198f5..b594cc8ab03 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/messaginglayer/OutputManager.scala @@ -251,7 +251,7 @@ class OutputManager( * * If the writer thread captured a failure (e.g., iceberg commit retries * exhausted), re-throw it here so the DP thread surfaces a FatalError - * to the controller via pekko's supervisor strategy. Otherwise the worker + * to the coordinator via pekko's supervisor strategy. Otherwise the worker * would announce port completion as if the result was durably written. */ def closeOutputStorageWriterIfNeeded(outputPortId: PortIdentity): Unit = { diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManager.scala index 670014484b3..f92acd3198c 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManager.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManager.scala @@ -31,13 +31,13 @@ import org.apache.texera.amber.engine.architecture.common.{ PekkoActorService, ExecutorDeployment } -import org.apache.texera.amber.engine.architecture.controller.execution.{ +import org.apache.texera.amber.engine.architecture.coordinator.execution.{ OperatorExecution, RegionExecution, WorkflowExecution } -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerConfig, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorConfig, ExecutionStatsUpdate, RuntimeStatisticsPersist, WorkerAssignmentUpdate @@ -55,7 +55,7 @@ import org.apache.texera.amber.engine.architecture.worker.statistics.WorkerState import org.apache.texera.amber.engine.common.AmberLogging import org.apache.texera.amber.engine.common.FutureBijection._ import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.web.SessionState import org.apache.texera.web.model.websocket.event.RegionStateEvent import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource @@ -104,7 +104,7 @@ class RegionExecutionManager( isRestart: Boolean, workflowExecution: WorkflowExecution, asyncRPCClient: AsyncRPCClient, - controllerConfig: ControllerConfig, + coordinatorConfig: CoordinatorConfig, actorService: PekkoActorService, actorRefService: PekkoActorRefMappingService, maxTerminationAttempts: Int = RegionExecutionManager.DefaultMaxTerminationAttempts, @@ -194,7 +194,7 @@ class RegionExecutionManager( // Remove the actorRef so that no other actors can find the worker and send messages. actorRefService.removeActorRef(workerId) // Restarted regions reuse actorId. Remove stale control channels so the - // controller does not reuse old control-message sequence numbers for new workers. + // coordinator does not reuse old control-message sequence numbers for new workers. asyncRPCClient.inputGateway.removeControlChannel(workerId) asyncRPCClient.outputGateway.removeControlChannel(workerId) gracefulStop(actorRef, ScalaDuration(5, TimeUnit.SECONDS)).asTwitter() @@ -413,8 +413,8 @@ class RegionExecutionManager( actorService, operatorExecution, operatorConfig, - controllerConfig.stateRestoreConfOpt, - controllerConfig.faultToleranceConfOpt + coordinatorConfig.stateRestoreConfOpt, + coordinatorConfig.faultToleranceConfOpt ) } @@ -532,9 +532,9 @@ class RegionExecutionManager( private def connectChannels(links: Set[PhysicalLink]): Future[Seq[EmptyReturn]] = { Future.collect( links.map { link: PhysicalLink => - asyncRPCClient.controllerInterface.linkWorkers( + asyncRPCClient.coordinatorInterface.linkWorkers( LinkWorkersRequest(link), - asyncRPCClient.mkContext(CONTROLLER) + asyncRPCClient.mkContext(COORDINATOR) ) }.toSeq ) @@ -659,5 +659,5 @@ class RegionExecutionManager( } } - override def actorId: ActorVirtualIdentity = CONTROLLER + override def actorId: ActorVirtualIdentity = COORDINATOR } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManager.scala index 1c4982f90d7..1f11f052a07 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManager.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManager.scala @@ -26,9 +26,9 @@ import org.apache.texera.amber.engine.architecture.common.{ PekkoActorRefMappingService, PekkoActorService } -import org.apache.texera.amber.engine.architecture.controller.ControllerConfig -import org.apache.texera.amber.engine.architecture.controller.ExecutionStateUpdate -import org.apache.texera.amber.engine.architecture.controller.execution.WorkflowExecution +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorConfig +import org.apache.texera.amber.engine.architecture.coordinator.ExecutionStateUpdate +import org.apache.texera.amber.engine.architecture.coordinator.execution.WorkflowExecution import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient import java.util.concurrent.atomic.AtomicBoolean @@ -36,7 +36,7 @@ import scala.collection.mutable class WorkflowExecutionManager( workflowExecution: WorkflowExecution, - controllerConfig: ControllerConfig, + coordinatorConfig: CoordinatorConfig, asyncRPCClient: AsyncRPCClient ) extends LazyLogging { @@ -107,7 +107,7 @@ class WorkflowExecutionManager( isRestart, workflowExecution, asyncRPCClient, - controllerConfig, + coordinatorConfig, actorService, actorRefService ) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/WorkerConfig.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/WorkerConfig.scala index dd81537d5cb..da773d7a6b5 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/WorkerConfig.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/config/WorkerConfig.scala @@ -57,7 +57,7 @@ case class WorkerConfig( workerId: ActorVirtualIdentity, pveName: String = "", cuid: Option[Int] = None, - // Controller-named, execution-scoped base URI under which this worker's large binaries + // Coordinator-named, execution-scoped base URI under which this worker's large binaries // live; create() appends a unique suffix. Empty when large binaries are unconfigured. largeBinaryBaseUri: String = "" ) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DPThread.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DPThread.scala index fd4d7786c8c..f845836843d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DPThread.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DPThread.scala @@ -48,7 +48,7 @@ class DPThread( dp: DataProcessor, logManager: ReplayLogManager, internalQueue: LinkedBlockingQueue[DPInputQueueElement], - // Controller-named, execution-scoped base URI for large binaries created on this + // Coordinator-named, execution-scoped base URI for large binaries created on this // thread (empty when unconfigured). Seeded into LargeBinaryManager at thread start. largeBinaryBaseUri: String = "" ) extends AmberLogging { diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala index 84f1e8ec659..e88b0016101 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala @@ -57,7 +57,7 @@ import org.apache.texera.amber.engine.architecture.worker.statistics.WorkerState import org.apache.texera.amber.engine.architecture.worker.statistics.WorkerStatistics import org.apache.texera.amber.engine.common.ambermessage._ import org.apache.texera.amber.engine.common.statetransition.WorkerStateManager -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.error.ErrorUtils.{mkConsoleMessage, safely} import java.util.concurrent.LinkedBlockingQueue @@ -167,17 +167,17 @@ class DataProcessor( s"input tuple count = ${statisticsManager.getInputTupleCount}, " + s"output tuple count = ${statisticsManager.getOutputTupleCount}" ) - asyncRPCClient.controllerInterface.workerExecutionCompleted( + asyncRPCClient.coordinatorInterface.workerExecutionCompleted( EmptyRequest(), - asyncRPCClient.mkContext(CONTROLLER) + asyncRPCClient.mkContext(COORDINATOR) ) case FinalizePort(portId, input) => if (!input) { outputManager.closeOutputStorageWriterIfNeeded(portId) } - asyncRPCClient.controllerInterface.portCompleted( + asyncRPCClient.coordinatorInterface.portCompleted( PortCompletedRequest(portId, input), - asyncRPCClient.mkContext(CONTROLLER) + asyncRPCClient.mkContext(COORDINATOR) ) case schemaEnforceable: SchemaEnforceable => val portIdentity = outputPortOpt.getOrElse(outputManager.getSingleOutputPortIdentity) @@ -212,9 +212,9 @@ class DataProcessor( READY, RUNNING, () => { - asyncRPCClient.controllerInterface.workerStateUpdated( + asyncRPCClient.coordinatorInterface.workerStateUpdated( WorkerStateUpdatedRequest(stateManager.getCurrentState), - asyncRPCClient.mkContext(CONTROLLER) + asyncRPCClient.mkContext(COORDINATOR) ) } ) @@ -246,7 +246,7 @@ class DataProcessor( // (recorded in command.context.sender), not to channelId.fromWorkerId. // For ECM-embedded commands those differ: channelId is the data // channel between two workers, while the originator is typically the - // controller. Fall back to the channel sender when the context is + // coordinator. Fall back to the channel sender when the context is // unset (e.g. unit-test inputs). val ctx = command.get.context val replyTo = @@ -300,9 +300,9 @@ class DataProcessor( } def handleExecutorException(e: Throwable): Unit = { - asyncRPCClient.controllerInterface.consoleMessageTriggered( + asyncRPCClient.coordinatorInterface.consoleMessageTriggered( ConsoleMessageTriggeredRequest(mkConsoleMessage(actorId, e)), - asyncRPCClient.mkContext(CONTROLLER) + asyncRPCClient.mkContext(COORDINATOR) ) logger.warn(e.getLocalizedMessage + "\n" + e.getStackTrace.mkString("\n")) // invoke a pause in-place diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/WorkflowWorker.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/WorkflowWorker.scala index 13b1d94b048..51bf59b2f1f 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/WorkflowWorker.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/WorkflowWorker.scala @@ -26,7 +26,7 @@ import org.apache.texera.amber.core.virtualidentity.{ } import org.apache.texera.amber.engine.architecture.common.WorkflowActor import org.apache.texera.amber.engine.architecture.common.WorkflowActor.NetworkAck -import org.apache.texera.amber.engine.architecture.controller.ReplayStatusUpdate +import org.apache.texera.amber.engine.architecture.coordinator.ReplayStatusUpdate import org.apache.texera.amber.engine.architecture.messaginglayer.WorkerTimerService import org.apache.texera.amber.engine.architecture.rpc.controlcommands.ControlInvocation import org.apache.texera.amber.engine.architecture.scheduling.config.WorkerConfig diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala index bcabde00894..f0b814da1de 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThread.scala @@ -37,9 +37,9 @@ class OutputPortStorageWriterThread( Queues.newLinkedBlockingQueue[Either[Tuple, TerminateSignal]]() // Captured failure from put-one or close() so the worker DP thread can - // re-throw and let the controller's pekko supervisor surface a FatalError + // re-throw and let the coordinator's pekko supervisor surface a FatalError // to the client. Without this, the writer thread dies silently and the - // worker keeps reporting normal port completion to the controller while + // worker keeps reporting normal port completion to the coordinator while // results are missing or stale, leading to e2e timeouts that hide the // real cause. @volatile private var failure: Option[Throwable] = None diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandler.scala index 0504e66f52b..f91a28c627b 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandler.scala @@ -35,8 +35,8 @@ trait EndHandler { this: DataProcessorRPCHandlerInitializer => /** - * The response of endWorker to the controller indicates that this worker has finished not only - * the data processing logic, but also , but also the processing of all the control messages. + * The response of endWorker to the coordinator indicates that this worker has finished not only + * the data processing logic, but also the processing of all the control messages. */ override def endWorker( request: EmptyRequest, diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/RecoveryPayload.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/RecoveryPayload.scala index dcd83f3a3fc..5f9302d21a7 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/RecoveryPayload.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/ambermessage/RecoveryPayload.scala @@ -24,11 +24,11 @@ import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity sealed trait RecoveryPayload extends Serializable {} -// Notify controller on worker recovery starts/ends +// Notify coordinator on worker recovery starts/ends final case class UpdateRecoveryStatus(isRecovering: Boolean) extends RecoveryPayload // Notify upstream worker to resend output to another worker for recovery final case class ResendOutputTo(vid: ActorVirtualIdentity, ref: ActorRef) extends RecoveryPayload -// Notify controller when the machine fails and triggers recovery +// Notify coordinator when the machine fails and triggers recovery final case class NotifyFailedNode(addr: Address) extends RecoveryPayload diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/client/AmberClient.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/client/AmberClient.scala index 91d2ba06ba5..7f8df1ca75d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/client/AmberClient.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/client/AmberClient.scala @@ -27,9 +27,9 @@ import io.reactivex.rxjava3.core.Observable import io.reactivex.rxjava3.disposables.Disposable import io.reactivex.rxjava3.subjects.PublishSubject import org.apache.texera.amber.core.workflow.{PhysicalPlan, WorkflowContext} -import org.apache.texera.amber.engine.architecture.controller.ControllerConfig +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorConfig import org.apache.texera.amber.engine.architecture.rpc.controlcommands.ControlRequest -import org.apache.texera.amber.engine.architecture.rpc.controllerservice.ControllerServiceFs2Grpc +import org.apache.texera.amber.engine.architecture.rpc.coordinatorservice.CoordinatorServiceFs2Grpc import org.apache.texera.amber.engine.architecture.rpc.controlreturns.ControlReturn import org.apache.texera.amber.engine.common.FutureBijection._ import org.apache.texera.amber.engine.common.ambermessage.{ @@ -53,7 +53,7 @@ class AmberClient( system: ActorSystem, workflowContext: WorkflowContext, physicalPlan: PhysicalPlan, - controllerConfig: ControllerConfig, + coordinatorConfig: CoordinatorConfig, errorHandler: Throwable => Unit ) { @@ -66,7 +66,7 @@ class AmberClient( clientActor ? InitializeRequest( workflowContext, physicalPlan, - controllerConfig + coordinatorConfig ), 10.seconds ) @@ -78,8 +78,8 @@ class AmberClient( } } - val controllerInterface: ControllerServiceFs2Grpc[Future, Unit] = - createProxy[ControllerServiceFs2Grpc[Future, Unit]]() + val coordinatorInterface: CoordinatorServiceFs2Grpc[Future, Unit] = + createProxy[CoordinatorServiceFs2Grpc[Future, Unit]]() private def createProxy[T]()(implicit ct: ClassTag[T]): T = { val handler = new InvocationHandler { diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/client/ClientActor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/client/ClientActor.scala index 9df86ec3ffe..6cff3282969 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/client/ClientActor.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/client/ClientActor.scala @@ -30,10 +30,10 @@ import org.apache.texera.amber.engine.architecture.common.WorkflowActor.{ NetworkAck, NetworkMessage } -import org.apache.texera.amber.engine.architecture.controller.{ +import org.apache.texera.amber.engine.architecture.coordinator.{ ClientEvent, - Controller, - ControllerConfig + Coordinator, + CoordinatorConfig } import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, @@ -59,7 +59,7 @@ import org.apache.texera.amber.engine.common.client.ClientActor.{ ObservableRequest } import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient -import org.apache.texera.amber.engine.common.virtualidentity.util.{CLIENT, CONTROLLER} +import org.apache.texera.amber.engine.common.virtualidentity.util.{CLIENT, COORDINATOR} import org.apache.texera.amber.error.ErrorUtils.reconstructThrowable import scala.collection.mutable @@ -69,7 +69,7 @@ private[client] object ClientActor { case class InitializeRequest( workflowContext: WorkflowContext, physicalPlan: PhysicalPlan, - controllerConfig: ControllerConfig + coordinatorConfig: CoordinatorConfig ) case class ObservableRequest(pf: PartialFunction[Any, Unit]) @@ -85,7 +85,7 @@ private[client] object ClientActor { private[client] class ClientActor extends Actor with AmberLogging { var actorId: ActorVirtualIdentity = ActorVirtualIdentity("Client") - var controller: ActorRef = _ + var coordinator: ActorRef = _ var controlId = 0L val promiseMap = new mutable.LongMap[Promise[ControlReturn]]() var handlers: PartialFunction[Any, Unit] = PartialFunction.empty @@ -101,10 +101,10 @@ private[client] class ClientActor extends Actor with AmberLogging { } override def receive: Receive = { - case InitializeRequest(workflowContext, physicalPlan, controllerConfig) => - assert(controller == null) - controller = context.actorOf( - Controller.props(workflowContext, physicalPlan, controllerConfig) + case InitializeRequest(workflowContext, physicalPlan, coordinatorConfig) => + assert(coordinator == null) + coordinator = context.actorOf( + Coordinator.props(workflowContext, physicalPlan, coordinatorConfig) ) sender() ! Ack case CreditRequest(channelId: ChannelIdentity) => @@ -117,10 +117,10 @@ private[client] class ClientActor extends Actor with AmberLogging { sender() ! e } case commandRequest: CommandRequest => - controller ! AsyncRPCClient.ControlInvocation( + coordinator ! AsyncRPCClient.ControlInvocation( commandRequest.methodName, commandRequest.command, - AsyncRPCContext(CLIENT, CONTROLLER), + AsyncRPCContext(CLIENT, COORDINATOR), controlId ) promiseMap(controlId) = commandRequest.promise @@ -154,7 +154,7 @@ private[client] class ClientActor extends Actor with AmberLogging { } case x: WorkflowRecoveryMessage => sender() ! Ack - controller ! x + coordinator ! x case other => logger.warn("client actor cannot handle " + other) //skip } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCClient.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCClient.scala index f7e26803b47..88f6f7b3505 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCClient.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCClient.scala @@ -26,13 +26,13 @@ import org.apache.texera.amber.core.virtualidentity.{ ChannelIdentity, EmbeddedControlMessageIdentity } -import org.apache.texera.amber.engine.architecture.controller.ClientEvent +import org.apache.texera.amber.engine.architecture.coordinator.ClientEvent import org.apache.texera.amber.engine.architecture.messaginglayer.{ NetworkInputGateway, NetworkOutputGateway } import org.apache.texera.amber.engine.architecture.rpc.controlcommands._ -import org.apache.texera.amber.engine.architecture.rpc.controllerservice.ControllerServiceFs2Grpc +import org.apache.texera.amber.engine.architecture.rpc.coordinatorservice.CoordinatorServiceFs2Grpc import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{ ControlError, ControlReturn, @@ -135,8 +135,8 @@ class AsyncRPCClient( private val unfulfilledPromises = mutable.HashMap[Long, Promise[ControlReturn]]() private var promiseID = 0L - @transient lazy val controllerInterface: ControllerServiceFs2Grpc[Future, AsyncRPCContext] = - createProxy[ControllerServiceFs2Grpc[Future, AsyncRPCContext]](createPromise, outputGateway) + @transient lazy val coordinatorInterface: CoordinatorServiceFs2Grpc[Future, AsyncRPCContext] = + createProxy[CoordinatorServiceFs2Grpc[Future, AsyncRPCContext]](createPromise, outputGateway) @transient lazy val workerInterface: WorkerServiceFs2Grpc[Future, AsyncRPCContext] = createProxy[WorkerServiceFs2Grpc[Future, AsyncRPCContext]](createPromise, outputGateway) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCHandlerInitializer.scala index f658e0875f7..037c113e965 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCHandlerInitializer.scala @@ -25,9 +25,9 @@ import org.apache.texera.amber.core.virtualidentity.{ ChannelIdentity, EmbeddedControlMessageIdentity } -import org.apache.texera.amber.engine.architecture.controller.ClientEvent +import org.apache.texera.amber.engine.architecture.coordinator.ClientEvent import org.apache.texera.amber.engine.architecture.rpc.controlcommands._ -import org.apache.texera.amber.engine.architecture.rpc.controllerservice.ControllerServiceFs2Grpc +import org.apache.texera.amber.engine.architecture.rpc.coordinatorservice.CoordinatorServiceFs2Grpc import org.apache.texera.amber.engine.architecture.rpc.controlreturns._ import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceFs2Grpc @@ -48,8 +48,8 @@ class AsyncRPCHandlerInitializer( // register all handlers ctrlReceiver.handler = this - def controllerInterface: ControllerServiceFs2Grpc[Future, AsyncRPCContext] = - ctrlSource.controllerInterface + def coordinatorInterface: CoordinatorServiceFs2Grpc[Future, AsyncRPCContext] = + ctrlSource.coordinatorInterface def workerInterface: WorkerServiceFs2Grpc[Future, AsyncRPCContext] = ctrlSource.workerInterface diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/virtualidentity/util.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/virtualidentity/util.scala index 481aecab1dc..a2cf563a96f 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/virtualidentity/util.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/virtualidentity/util.scala @@ -23,7 +23,7 @@ import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity object util { - lazy val CONTROLLER: ActorVirtualIdentity = ActorVirtualIdentity("CONTROLLER") + lazy val COORDINATOR: ActorVirtualIdentity = ActorVirtualIdentity("COORDINATOR") lazy val SELF: ActorVirtualIdentity = ActorVirtualIdentity("SELF") lazy val CLIENT: ActorVirtualIdentity = ActorVirtualIdentity("CLIENT") } diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index d4a061781c9..6616ff47d1e 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -29,7 +29,7 @@ import org.apache.texera.common.config.{ApplicationConfig, StorageConfig} import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity import org.apache.texera.amber.core.workflow.{PhysicalPlan, WorkflowContext} -import org.apache.texera.amber.engine.architecture.controller.ControllerConfig +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorConfig import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.{ COMPLETED, FAILED @@ -69,7 +69,7 @@ object ComputingUnitMaster { def createAmberRuntime( workflowContext: WorkflowContext, physicalPlan: PhysicalPlan, - conf: ControllerConfig, + conf: CoordinatorConfig, errorHandler: Throwable => Unit ): AmberClient = { new AmberClient( diff --git a/amber/src/main/scala/org/apache/texera/web/resource/MockKillWorkerResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/MockKillWorkerResource.scala index d51556f53e8..fda4a232965 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/MockKillWorkerResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/MockKillWorkerResource.scala @@ -30,9 +30,9 @@ // @POST // @Path("/worker") def mockKillWorker: Unit = { // WorkflowWebsocketResource.sessionJobs.foreach(p => { -// val controller = p._2._2 +// val coordinator = p._2._2 // Thread.sleep(1500) -// controller ! KillAndRecover +// coordinator ! KillAndRecover // }) // } // diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala index 72fb1c364e5..a6e0e7f6243 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala @@ -625,7 +625,7 @@ class WorkflowExecutionsResource { val storage = SequentialRecordStorage.getStorage[ReplayLogRecord](Some(new URI(logLocation))) val result = new mutable.ArrayBuffer[EmbeddedControlMessageIdentity]() - storage.getReader("CONTROLLER").mkRecordIterator().foreach { + storage.getReader("COORDINATOR").mkRecordIterator().foreach { case destination: ReplayDestination => result.append(destination.id) case _ => diff --git a/amber/src/main/scala/org/apache/texera/web/service/ExecutionConsoleService.scala b/amber/src/main/scala/org/apache/texera/web/service/ExecutionConsoleService.scala index 3811bfaeb42..3aae5f2b895 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/ExecutionConsoleService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/ExecutionConsoleService.scala @@ -29,7 +29,7 @@ import org.apache.texera.amber.core.storage.{DocumentFactory, VFSURIFactory} import org.apache.texera.amber.core.tuple.Tuple import org.apache.texera.amber.core.virtualidentity.{ActorVirtualIdentity, OperatorIdentity} import org.apache.texera.amber.core.workflow.WorkflowContext -import org.apache.texera.amber.engine.architecture.controller.ExecutionStateUpdate +import org.apache.texera.amber.engine.architecture.coordinator.ExecutionStateUpdate import org.apache.texera.amber.engine.architecture.rpc.controlcommands.ConsoleMessageType.COMMAND import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ ConsoleMessage, @@ -286,7 +286,7 @@ class ExecutionConsoleService( //Receive evaluate python expression addSubscription(wsInput.subscribe((req: PythonExpressionEvaluateRequest, uidOpt) => { val result = Await.result( - client.controllerInterface.evaluatePythonExpression( + client.coordinatorInterface.evaluatePythonExpression( EvaluatePythonExpressionRequest(req.expression, req.operatorId), () ), @@ -325,7 +325,7 @@ class ExecutionConsoleService( addConsoleMessage(consoleStore, req.operatorId, newMessage) } - client.controllerInterface.debugCommand(AmberDebugCommandRequest(req.workerId, req.cmd), ()) + client.coordinatorInterface.debugCommand(AmberDebugCommandRequest(req.workerId, req.cmd), ()) })) diff --git a/amber/src/main/scala/org/apache/texera/web/service/ExecutionReconfigurationService.scala b/amber/src/main/scala/org/apache/texera/web/service/ExecutionReconfigurationService.scala index e7617fdfe16..31611413fde 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/ExecutionReconfigurationService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/ExecutionReconfigurationService.scala @@ -20,7 +20,7 @@ package org.apache.texera.web.service import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity -import org.apache.texera.amber.engine.architecture.controller.{UpdateExecutorCompleted, Workflow} +import org.apache.texera.amber.engine.architecture.coordinator.{UpdateExecutorCompleted, Workflow} import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ UpdateExecutorRequest, WorkflowReconfigureRequest @@ -107,7 +107,7 @@ class ExecutionReconfigurationService( // Seam for unit testing the dispatch path without spinning up an AmberClient. protected def dispatch(request: WorkflowReconfigureRequest): Unit = { - client.controllerInterface.reconfigureWorkflow(request, ()) + client.coordinatorInterface.reconfigureWorkflow(request, ()) } // Seam for unit testing — production wires the engine's UpdateExecutorCompleted diff --git a/amber/src/main/scala/org/apache/texera/web/service/ExecutionResultService.scala b/amber/src/main/scala/org/apache/texera/web/service/ExecutionResultService.scala index 37c803951ec..924ed099c51 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/ExecutionResultService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/ExecutionResultService.scala @@ -35,7 +35,7 @@ import org.apache.texera.amber.core.virtualidentity.{ } import org.apache.texera.amber.core.workflow.OutputPort.OutputMode import org.apache.texera.amber.core.workflow.{PhysicalOp, PhysicalPlan, PortIdentity} -import org.apache.texera.amber.engine.architecture.controller.{ExecutionStateUpdate, FatalError} +import org.apache.texera.amber.engine.architecture.coordinator.{ExecutionStateUpdate, FatalError} import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.{ COMPLETED, FAILED, diff --git a/amber/src/main/scala/org/apache/texera/web/service/ExecutionRuntimeService.scala b/amber/src/main/scala/org/apache/texera/web/service/ExecutionRuntimeService.scala index 70b2f07920a..8b88f416850 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/ExecutionRuntimeService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/ExecutionRuntimeService.scala @@ -21,7 +21,7 @@ package org.apache.texera.web.service import com.typesafe.scalalogging.LazyLogging import org.apache.texera.amber.core.virtualidentity.EmbeddedControlMessageIdentity -import org.apache.texera.amber.engine.architecture.controller.ExecutionStateUpdate +import org.apache.texera.amber.engine.architecture.coordinator.ExecutionStateUpdate import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ EmptyRequest, TakeGlobalCheckpointRequest @@ -85,7 +85,7 @@ class ExecutionRuntimeService( stateStore.metadataStore.updateState(metadataStore => updateWorkflowState(PAUSING, metadataStore) ) - client.controllerInterface.pauseWorkflow(EmptyRequest(), ()) + client.coordinatorInterface.pauseWorkflow(EmptyRequest(), ()) })) // Receive Resume @@ -94,7 +94,7 @@ class ExecutionRuntimeService( stateStore.metadataStore.updateState(metadataStore => updateWorkflowState(RESUMING, metadataStore) ) - client.controllerInterface + client.coordinatorInterface .resumeWorkflow(EmptyRequest(), ()) .onSuccess(_ => stateStore.metadataStore.updateState(metadataStore => @@ -120,7 +120,7 @@ class ExecutionRuntimeService( ) val checkpointId = EmbeddedControlMessageIdentity(s"Checkpoint_${UUID.randomUUID().toString}") val uri = logConf.get.writeTo.resolve(checkpointId.toString) - client.controllerInterface.takeGlobalCheckpoint( + client.coordinatorInterface.takeGlobalCheckpoint( TakeGlobalCheckpointRequest(estimationOnly = false, checkpointId, uri.toString), () ) diff --git a/amber/src/main/scala/org/apache/texera/web/service/ExecutionStatsService.scala b/amber/src/main/scala/org/apache/texera/web/service/ExecutionStatsService.scala index 3703a2bf417..f112f4f65da 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/ExecutionStatsService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/ExecutionStatsService.scala @@ -28,7 +28,7 @@ import org.apache.texera.amber.core.tuple.Tuple import org.apache.texera.amber.core.workflow.WorkflowContext import org.apache.texera.amber.core.workflowruntimestate.FatalErrorType.EXECUTION_FAILURE import org.apache.texera.amber.core.workflowruntimestate.WorkflowFatalError -import org.apache.texera.amber.engine.architecture.controller.{ +import org.apache.texera.amber.engine.architecture.coordinator.{ ExecutionStateUpdate, ExecutionStatsUpdate, FatalError, diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala index 031b9fec70e..43b2288b39f 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala @@ -22,7 +22,7 @@ package org.apache.texera.web.service import com.typesafe.scalalogging.LazyLogging import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.WorkflowContext -import org.apache.texera.amber.engine.architecture.controller.{ControllerConfig, Workflow} +import org.apache.texera.amber.engine.architecture.coordinator.{CoordinatorConfig, Workflow} import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmptyRequest import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState._ import org.apache.texera.amber.engine.common.Utils @@ -55,7 +55,7 @@ object WorkflowExecutionService { } class WorkflowExecutionService( - controllerConfig: ControllerConfig, + coordinatorConfig: CoordinatorConfig, val workflowContext: WorkflowContext, resultService: ExecutionResultService, request: WorkflowExecuteRequest, @@ -120,7 +120,7 @@ class WorkflowExecutionService( client = ComputingUnitMaster.createAmberRuntime( workflow.context, workflow.physicalPlan, - controllerConfig, + coordinatorConfig, errorHandler ) executionReconfigurationService = @@ -131,7 +131,7 @@ class WorkflowExecutionService( executionStateStore, wsInput, executionReconfigurationService, - controllerConfig.faultToleranceConfOpt, + coordinatorConfig.faultToleranceConfOpt, workflowContext.workflowId.id, request.emailNotificationEnabled, userEmailOpt, @@ -154,7 +154,7 @@ class WorkflowExecutionService( executionStateStore.statsStore.updateState(stats => stats.withStartTimeStamp(System.currentTimeMillis()) ) - client.controllerInterface + client.coordinatorInterface .startWorkflow(EmptyRequest(), ()) .onFailure(err => { errorHandler(err) diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index b0a4f0c846c..46e738f35bf 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -35,7 +35,7 @@ import org.apache.texera.amber.core.virtualidentity.{ import org.apache.texera.amber.core.workflow.WorkflowContext import org.apache.texera.amber.core.workflowruntimestate.FatalErrorType.EXECUTION_FAILURE import org.apache.texera.amber.core.workflowruntimestate.WorkflowFatalError -import org.apache.texera.amber.engine.architecture.controller.ControllerConfig +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorConfig import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.{ COMPLETED, FAILED @@ -198,7 +198,7 @@ class WorkflowService( ) val workflowContext: WorkflowContext = createWorkflowContext() - var controllerConf = ControllerConfig.default + var coordinatorConf = CoordinatorConfig.default // clean up results from previous run val previousExecutionId = @@ -222,7 +222,7 @@ class WorkflowService( ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { execution => execution.setLogLocation(writeLocation.toString) } - controllerConf = controllerConf.copy(faultToleranceConfOpt = + coordinatorConf = coordinatorConf.copy(faultToleranceConfOpt = Some(FaultToleranceConfig(writeTo = writeLocation)) ) } @@ -232,7 +232,7 @@ class WorkflowService( .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) .foreach { execution => val readLocation = new URI(execution.getLogLocation) - controllerConf = controllerConf.copy(stateRestoreConfOpt = + coordinatorConf = coordinatorConf.copy(stateRestoreConfOpt = Some( StateRestoreConfig( readFrom = readLocation, @@ -282,7 +282,7 @@ class WorkflowService( // WorkflowErrorEvent that `connectToExecution` forwards. try { val execution = new WorkflowExecutionService( - controllerConf, + coordinatorConf, workflowContext, resultService, req, diff --git a/amber/src/main/scala/org/apache/texera/workflow/WorkflowCompiler.scala b/amber/src/main/scala/org/apache/texera/workflow/WorkflowCompiler.scala index b93aa3e4db3..ae576c4053f 100644 --- a/amber/src/main/scala/org/apache/texera/workflow/WorkflowCompiler.scala +++ b/amber/src/main/scala/org/apache/texera/workflow/WorkflowCompiler.scala @@ -22,7 +22,7 @@ package org.apache.texera.workflow import com.typesafe.scalalogging.LazyLogging import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.core.workflow._ -import org.apache.texera.amber.engine.architecture.controller.Workflow +import org.apache.texera.amber.engine.architecture.coordinator.Workflow import org.apache.texera.web.model.websocket.request.LogicalPlanPojo import scala.collection.mutable diff --git a/amber/src/test/integration/org/apache/texera/amber/engine/e2e/ReconfigurationIntegrationSpec.scala b/amber/src/test/integration/org/apache/texera/amber/engine/e2e/ReconfigurationIntegrationSpec.scala index b4545a19b15..3be29cebe57 100644 --- a/amber/src/test/integration/org/apache/texera/amber/engine/e2e/ReconfigurationIntegrationSpec.scala +++ b/amber/src/test/integration/org/apache/texera/amber/engine/e2e/ReconfigurationIntegrationSpec.scala @@ -29,8 +29,8 @@ import org.apache.texera.amber.core.executor.{OpExecInitInfo, OpExecWithCode} import org.apache.texera.amber.core.tuple.Tuple import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerConfig, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorConfig, ExecutionStateUpdate } import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmptyRequest @@ -134,7 +134,7 @@ class ReconfigurationIntegrationSpec system, workflow.context, workflow.physicalPlan, - ControllerConfig.default, + CoordinatorConfig.default, _ => {} ) val completion = Promise[Unit]() @@ -142,7 +142,7 @@ class ReconfigurationIntegrationSpec if (evt.state == COMPLETED) completion.updateIfEmpty(Return(())) }) Await.result( - client.controllerInterface.startWorkflow(EmptyRequest(), ()), + client.coordinatorInterface.startWorkflow(EmptyRequest(), ()), warmupCap ) Await.result(completion, warmupCap) diff --git a/amber/src/test/python/core/architecture/rpc/test_async_rpc_client.py b/amber/src/test/python/core/architecture/rpc/test_async_rpc_client.py index f4b3c63e1e4..afa048b9981 100644 --- a/amber/src/test/python/core/architecture/rpc/test_async_rpc_client.py +++ b/amber/src/test/python/core/architecture/rpc/test_async_rpc_client.py @@ -30,7 +30,7 @@ ChannelIdentity, ) from proto.org.apache.texera.amber.engine.architecture.rpc import ( - ControllerServiceStub, + CoordinatorServiceStub, ControlReturn, ReturnInvocation, ) @@ -205,25 +205,25 @@ def test_stream_stream_blocked(self): proxy._stream_stream() -class TestControllerStub: - def test_controller_stub_returns_configured_stub(self): +class TestCoordinatorStub: + def test_coordinator_stub_returns_configured_stub(self): client = _make_client() - stub = client.controller_stub() + stub = client.coordinator_stub() # Identity check: same instance every call (lazily configured in __init__). - assert stub is client._controller_service_stub - assert stub is client.controller_stub() + assert stub is client._coordinator_service_stub + assert stub is client.coordinator_stub() - def test_controller_stub_unary_unary_is_rewired_with_async_context(self): + def test_coordinator_stub_unary_unary_is_rewired_with_async_context(self): # AsyncRPCClient.__init__ replaces the stub's `_unary_unary` with the # closure produced by `_assign_context`, then `_wrap_all_async_methods` # wraps that (originally async) function with `async_run`. The end # state is therefore: the handler is no longer the bound method from - # ControllerServiceStub, but a synchronous async_run wrapper. A + # CoordinatorServiceStub, but a synchronous async_run wrapper. A # regression that returned an unconfigured stub would pass the identity # check above, but cannot pass this one. client = _make_client() - stub = client.controller_stub() - baseline = ControllerServiceStub("") + stub = client.coordinator_stub() + baseline = CoordinatorServiceStub("") assert stub._unary_unary is not baseline._unary_unary # The _assign_context wrapper closes over the AsyncRPCClient self, so # if the rewiring really happened the function we end up with mentions @@ -232,15 +232,15 @@ def test_controller_stub_unary_unary_is_rewired_with_async_context(self): target = getattr(stub._unary_unary, "__wrapped__", stub._unary_unary) assert "_assign_context" in target.__qualname__ - def test_controller_stub_async_methods_are_wrapped_with_async_run(self): + def test_coordinator_stub_async_methods_are_wrapped_with_async_run(self): # AsyncRPCClient also runs `_wrap_all_async_methods_with_async_run`, # which replaces every coroutinefunction on the stub with the sync # `async_run` wrapper. So whatever methods were async on a fresh - # `ControllerServiceStub` must now be NON-coroutine on the configured + # `CoordinatorServiceStub` must now be NON-coroutine on the configured # stub. Without this assertion the wrap-all pass could no-op silently. client = _make_client() - stub = client.controller_stub() - baseline = ControllerServiceStub("") + stub = client.coordinator_stub() + baseline = CoordinatorServiceStub("") async_method_names = [ name for name in dir(baseline) @@ -249,7 +249,7 @@ def test_controller_stub_async_methods_are_wrapped_with_async_run(self): ] # Sanity: the upstream stub really does ship with async methods. assert async_method_names, ( - "ControllerServiceStub no longer has any async methods; this test " + "CoordinatorServiceStub no longer has any async methods; this test " "needs to be reconsidered." ) for name in async_method_names: diff --git a/amber/src/test/python/core/runnables/test_console_message.py b/amber/src/test/python/core/runnables/test_console_message.py index fbe9041f96d..22362cfe5f2 100644 --- a/amber/src/test/python/core/runnables/test_console_message.py +++ b/amber/src/test/python/core/runnables/test_console_message.py @@ -55,18 +55,18 @@ def console_message(self): ) @pytest.fixture - def mock_controller_channel(self): + def mock_coordinator_channel(self): return ChannelIdentity( - ActorVirtualIdentity("CONTROLLER"), ActorVirtualIdentity("test"), True + ActorVirtualIdentity("COORDINATOR"), ActorVirtualIdentity("test"), True ) @pytest.mark.timeout(2) def test_console_message_serialization( - self, mock_controller_channel, console_message + self, mock_coordinator_channel, console_message ): """ Test the serialization of the console message - :param mock_controller_channel: the mock control channel id + :param mock_coordinator_channel: the mock control channel id :param console_message: the test message """ # below statements wrap the console message as the python control message @@ -78,7 +78,7 @@ def test_console_message_serialization( ), ) python_control_message = PythonControlMessage( - tag=mock_controller_channel, payload=payload + tag=mock_coordinator_channel, payload=payload ) # serialize the python control message to bytes python_control_message_bytes = bytes(python_control_message) diff --git a/amber/src/test/python/core/runnables/test_data_processor.py b/amber/src/test/python/core/runnables/test_data_processor.py index 61cc5bf7e95..61482363a15 100644 --- a/amber/src/test/python/core/runnables/test_data_processor.py +++ b/amber/src/test/python/core/runnables/test_data_processor.py @@ -145,7 +145,7 @@ def capturing_switch(): assert "boom-from-executor" in str(exc_info[1]) # And the stack-trace console message was queued *before* the # finally-clause switch — without this, the worker would pause - # before ever sending the error to the controller. + # before ever sending the error to the coordinator. assert len(seen_at_switch) == 1 msg = seen_at_switch[0] assert msg.worker_id == "test-worker" diff --git a/amber/src/test/python/core/runnables/test_main_loop.py b/amber/src/test/python/core/runnables/test_main_loop.py index c32f45b8886..2f9a68ccb78 100644 --- a/amber/src/test/python/core/runnables/test_main_loop.py +++ b/amber/src/test/python/core/runnables/test_main_loop.py @@ -139,7 +139,7 @@ def mock_data_output_channel(self): @pytest.fixture def mock_control_input_channel(self): return ChannelIdentity( - ActorVirtualIdentity("CONTROLLER"), + ActorVirtualIdentity("COORDINATOR"), ActorVirtualIdentity("dummy_worker_id"), True, ) @@ -148,7 +148,7 @@ def mock_control_input_channel(self): def mock_control_output_channel(self): return ChannelIdentity( ActorVirtualIdentity("dummy_worker_id"), - ActorVirtualIdentity("CONTROLLER"), + ActorVirtualIdentity("COORDINATOR"), True, ) @@ -717,7 +717,7 @@ def test_main_loop_thread_can_process_messages( command_id=0, context=AsyncRpcContext( sender=ActorVirtualIdentity(name="dummy_worker_id"), - receiver=ActorVirtualIdentity(name="CONTROLLER"), + receiver=ActorVirtualIdentity(name="COORDINATOR"), ), command=ControlRequest( port_completed_request=PortCompletedRequest( @@ -737,7 +737,7 @@ def test_main_loop_thread_can_process_messages( command_id=1, context=AsyncRpcContext( sender=ActorVirtualIdentity(name="dummy_worker_id"), - receiver=ActorVirtualIdentity(name="CONTROLLER"), + receiver=ActorVirtualIdentity(name="COORDINATOR"), ), command=ControlRequest( port_completed_request=PortCompletedRequest( @@ -757,7 +757,7 @@ def test_main_loop_thread_can_process_messages( command_id=2, context=AsyncRpcContext( sender=ActorVirtualIdentity(name="dummy_worker_id"), - receiver=ActorVirtualIdentity(name="CONTROLLER"), + receiver=ActorVirtualIdentity(name="COORDINATOR"), ), command=ControlRequest(empty_request=EmptyRequest()), ) @@ -1730,9 +1730,9 @@ def test_main_loop_thread_can_process_state_after_tuple( def test_console_message_rpc_fires_before_exception_pause( self, main_loop, monkeypatch ): - # Pin the controller-facing contract: when DataProcessor raises + # Pin the coordinator-facing contract: when DataProcessor raises # during an executor call, the stack-trace ConsoleMessage must - # reach the controller *before* the worker enters EXCEPTION_PAUSE + # reach the coordinator *before* the worker enters EXCEPTION_PAUSE # — otherwise the UI sees a paused worker with no error to show # until the user resumes. The DataProcessor side queues the # message before the switch (covered by @@ -1771,7 +1771,7 @@ def test_console_message_rpc_fires_before_exception_pause( kinds = [e[0] for e in events] assert kinds == ["rpc", "pause"], ( - "console message must reach controller before pause; " + "console message must reach coordinator before pause; " f"observed order: {kinds}" ) assert events[0][1].msg_type == ConsoleMessageType.ERROR diff --git a/amber/src/test/python/core/util/test_virtual_identity.py b/amber/src/test/python/core/util/test_virtual_identity.py index b94213c4143..75c8e30a74b 100644 --- a/amber/src/test/python/core/util/test_virtual_identity.py +++ b/amber/src/test/python/core/util/test_virtual_identity.py @@ -57,9 +57,9 @@ def test_handles_multi_digit_indexes(self): assert get_worker_index("Worker:WF42-someOp-layerX-1234") == 1234 def test_raises_value_error_on_unmatched_actor_name(self): - # Companions like CONTROLLER / SELF do not match the worker pattern. + # Companions like COORDINATOR / SELF do not match the worker pattern. with pytest.raises(ValueError, match="Invalid worker ID format"): - get_worker_index("CONTROLLER") + get_worker_index("COORDINATOR") def test_raises_value_error_on_partial_match(self): # Missing trailing index also fails the match. @@ -104,9 +104,9 @@ def test_operator_id_ending_in_digits(self): assert get_logical_op_id("Worker:WF1-op123-main-0") == "op123" def test_raises_value_error_on_special_actor_id(self): - # Companions like CONTROLLER / SELF must fail loudly, not return junk. + # Companions like COORDINATOR / SELF must fail loudly, not return junk. with pytest.raises(ValueError, match="Invalid worker ID format"): - get_logical_op_id("CONTROLLER") + get_logical_op_id("COORDINATOR") def test_raises_value_error_on_partial_match(self): with pytest.raises(ValueError, match="Invalid worker ID format"): diff --git a/amber/src/test/python/pytexera/storage/test_large_binary_manager.py b/amber/src/test/python/pytexera/storage/test_large_binary_manager.py index e1da5eee18b..7bc3d7f5557 100644 --- a/amber/src/test/python/pytexera/storage/test_large_binary_manager.py +++ b/amber/src/test/python/pytexera/storage/test_large_binary_manager.py @@ -25,7 +25,7 @@ # The manager is a singleton; bind the shared instance for the tests. large_binary_manager = LargeBinaryManager() -# Execution-scoped base URI the controller hands down; create() appends a unique suffix. +# Execution-scoped base URI the coordinator hands down; create() appends a unique suffix. TEST_BASE_URI = "s3://texera-large-binaries/objects/1/" @@ -143,7 +143,7 @@ def test_create_appends_unique_suffix_to_base_uri(self): def test_create_matches_execution_scoped_key_shape(monkeypatch): - # The base URI is execution-scoped (controller-named); create() only appends a uuid. + # The base URI is execution-scoped (coordinator-named); create() only appends a uuid. monkeypatch.setattr( StorageConfig, "S3_LARGE_BINARIES_BASE_URI", diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/control/TrivialControlSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/control/TrivialControlSpec.scala index 79726f7fbf0..cb37ddf9130 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/control/TrivialControlSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/control/TrivialControlSpec.scala @@ -40,7 +40,7 @@ import org.apache.texera.amber.engine.architecture.rpc.testerservice.RPCTesterGr import org.apache.texera.amber.engine.common.ambermessage.WorkflowFIFOMessage import org.apache.texera.amber.engine.common.ambermessage.WorkflowMessage.getInMemSize import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient.ControlInvocation -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} @@ -106,7 +106,7 @@ class TrivialControlSpec probe.childActorOf(Props(new TrivialControlTester(id))) idMap(id) = ref } - idMap(CONTROLLER) = probe.ref + idMap(COORDINATOR) = probe.ref var seqNum = 0 cmd.foreach { case (methodName, msg) => @@ -115,12 +115,12 @@ class TrivialControlSpec NetworkMessage( seqNum, WorkflowFIFOMessage( - ChannelIdentity(CONTROLLER, ActorVirtualIdentity("0"), isControl = true), + ChannelIdentity(COORDINATOR, ActorVirtualIdentity("0"), isControl = true), seqNum, ControlInvocation( methodName, msg, - AsyncRPCContext(CONTROLLER, ActorVirtualIdentity("0")), + AsyncRPCContext(COORDINATOR, ActorVirtualIdentity("0")), seqNum ) ) diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/ControllerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/ControllerSpec.scala deleted file mode 100644 index f3f8ca5dd0d..00000000000 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/ControllerSpec.scala +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.engine.architecture.controller - -import org.apache.pekko.actor.{ActorSystem, Props} -import org.apache.pekko.testkit.{ImplicitSender, TestKit} -import org.apache.pekko.util.Timeout -import org.apache.texera.amber.clustering.SingleNodeListener -import org.scalatest.BeforeAndAfterAll -import org.scalatest.flatspec.AnyFlatSpecLike - -import scala.concurrent.ExecutionContextExecutor -import scala.concurrent.duration._ - -class ControllerSpec - extends TestKit(ActorSystem("ControllerSpec")) - with ImplicitSender - with AnyFlatSpecLike - with BeforeAndAfterAll { - - implicit val timeout: Timeout = Timeout(5.seconds) - implicit val executionContext: ExecutionContextExecutor = system.dispatcher - - override def beforeAll(): Unit = { - system.actorOf(Props[SingleNodeListener](), "cluster-info") - } - - override def afterAll(): Unit = { - TestKit.shutdownActorSystem(system) - } - - // private val logicalPlan1 = - // """{ - // |"operators":[ - // |{"tableName":"D:\\large_input.csv","operatorId":"Scan","operatorType":"LocalScanSource","delimiter":","}, - // |{"attributeName":0,"keyword":"Asia","operatorId":"KeywordSearch","operatorType":"KeywordMatcher"}, - // |{"operatorId":"Count","operatorType":"Aggregation"}, - // |{"operatorId":"Sink","operatorType":"Sink"}], - // |"links":[ - // |{"origin":"Scan","destination":"KeywordSearch"}, - // |{"origin":"KeywordSearch","destination":"Count"}, - // |{"origin":"Count","destination":"Sink"}] - // |}""".stripMargin - // - // private val logicalPlan2 = - // """{ - // |"operators":[ - // |{"tableName":"D:\\large_input.csv","operatorId":"Scan","operatorType":"LocalScanSource","delimiter":","}, - // |{"operatorId":"Count","operatorType":"Aggregation"}, - // |{"operatorId":"Sink","operatorType":"Sink"}], - // |"links":[ - // |{"origin":"Scan","destination":"Count"}, - // |{"origin":"Count","destination":"Sink"}] - // |}""".stripMargin - // - // private val logicalPlan3 = - // """{ - // |"operators":[ - // |{"tableName":"D:\\test.txt","operatorId":"Scan","operatorType":"LocalScanSource","delimiter":"|"}, - // |{"attributeName":15,"keyword":"package","operatorId":"KeywordSearch","operatorType":"KeywordMatcher"}, - // |{"operatorId":"Count","operatorType":"Aggregation"}, - // |{"operatorId":"Sink","operatorType":"Sink"}], - // |"links":[ - // |{"origin":"Scan","destination":"KeywordSearch"}, - // |{"origin":"KeywordSearch","destination":"Count"}, - // |{"origin":"Count","destination":"Sink"}] - // |}""".stripMargin - // - // private val logicalPlan4 = - // """{ - // |"operators":[ - // |{"tableName":"D:\\test.txt","operatorId":"Scan1","operatorType":"LocalScanSource","delimiter":"|","indicesToKeep":null}, - // |{"tableName":"D:\\test.txt","operatorId":"Scan2","operatorType":"LocalScanSource","delimiter":"|","indicesToKeep":null}, - // |{"attributeName":15,"keyword":"package","operatorId":"KeywordSearch","operatorType":"KeywordMatcher"}, - // |{"operatorId":"Join","operatorType":"HashJoin","innerTableIndex":0,"outerTableIndex":0}, - // |{"operatorId":"Count","operatorType":"Aggregation"}, - // |{"operatorId":"Sink","operatorType":"Sink"}], - // |"links":[ - // |{"origin":"Scan1","destination":"KeywordSearch"}, - // |{"origin":"KeywordSearch","destination":"Join"}, - // |{"origin":"Scan2","destination":"Join"}, - // |{"origin":"Join","destination":"Count"}, - // |{"origin":"Count","destination":"Sink"}] - // |}""".stripMargin - // - // "A controller" should "be able to set and trigger count breakpoint in the workflow1" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan1)) - // controller ! AckedControllerInitialization - // parent.expectMsg(30.seconds, ReportState(ControllerState.Ready)) - // controller ! PassBreakpointTo("KeywordSearch", new CountGlobalBreakpoint("break1", 100000)) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // var isCompleted = false - // parent.receiveWhile(30.seconds, 10.seconds) { - // case ReportState(ControllerState.Paused) => - // controller ! Resume - // case ReportState(ControllerState.Completed) => - // isCompleted = true - // case _ => - // } - // assert(isCompleted) - // parent.ref ! PoisonPill - // } - // - // "A controller" should "execute the workflow1 normally" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan1)) - // controller ! AckedControllerInitialization - // parent.expectMsg(30.seconds, ReportState(ControllerState.Ready)) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // parent.expectMsg(1.minute, ReportState(ControllerState.Completed)) - // parent.ref ! PoisonPill - // } - // - // "A controller" should "execute the workflow3 normally" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan3)) - // controller ! AckedControllerInitialization - // parent.expectMsg(30.seconds, ReportState(ControllerState.Ready)) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // parent.expectMsg(1.minute, ReportState(ControllerState.Completed)) - // parent.ref ! PoisonPill - // } - // - // "A controller" should "execute the workflow2 normally" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan2)) - // controller ! AckedControllerInitialization - // parent.expectMsg(ReportState(ControllerState.Ready)) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // parent.expectMsg(1.minute, ReportState(ControllerState.Completed)) - // parent.ref ! PoisonPill - // } - // - // "A controller" should "be able to pause/resume the workflow1" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan1)) - // controller ! AckedControllerInitialization - // parent.expectMsg(ReportState(ControllerState.Ready)) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // controller ! Pause - // parent.expectMsg(ReportState(ControllerState.Pausing)) - // parent.expectMsg(ReportState(ControllerState.Paused)) - // controller ! Resume - // parent.expectMsg(ReportState(ControllerState.Resuming)) - // parent.expectMsg(ReportState(ControllerState.Running)) - // controller ! Pause - // parent.expectMsg(ReportState(ControllerState.Pausing)) - // parent.expectMsg(ReportState(ControllerState.Paused)) - // controller ! Resume - // parent.expectMsg(ReportState(ControllerState.Resuming)) - // parent.expectMsg(ReportState(ControllerState.Running)) - // controller ! Pause - // parent.expectMsg(ReportState(ControllerState.Pausing)) - // parent.expectMsg(ReportState(ControllerState.Paused)) - // controller ! Resume - // parent.expectMsg(ReportState(ControllerState.Resuming)) - // parent.expectMsg(ReportState(ControllerState.Running)) - // controller ! Pause - // parent.expectMsg(ReportState(ControllerState.Pausing)) - // parent.expectMsg(ReportState(ControllerState.Paused)) - // controller ! Resume - // parent.expectMsg(ReportState(ControllerState.Resuming)) - // parent.expectMsg(ReportState(ControllerState.Running)) - // parent.expectMsg(1.minute, ReportState(ControllerState.Completed)) - // parent.ref ! PoisonPill - // } - - // "A controller" should "be able to modify the logic after pausing the workflow1" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan1)) - // controller ! AckedControllerInitialization - // parent.expectMsg(30.seconds, ReportState(ControllerState.Ready)) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // Thread.sleep(300) - // controller ! Pause - // parent.expectMsg(ReportState(ControllerState.Pausing)) - // parent.expectMsg(ReportState(ControllerState.Paused)) - // controller ! ModifyLogic( - // new KeywordSearchMetadata( - // OperatorTag("sample", "KeywordSearch"), - // Constants.currentWorkerNum, - // 0, - // "asia" - // ) - // ) - // parent.expectMsg(Ack) - // Thread.sleep(10000) - // controller ! Resume - // parent.expectMsg(ReportState(ControllerState.Resuming)) - // parent.expectMsg(ReportState(ControllerState.Running)) - // parent.expectMsg(1.minute, ReportState(ControllerState.Completed)) - // parent.ref ! PoisonPill - // } - - // "A controller" should "be able to set and trigger conditional breakpoint in the workflow1" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan1)) - // controller ! AckedControllerInitialization - // parent.expectMsg(30.seconds, ReportState(ControllerState.Ready)) - // controller ! PassBreakpointTo( - // "KeywordSearch", - // new ConditionalGlobalBreakpoint("break2", x => x.getString(8).toInt == 9884) - // ) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // var isCompleted = false - // parent.receiveWhile(30.seconds, 10.seconds) { - // case ReportState(ControllerState.Paused) => - // controller ! Resume - // case ReportState(ControllerState.Completed) => - // isCompleted = true - // case _ => - // } - // assert(isCompleted) - // parent.ref ! PoisonPill - // } - // - // "A controller" should "be able to set and trigger count breakpoint on complete in the workflow1" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan1)) - // controller ! AckedControllerInitialization - // parent.expectMsg(30.seconds, ReportState(ControllerState.Ready)) - // controller ! PassBreakpointTo("KeywordSearch", new CountGlobalBreakpoint("break1", 146017)) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // var isCompleted = false - // parent.receiveWhile(30.seconds, 10.seconds) { - // case ReportState(ControllerState.Paused) => - // controller ! Resume - // case ReportState(ControllerState.Completed) => - // isCompleted = true - // case _ => - // } - // assert(isCompleted) - // parent.ref ! PoisonPill - // } - // - // "A controller" should "be able to pause/resume with conditional breakpoint in the workflow1" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan1)) - // controller ! AckedControllerInitialization - // parent.expectMsg(30.seconds, ReportState(ControllerState.Ready)) - // controller ! PassBreakpointTo( - // "KeywordSearch", - // new ConditionalGlobalBreakpoint("break2", x => x.getString(8).toInt == 9884) - // ) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // val random = new Random() - // for (i <- 0 until 100) { - // if (random.nextBoolean()) { - // controller ! Pause - // } else { - // controller ! Resume - // } - // } - // controller ! Resume - // var isCompleted = false - // parent.receiveWhile(30.seconds, 10.seconds) { - // case ReportState(ControllerState.Paused) => - // controller ! Resume - // case ReportState(ControllerState.Completed) => - // isCompleted = true - // case _ => - // } - // assert(isCompleted) - // parent.ref ! PoisonPill - // } - // - // "A controller" should "be able to pause/resume with count breakpoint in the workflow1" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan1)) - // controller ! AckedControllerInitialization - // parent.expectMsg(30.seconds, ReportState(ControllerState.Ready)) - // controller ! PassBreakpointTo("KeywordSearch", new CountGlobalBreakpoint("break1", 100000)) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // val random = new Random() - // for (i <- 0 until 100) { - // if (random.nextBoolean()) { - // controller ! Pause - // } else { - // controller ! Resume - // } - // } - // controller ! Resume - // var isCompleted = false - // parent.receiveWhile(30.seconds, 10.seconds) { - // case ReportState(ControllerState.Paused) => - // controller ! Resume - // case ReportState(ControllerState.Completed) => - // isCompleted = true - // case _ => - // } - // assert(isCompleted) - // parent.ref ! PoisonPill - // } - // - // "A controller" should "execute the workflow4 normally" in { - // val parent = TestProbe() - // val controller = parent.childActorOf(CONTROLLER.props(logicalPlan4)) - // controller ! AckedControllerInitialization - // parent.expectMsg(ReportState(ControllerState.Ready)) - // controller ! Start - // parent.expectMsg(ReportState(ControllerState.Running)) - // parent.expectMsg(1.minute, ReportState(ControllerState.Completed)) - // parent.ref ! PoisonPill - // } - -} diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/ClientEventSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/ClientEventSpec.scala similarity index 99% rename from amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/ClientEventSpec.scala rename to amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/ClientEventSpec.scala index 8d3e7538ce2..54df15cfddd 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/ClientEventSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/ClientEventSpec.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.pekko.actor.ActorSystem import org.apache.pekko.serialization.{Serialization, SerializationExtension} diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorSpec.scala new file mode 100644 index 00000000000..a9328563e18 --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/CoordinatorSpec.scala @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.engine.architecture.coordinator + +import org.apache.pekko.actor.{ActorSystem, Props} +import org.apache.pekko.testkit.{ImplicitSender, TestKit} +import org.apache.pekko.util.Timeout +import org.apache.texera.amber.clustering.SingleNodeListener +import org.scalatest.BeforeAndAfterAll +import org.scalatest.flatspec.AnyFlatSpecLike + +import scala.concurrent.ExecutionContextExecutor +import scala.concurrent.duration._ + +class CoordinatorSpec + extends TestKit(ActorSystem("CoordinatorSpec")) + with ImplicitSender + with AnyFlatSpecLike + with BeforeAndAfterAll { + + implicit val timeout: Timeout = Timeout(5.seconds) + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + override def beforeAll(): Unit = { + system.actorOf(Props[SingleNodeListener](), "cluster-info") + } + + override def afterAll(): Unit = { + TestKit.shutdownActorSystem(system) + } + + // private val logicalPlan1 = + // """{ + // |"operators":[ + // |{"tableName":"D:\\large_input.csv","operatorId":"Scan","operatorType":"LocalScanSource","delimiter":","}, + // |{"attributeName":0,"keyword":"Asia","operatorId":"KeywordSearch","operatorType":"KeywordMatcher"}, + // |{"operatorId":"Count","operatorType":"Aggregation"}, + // |{"operatorId":"Sink","operatorType":"Sink"}], + // |"links":[ + // |{"origin":"Scan","destination":"KeywordSearch"}, + // |{"origin":"KeywordSearch","destination":"Count"}, + // |{"origin":"Count","destination":"Sink"}] + // |}""".stripMargin + // + // private val logicalPlan2 = + // """{ + // |"operators":[ + // |{"tableName":"D:\\large_input.csv","operatorId":"Scan","operatorType":"LocalScanSource","delimiter":","}, + // |{"operatorId":"Count","operatorType":"Aggregation"}, + // |{"operatorId":"Sink","operatorType":"Sink"}], + // |"links":[ + // |{"origin":"Scan","destination":"Count"}, + // |{"origin":"Count","destination":"Sink"}] + // |}""".stripMargin + // + // private val logicalPlan3 = + // """{ + // |"operators":[ + // |{"tableName":"D:\\test.txt","operatorId":"Scan","operatorType":"LocalScanSource","delimiter":"|"}, + // |{"attributeName":15,"keyword":"package","operatorId":"KeywordSearch","operatorType":"KeywordMatcher"}, + // |{"operatorId":"Count","operatorType":"Aggregation"}, + // |{"operatorId":"Sink","operatorType":"Sink"}], + // |"links":[ + // |{"origin":"Scan","destination":"KeywordSearch"}, + // |{"origin":"KeywordSearch","destination":"Count"}, + // |{"origin":"Count","destination":"Sink"}] + // |}""".stripMargin + // + // private val logicalPlan4 = + // """{ + // |"operators":[ + // |{"tableName":"D:\\test.txt","operatorId":"Scan1","operatorType":"LocalScanSource","delimiter":"|","indicesToKeep":null}, + // |{"tableName":"D:\\test.txt","operatorId":"Scan2","operatorType":"LocalScanSource","delimiter":"|","indicesToKeep":null}, + // |{"attributeName":15,"keyword":"package","operatorId":"KeywordSearch","operatorType":"KeywordMatcher"}, + // |{"operatorId":"Join","operatorType":"HashJoin","innerTableIndex":0,"outerTableIndex":0}, + // |{"operatorId":"Count","operatorType":"Aggregation"}, + // |{"operatorId":"Sink","operatorType":"Sink"}], + // |"links":[ + // |{"origin":"Scan1","destination":"KeywordSearch"}, + // |{"origin":"KeywordSearch","destination":"Join"}, + // |{"origin":"Scan2","destination":"Join"}, + // |{"origin":"Join","destination":"Count"}, + // |{"origin":"Count","destination":"Sink"}] + // |}""".stripMargin + // + // "A coordinator" should "be able to set and trigger count breakpoint in the workflow1" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan1)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(30.seconds, ReportState(CoordinatorState.Ready)) + // coordinator ! PassBreakpointTo("KeywordSearch", new CountGlobalBreakpoint("break1", 100000)) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // var isCompleted = false + // parent.receiveWhile(30.seconds, 10.seconds) { + // case ReportState(CoordinatorState.Paused) => + // coordinator ! Resume + // case ReportState(CoordinatorState.Completed) => + // isCompleted = true + // case _ => + // } + // assert(isCompleted) + // parent.ref ! PoisonPill + // } + // + // "A coordinator" should "execute the workflow1 normally" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan1)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(30.seconds, ReportState(CoordinatorState.Ready)) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // parent.expectMsg(1.minute, ReportState(CoordinatorState.Completed)) + // parent.ref ! PoisonPill + // } + // + // "A coordinator" should "execute the workflow3 normally" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan3)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(30.seconds, ReportState(CoordinatorState.Ready)) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // parent.expectMsg(1.minute, ReportState(CoordinatorState.Completed)) + // parent.ref ! PoisonPill + // } + // + // "A coordinator" should "execute the workflow2 normally" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan2)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(ReportState(CoordinatorState.Ready)) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // parent.expectMsg(1.minute, ReportState(CoordinatorState.Completed)) + // parent.ref ! PoisonPill + // } + // + // "A coordinator" should "be able to pause/resume the workflow1" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan1)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(ReportState(CoordinatorState.Ready)) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // coordinator ! Pause + // parent.expectMsg(ReportState(CoordinatorState.Pausing)) + // parent.expectMsg(ReportState(CoordinatorState.Paused)) + // coordinator ! Resume + // parent.expectMsg(ReportState(CoordinatorState.Resuming)) + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // coordinator ! Pause + // parent.expectMsg(ReportState(CoordinatorState.Pausing)) + // parent.expectMsg(ReportState(CoordinatorState.Paused)) + // coordinator ! Resume + // parent.expectMsg(ReportState(CoordinatorState.Resuming)) + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // coordinator ! Pause + // parent.expectMsg(ReportState(CoordinatorState.Pausing)) + // parent.expectMsg(ReportState(CoordinatorState.Paused)) + // coordinator ! Resume + // parent.expectMsg(ReportState(CoordinatorState.Resuming)) + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // coordinator ! Pause + // parent.expectMsg(ReportState(CoordinatorState.Pausing)) + // parent.expectMsg(ReportState(CoordinatorState.Paused)) + // coordinator ! Resume + // parent.expectMsg(ReportState(CoordinatorState.Resuming)) + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // parent.expectMsg(1.minute, ReportState(CoordinatorState.Completed)) + // parent.ref ! PoisonPill + // } + + // "A coordinator" should "be able to modify the logic after pausing the workflow1" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan1)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(30.seconds, ReportState(CoordinatorState.Ready)) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // Thread.sleep(300) + // coordinator ! Pause + // parent.expectMsg(ReportState(CoordinatorState.Pausing)) + // parent.expectMsg(ReportState(CoordinatorState.Paused)) + // coordinator ! ModifyLogic( + // new KeywordSearchMetadata( + // OperatorTag("sample", "KeywordSearch"), + // Constants.currentWorkerNum, + // 0, + // "asia" + // ) + // ) + // parent.expectMsg(Ack) + // Thread.sleep(10000) + // coordinator ! Resume + // parent.expectMsg(ReportState(CoordinatorState.Resuming)) + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // parent.expectMsg(1.minute, ReportState(CoordinatorState.Completed)) + // parent.ref ! PoisonPill + // } + + // "A coordinator" should "be able to set and trigger conditional breakpoint in the workflow1" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan1)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(30.seconds, ReportState(CoordinatorState.Ready)) + // coordinator ! PassBreakpointTo( + // "KeywordSearch", + // new ConditionalGlobalBreakpoint("break2", x => x.getString(8).toInt == 9884) + // ) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // var isCompleted = false + // parent.receiveWhile(30.seconds, 10.seconds) { + // case ReportState(CoordinatorState.Paused) => + // coordinator ! Resume + // case ReportState(CoordinatorState.Completed) => + // isCompleted = true + // case _ => + // } + // assert(isCompleted) + // parent.ref ! PoisonPill + // } + // + // "A coordinator" should "be able to set and trigger count breakpoint on complete in the workflow1" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan1)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(30.seconds, ReportState(CoordinatorState.Ready)) + // coordinator ! PassBreakpointTo("KeywordSearch", new CountGlobalBreakpoint("break1", 146017)) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // var isCompleted = false + // parent.receiveWhile(30.seconds, 10.seconds) { + // case ReportState(CoordinatorState.Paused) => + // coordinator ! Resume + // case ReportState(CoordinatorState.Completed) => + // isCompleted = true + // case _ => + // } + // assert(isCompleted) + // parent.ref ! PoisonPill + // } + // + // "A coordinator" should "be able to pause/resume with conditional breakpoint in the workflow1" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan1)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(30.seconds, ReportState(CoordinatorState.Ready)) + // coordinator ! PassBreakpointTo( + // "KeywordSearch", + // new ConditionalGlobalBreakpoint("break2", x => x.getString(8).toInt == 9884) + // ) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // val random = new Random() + // for (i <- 0 until 100) { + // if (random.nextBoolean()) { + // coordinator ! Pause + // } else { + // coordinator ! Resume + // } + // } + // coordinator ! Resume + // var isCompleted = false + // parent.receiveWhile(30.seconds, 10.seconds) { + // case ReportState(CoordinatorState.Paused) => + // coordinator ! Resume + // case ReportState(CoordinatorState.Completed) => + // isCompleted = true + // case _ => + // } + // assert(isCompleted) + // parent.ref ! PoisonPill + // } + // + // "A coordinator" should "be able to pause/resume with count breakpoint in the workflow1" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan1)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(30.seconds, ReportState(CoordinatorState.Ready)) + // coordinator ! PassBreakpointTo("KeywordSearch", new CountGlobalBreakpoint("break1", 100000)) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // val random = new Random() + // for (i <- 0 until 100) { + // if (random.nextBoolean()) { + // coordinator ! Pause + // } else { + // coordinator ! Resume + // } + // } + // coordinator ! Resume + // var isCompleted = false + // parent.receiveWhile(30.seconds, 10.seconds) { + // case ReportState(CoordinatorState.Paused) => + // coordinator ! Resume + // case ReportState(CoordinatorState.Completed) => + // isCompleted = true + // case _ => + // } + // assert(isCompleted) + // parent.ref ! PoisonPill + // } + // + // "A coordinator" should "execute the workflow4 normally" in { + // val parent = TestProbe() + // val coordinator = parent.childActorOf(COORDINATOR.props(logicalPlan4)) + // coordinator ! AckedCoordinatorInitialization + // parent.expectMsg(ReportState(CoordinatorState.Ready)) + // coordinator ! Start + // parent.expectMsg(ReportState(CoordinatorState.Running)) + // parent.expectMsg(1.minute, ReportState(CoordinatorState.Completed)) + // parent.ref ! PoisonPill + // } + +} diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/GlobalReplayManagerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/GlobalReplayManagerSpec.scala similarity index 98% rename from amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/GlobalReplayManagerSpec.scala rename to amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/GlobalReplayManagerSpec.scala index 2361f874bd7..53cfff38158 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/GlobalReplayManagerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/GlobalReplayManagerSpec.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity import org.scalatest.flatspec.AnyFlatSpec diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowSchedulerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowSchedulerSpec.scala similarity index 90% rename from amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowSchedulerSpec.scala rename to amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowSchedulerSpec.scala index ac7358b438f..ae4ce991d98 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowSchedulerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/WorkflowSchedulerSpec.scala @@ -17,10 +17,10 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller +package org.apache.texera.amber.engine.architecture.coordinator import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.engine.e2e.TestUtils.buildWorkflow import org.apache.texera.amber.operator.TestOperators import org.apache.texera.workflow.LogicalLink @@ -47,7 +47,7 @@ class WorkflowSchedulerSpec extends AnyFlatSpec { "WorkflowScheduler.updateSchedule" should "populate the schedule and physicalPlan fields" in { val workflow = buildHeaderlessCsvKeywordWorkflow() - val scheduler = new WorkflowScheduler(workflow.context, CONTROLLER) + val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) assert(scheduler.getSchedule == null) assert(scheduler.physicalPlan == null) @@ -61,7 +61,7 @@ class WorkflowSchedulerSpec extends AnyFlatSpec { it should "include every workflow operator in some region of the produced schedule" in { val workflow = buildHeaderlessCsvKeywordWorkflow() - val scheduler = new WorkflowScheduler(workflow.context, CONTROLLER) + val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) scheduler.updateSchedule(workflow.physicalPlan) val operatorsInSchedule = scheduler.getSchedule.getRegions @@ -74,7 +74,7 @@ class WorkflowSchedulerSpec extends AnyFlatSpec { "WorkflowScheduler.getNextRegions" should "exhaust the schedule and then return an empty set" in { val workflow = buildHeaderlessCsvKeywordWorkflow() - val scheduler = new WorkflowScheduler(workflow.context, CONTROLLER) + val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) scheduler.updateSchedule(workflow.physicalPlan) val pulledLevels = Iterator @@ -88,7 +88,7 @@ class WorkflowSchedulerSpec extends AnyFlatSpec { it should "yield region sets that together cover every region in the schedule" in { val workflow = buildHeaderlessCsvKeywordWorkflow() - val scheduler = new WorkflowScheduler(workflow.context, CONTROLLER) + val scheduler = new WorkflowScheduler(workflow.context, COORDINATOR) scheduler.updateSchedule(workflow.physicalPlan) val expectedRegions = scheduler.getSchedule.getRegions.toSet diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/ExecutionUtilsSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtilsSpec.scala similarity index 99% rename from amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/ExecutionUtilsSpec.scala rename to amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtilsSpec.scala index cf07c228438..1a66e3fdbbc 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/ExecutionUtilsSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtilsSpec.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/LinkExecutionSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/LinkExecutionSpec.scala similarity index 96% rename from amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/LinkExecutionSpec.scala rename to amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/LinkExecutionSpec.scala index 443a31c2b42..5f115612522 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/LinkExecutionSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/LinkExecutionSpec.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.apache.texera.amber.core.virtualidentity.{ActorVirtualIdentity, ChannelIdentity} import org.scalatest.flatspec.AnyFlatSpec diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/OperatorExecutionSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/OperatorExecutionSpec.scala similarity index 99% rename from amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/OperatorExecutionSpec.scala rename to amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/OperatorExecutionSpec.scala index e8b85391eab..de7017abb5a 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/OperatorExecutionSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/OperatorExecutionSpec.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity import org.apache.texera.amber.core.workflow.PortIdentity diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecutionSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/RegionExecutionSpec.scala similarity index 98% rename from amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecutionSpec.scala rename to amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/RegionExecutionSpec.scala index 4b7ef73bdf0..f5cf857dfdb 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/RegionExecutionSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/RegionExecutionSpec.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.apache.texera.amber.core.virtualidentity.{ ActorVirtualIdentity, @@ -223,7 +223,7 @@ class RegionExecutionSpec extends AnyFlatSpec { // mutable operator / link maps. Registering an operator on one must not // be observable on the other — a regression here (e.g. accidentally // sharing the map via a `val` on the wrapping Region) would corrupt - // controller bookkeeping across regions. + // coordinator bookkeeping across regions. "Two RegionExecutions wrapping equal Regions" should "hold independent operator-execution maps" in { diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkerPortExecutionSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkerPortExecutionSpec.scala similarity index 95% rename from amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkerPortExecutionSpec.scala rename to amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkerPortExecutionSpec.scala index 4f3394debf4..2a6ac75129b 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkerPortExecutionSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkerPortExecutionSpec.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.scalatest.flatspec.AnyFlatSpec diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecutionSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkflowExecutionSpec.scala similarity index 98% rename from amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecutionSpec.scala rename to amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkflowExecutionSpec.scala index 94285e7f8c5..2bb93913324 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/controller/execution/WorkflowExecutionSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/WorkflowExecutionSpec.scala @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.texera.amber.engine.architecture.controller.execution +package org.apache.texera.amber.engine.architecture.coordinator.execution import org.apache.texera.amber.core.executor.OpExecInitInfo import org.apache.texera.amber.core.virtualidentity.{ diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/deploysemantics/AddressInfoSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/deploysemantics/AddressInfoSpec.scala index 0ea16a47dfd..d33b6ebe2c4 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/deploysemantics/AddressInfoSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/deploysemantics/AddressInfoSpec.scala @@ -29,10 +29,10 @@ class AddressInfoSpec extends AnyFlatSpec { "AddressInfo" should "expose the addresses it was constructed with" in { val nodes = Array(addr("h1", 2552), addr("h2", 2552), addr("h3", 2552)) - val controller = addr("ctrl", 2552) - val info = AddressInfo(nodes, controller) + val coordinator = addr("ctrl", 2552) + val info = AddressInfo(nodes, coordinator) assert(info.allAddresses.toList == nodes.toList) - assert(info.controllerAddress == controller) + assert(info.coordinatorAddress == coordinator) } it should "preserve the order of allAddresses" in { @@ -44,26 +44,26 @@ class AddressInfoSpec extends AnyFlatSpec { } it should "accept an empty allAddresses array" in { - // Edge case: no worker nodes (e.g., controller-only configuration). + // Edge case: no worker nodes (e.g., coordinator-only configuration). val info = AddressInfo(Array.empty[Address], addr("ctrl", 0)) assert(info.allAddresses.isEmpty) - assert(info.controllerAddress.host.contains("ctrl")) + assert(info.coordinatorAddress.host.contains("ctrl")) } - it should "allow the controller to also appear in allAddresses (collocated)" in { - val controller = addr("ctrl", 2552) - val info = AddressInfo(Array(controller, addr("worker", 2552)), controller) - assert(info.allAddresses.contains(controller)) - assert(info.controllerAddress == controller) + it should "allow the coordinator to also appear in allAddresses (collocated)" in { + val coordinator = addr("ctrl", 2552) + val info = AddressInfo(Array(coordinator, addr("worker", 2552)), coordinator) + assert(info.allAddresses.contains(coordinator)) + assert(info.coordinatorAddress == coordinator) } it should "support copy(), allowing one field to change while the other is preserved" in { val a = AddressInfo(Array(addr("h1", 1)), addr("ctrl-a", 0)) - val b = a.copy(controllerAddress = addr("ctrl-b", 0)) - assert(b.controllerAddress.host.contains("ctrl-b")) + val b = a.copy(coordinatorAddress = addr("ctrl-b", 0)) + assert(b.coordinatorAddress.host.contains("ctrl-b")) assert(b.allAddresses.toList == a.allAddresses.toList) // original is unchanged - assert(a.controllerAddress.host.contains("ctrl-a")) + assert(a.coordinatorAddress.host.contains("ctrl-a")) } it should "use Array reference equality (not element-wise) for the allAddresses field" in { diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/logreplay/LogreplayPrimitivesSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/logreplay/LogreplayPrimitivesSpec.scala index 6ec33a6988b..642dcb8d9ac 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/logreplay/LogreplayPrimitivesSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/logreplay/LogreplayPrimitivesSpec.scala @@ -342,8 +342,8 @@ class LogreplayPrimitivesSpec extends AnyFlatSpec with BeforeAndAfterAll { } // Production never writes a DataFrame to the replay log: both the - // controller and DP-thread paths filter for `DirectControlMessagePayload` - // before logging (see `Controller.scala` and `DPThread.scala` use of + // coordinator and DP-thread paths filter for `DirectControlMessagePayload` + // before logging (see `Coordinator.scala` and `DPThread.scala` use of // `_.payload.isInstanceOf[DirectControlMessagePayload]`). The trait has // two concrete subtypes that production actually serializes — // `ControlInvocation` (outgoing call) and `ReturnInvocation` (reply) — diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonWorkflowWorkerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonWorkflowWorkerSpec.scala index 1ca0572dca6..9741dc4e453 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonWorkflowWorkerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonWorkflowWorkerSpec.scala @@ -39,7 +39,7 @@ // WorkflowFIFOMessage //} //import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient.{ControlInvocation, ReturnInvocation} -//import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +//import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR //import org.apache.texera.amber.core.virtualidentity.{ // ActorVirtualIdentity, // PhysicalLink, @@ -98,7 +98,7 @@ // controls.foreach { ctrl => // worker ! NetworkMessage( // seq, -// WorkflowFIFOMessage(ChannelIdentity(CONTROLLER, identifier1, true), seq, ctrl) +// WorkflowFIFOMessage(ChannelIdentity(COORDINATOR, identifier1, true), seq, ctrl) // ) // val received = receiveWhile(3.seconds) { // case NetworkAck(id, credits) => diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGeneratorSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGeneratorSpec.scala index bff2917daad..c9c2843acdf 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGeneratorSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGeneratorSpec.scala @@ -26,7 +26,7 @@ import org.apache.texera.amber.core.workflow.{ WorkflowContext, WorkflowSettings } -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.engine.e2e.TestUtils.buildWorkflow import org.apache.texera.amber.operator.TestOperators import org.apache.texera.workflow.LogicalLink @@ -81,7 +81,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val globalSearchNoPruningResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).bottomUpSearch(globalSearch = true, oChains = false, oCleanEdges = false, oEarlyStop = false) // Should have explored all possible states (2^4 states) @@ -90,7 +90,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val globalSearchOChainsResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).bottomUpSearch(globalSearch = true, oCleanEdges = false, oEarlyStop = false) // By applying pruning based on Chains alone, it should skip 10 (8 + 2) states. 8 states where CSV->Build is @@ -102,7 +102,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val globalSearchOCleanEdgesResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).bottomUpSearch(globalSearch = true, oChains = false, oEarlyStop = false) // By applying pruning based on Clean edges (bridges) alone, it should skip 8 states. There is one clean edge @@ -112,7 +112,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val globalSearchOEarlyStopResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).bottomUpSearch(globalSearch = true, oChains = false, oCleanEdges = false) // By applying pruning based on Early Stop alone, only 6 states that are not descendants of a schedulable states @@ -122,7 +122,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val globalSearchAllPruningEnabledResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).bottomUpSearch(globalSearch = true) // By combining all pruning techniques, only 3 states should be visited (1 state where both CSV->KeywordFilter and @@ -176,7 +176,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val globalSearchNoPruningResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).topDownSearch(globalSearch = true, oChains = false, oCleanEdges = false) // Should have explored all possible states (2^4 states) @@ -185,7 +185,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val globalSearchOChainsResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).topDownSearch(globalSearch = true, oCleanEdges = false) // By applying pruning based on Chains alone, it should start with a state where CSV->Build is pipelined because @@ -195,7 +195,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val globalSearchOCleanEdgesResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).topDownSearch(globalSearch = true, oChains = false) // By applying pruning based on Clean Edges (bridges) alone, it should start with a state where Probe->Keyword2 is @@ -205,7 +205,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val globalSearchAllPruningEnabledResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).topDownSearch(globalSearch = true) // By combining both pruning techniques, the search should start with a state where both CSV->Build and @@ -232,7 +232,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val scheduleGenerator = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ) val result = scheduleGenerator.getFullyMaterializedSearchState @@ -278,7 +278,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val scheduleGenerator = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ) val result = scheduleGenerator.getFullyMaterializedSearchState @@ -343,7 +343,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val scheduleGenerator = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ) val result = scheduleGenerator.getFullyMaterializedSearchState @@ -408,7 +408,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val scheduleGenerator = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ) val result = scheduleGenerator.getFullyMaterializedSearchState @@ -487,7 +487,7 @@ class CostBasedScheduleGeneratorSpec extends AnyFlatSpec with MockFactory { val scheduleGenerator = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ) val result = scheduleGenerator.getFullyMaterializedSearchState diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/DefaultCostEstimatorSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/DefaultCostEstimatorSpec.scala index d1b2595cbb5..31158ba2d61 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/DefaultCostEstimatorSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/DefaultCostEstimatorSpec.scala @@ -29,7 +29,7 @@ import org.apache.texera.amber.engine.architecture.scheduling.resourcePolicies.{ DefaultResourceAllocator, ExecutionClusterInfo } -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.engine.e2e.TestUtils.buildWorkflow import org.apache.texera.amber.operator.TestOperators import org.apache.texera.amber.operator.aggregate.{AggregateOpDesc, AggregationFunction} @@ -142,7 +142,7 @@ class DefaultCostEstimatorSpec val costEstimator = new DefaultCostEstimator( workflow.context, resourceAllocator, - CONTROLLER + COORDINATOR ) val ports = workflow.physicalPlan.operators.flatMap(op => op.inputPorts.keys @@ -236,7 +236,7 @@ class DefaultCostEstimatorSpec val costEstimator = new DefaultCostEstimator( workflow.context, resourceAllocator, - CONTROLLER + COORDINATOR ) val ports = workflow.physicalPlan.operators.flatMap(op => @@ -348,7 +348,7 @@ class DefaultCostEstimatorSpec val searchResult = new CostBasedScheduleGenerator( workflow.context, workflow.physicalPlan, - CONTROLLER + COORDINATOR ).bottomUpSearch() val groupByRegion = @@ -366,7 +366,7 @@ class DefaultCostEstimatorSpec val costEstimator = new DefaultCostEstimator( workflow.context, resourceAllocator, - CONTROLLER + COORDINATOR ) val (_, groupByRegionCost) = costEstimator.allocateResourcesAndEstimateCost(groupByRegion, 1) diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerSpec.scala index 482af953d4a..62f59eee956 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerSpec.scala @@ -35,8 +35,8 @@ import org.apache.texera.amber.core.workflow.WorkflowContext.{ DEFAULT_WORKFLOW_ID } import org.apache.texera.amber.engine.architecture.common.PekkoActorRefMappingService -import org.apache.texera.amber.engine.architecture.controller.ControllerConfig -import org.apache.texera.amber.engine.architecture.controller.execution.WorkflowExecution +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorConfig +import org.apache.texera.amber.engine.architecture.coordinator.execution.WorkflowExecution import org.apache.texera.amber.engine.architecture.rpc.controlreturns._ import org.apache.texera.amber.engine.architecture.scheduling.RegionExecutionManagerTestSupport._ import org.apache.texera.amber.engine.architecture.scheduling.config.{ @@ -47,7 +47,7 @@ import org.apache.texera.amber.engine.architecture.scheduling.config.{ } import org.apache.texera.amber.engine.architecture.worker.statistics.WorkerState import org.apache.texera.amber.engine.common.AmberRuntime -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.scalatest.BeforeAndAfterAll import org.scalatest.flatspec.AnyFlatSpecLike @@ -58,7 +58,7 @@ import java.util.concurrent.atomic * Tests the real region-execution lifecycle around synchronous region kill. * * The tests let the manager call the real `AsyncRPCClient.workerInterface`, capture the generated - * `ControlInvocation`s at the controller output gateway, and fulfill those RPC promises + * `ControlInvocation`s at the coordinator output gateway, and fulfill those RPC promises * explicitly. This keeps the important production behavior under test: * * - regular launch RPCs (`initializeExecutor`, `openExecutor`, `startWorker`) are allowed to @@ -298,24 +298,24 @@ class RegionExecutionManagerSpec seedReusableWorkerExecution(workflowExecution, seedRegionId = 0, physicalOp, workerId) workflowExecution.initRegionExecution(region) - val rpcProbe = new ControllerRpcProbe(_ => None) - val controller = createControllerHarness() - registerLiveWorker(controller.actorRefService, workerId) + val rpcProbe = new CoordinatorRpcProbe(_ => None) + val coordinator = createCoordinatorHarness() + registerLiveWorker(coordinator.actorRefService, workerId) new RegionExecutionManager( region, isRestart = false, workflowExecution, rpcProbe.asyncRPCClient, - ControllerConfig(None, None, None, None), - controller.actorService, - controller.actorRefService + CoordinatorConfig(None, None, None, None), + coordinator.actorService, + coordinator.actorRefService ) } private case class SingleRegionFixture( manager: RegionExecutionManager, - rpcProbe: ControllerRpcProbe, + rpcProbe: CoordinatorRpcProbe, workflowExecution: WorkflowExecution, region: Region, physicalOp: PhysicalOp, @@ -336,14 +336,14 @@ class RegionExecutionManagerSpec seedReusableWorkerExecution(workflowExecution, seedRegionId = 0, physicalOp, workerId) workflowExecution.initRegionExecution(region) - val rpcProbe = new ControllerRpcProbe(endWorkerResponse) - val controller = createControllerHarness() - registerLiveWorker(controller.actorRefService, workerId) + val rpcProbe = new CoordinatorRpcProbe(endWorkerResponse) + val coordinator = createCoordinatorHarness() + registerLiveWorker(coordinator.actorRefService, workerId) // Seed stale control channels to verify that successful termination removes them. - rpcProbe.inputGateway.getChannel(ChannelIdentity(workerId, CONTROLLER, isControl = true)) + rpcProbe.inputGateway.getChannel(ChannelIdentity(workerId, COORDINATOR, isControl = true)) rpcProbe.outputGateway.getSequenceNumber( - ChannelIdentity(CONTROLLER, workerId, isControl = true) + ChannelIdentity(COORDINATOR, workerId, isControl = true) ) val manager = new RegionExecutionManager( @@ -351,9 +351,9 @@ class RegionExecutionManagerSpec isRestart = false, workflowExecution, rpcProbe.asyncRPCClient, - ControllerConfig(None, None, None, None), - controller.actorService, - controller.actorRefService, + CoordinatorConfig(None, None, None, None), + coordinator.actorService, + coordinator.actorRefService, maxTerminationAttempts, killRetryDelay ) @@ -365,13 +365,13 @@ class RegionExecutionManagerSpec region = region, physicalOp = physicalOp, workerId = workerId, - actorRefService = controller.actorRefService + actorRefService = coordinator.actorRefService ) } private case class MultiWorkerFixture( manager: RegionExecutionManager, - rpcProbe: ControllerRpcProbe, + rpcProbe: CoordinatorRpcProbe, workerIds: Seq[ActorVirtualIdentity] ) @@ -390,18 +390,18 @@ class RegionExecutionManagerSpec seedReusableWorkerExecutions(workflowExecution, seedRegionId = 0, physicalOp, workerIds) workflowExecution.initRegionExecution(region) - val rpcProbe = new ControllerRpcProbe(_ => Some(transientEndWorkerFailure)) - val controller = createControllerHarness() - workerIds.foreach(registerLiveWorker(controller.actorRefService, _)) + val rpcProbe = new CoordinatorRpcProbe(_ => Some(transientEndWorkerFailure)) + val coordinator = createCoordinatorHarness() + workerIds.foreach(registerLiveWorker(coordinator.actorRefService, _)) val manager = new RegionExecutionManager( region, isRestart = false, workflowExecution, rpcProbe.asyncRPCClient, - ControllerConfig(None, None, None, None), - controller.actorService, - controller.actorRefService, + CoordinatorConfig(None, None, None, None), + coordinator.actorService, + coordinator.actorRefService, maxTerminationAttempts, killRetryDelay ) @@ -429,12 +429,12 @@ class RegionExecutionManagerSpec private def assertControlChannelsAreRemoved(fixture: SingleRegionFixture): Unit = { assert( !fixture.rpcProbe.inputGateway.getAllControlChannels.exists( - _.channelId == ChannelIdentity(fixture.workerId, CONTROLLER, isControl = true) + _.channelId == ChannelIdentity(fixture.workerId, COORDINATOR, isControl = true) ) ) assert( !fixture.rpcProbe.outputGateway.getActiveChannels.exists( - _ == ChannelIdentity(CONTROLLER, fixture.workerId, isControl = true) + _ == ChannelIdentity(COORDINATOR, fixture.workerId, isControl = true) ) ) } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerTestSupport.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerTestSupport.scala index 1ad413244c0..8a012c0c209 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerTestSupport.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerTestSupport.scala @@ -39,7 +39,7 @@ import org.apache.texera.amber.engine.architecture.common.{ PekkoActorService, WorkflowActor } -import org.apache.texera.amber.engine.architecture.controller.execution.WorkflowExecution +import org.apache.texera.amber.engine.architecture.coordinator.execution.WorkflowExecution import org.apache.texera.amber.engine.architecture.messaginglayer.{ NetworkInputGateway, NetworkOutputGateway @@ -55,7 +55,7 @@ import org.apache.texera.amber.engine.architecture.worker.statistics.WorkerState import org.apache.texera.amber.engine.common.CheckpointState import org.apache.texera.amber.engine.common.ambermessage.WorkflowFIFOMessage import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.util.VirtualIdentityUtils import scala.collection.mutable @@ -77,24 +77,24 @@ object RegionExecutionManagerTestSupport { commandId: Long ) - case class ControllerHarnessFixture( + case class CoordinatorHarnessFixture( actorService: PekkoActorService, actorRefService: PekkoActorRefMappingService ) /** - * Captures controller-to-worker RPCs at the same boundary used by production + * Captures coordinator-to-worker RPCs at the same boundary used by production * `AsyncRPCClient.workerInterface`. * * Non-termination RPCs are completed immediately because these tests focus on termination * ordering. `endWorker` responses are controlled by `endWorkerResponse`, allowing each test to * hold termination pending, fail an attempt, or allow it to succeed. */ - class ControllerRpcProbe(endWorkerResponse: WorkerRpcCall => Option[ControlReturn]) { + class CoordinatorRpcProbe(endWorkerResponse: WorkerRpcCall => Option[ControlReturn]) { val calls: mutable.ArrayBuffer[WorkerRpcCall] = mutable.ArrayBuffer() - val inputGateway = new NetworkInputGateway(CONTROLLER) - val outputGateway = new NetworkOutputGateway(CONTROLLER, handleOutput) - val asyncRPCClient = new AsyncRPCClient(inputGateway, outputGateway, CONTROLLER) + val inputGateway = new NetworkInputGateway(COORDINATOR) + val outputGateway = new NetworkOutputGateway(COORDINATOR, handleOutput) + val asyncRPCClient = new AsyncRPCClient(inputGateway, outputGateway, COORDINATOR) def methodTrace: Seq[String] = calls.map(_.methodName).toSeq @@ -153,7 +153,7 @@ object RegionExecutionManagerTestSupport { override def receive: Receive = { case _ => () } } - class ControllerHarness extends WorkflowActor(None, CONTROLLER) { + class CoordinatorHarness extends WorkflowActor(None, COORDINATOR) { override def handleInputMessage(id: Long, workflowMsg: WorkflowFIFOMessage): Unit = () override def getQueuedCredit(channelId: ChannelIdentity): Long = 0 @@ -241,13 +241,13 @@ object RegionExecutionManagerTestSupport { trait RegionExecutionManagerTestSupport { self: TestKit => import RegionExecutionManagerTestSupport._ - protected def createControllerHarness(): ControllerHarnessFixture = { - val controllerRef = TestActorRef(new ControllerHarness) - controllerRef.underlyingActor.actorService.getAvailableNodeAddressesFunc = () => - Array(controllerRef.path.address) - ControllerHarnessFixture( - actorService = controllerRef.underlyingActor.actorService, - actorRefService = controllerRef.underlyingActor.actorRefMappingService + protected def createCoordinatorHarness(): CoordinatorHarnessFixture = { + val coordinatorRef = TestActorRef(new CoordinatorHarness) + coordinatorRef.underlyingActor.actorService.getAvailableNodeAddressesFunc = () => + Array(coordinatorRef.path.address) + CoordinatorHarnessFixture( + actorService = coordinatorRef.underlyingActor.actorService, + actorRefService = coordinatorRef.underlyingActor.actorRefMappingService ) } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManagerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManagerSpec.scala index 572b8322770..7ddc7d29678 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManagerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManagerSpec.scala @@ -29,8 +29,8 @@ import org.apache.texera.amber.core.virtualidentity.{ WorkflowIdentity } import org.apache.texera.amber.core.workflow.PhysicalOp -import org.apache.texera.amber.engine.architecture.controller.ControllerConfig -import org.apache.texera.amber.engine.architecture.controller.execution.WorkflowExecution +import org.apache.texera.amber.engine.architecture.coordinator.CoordinatorConfig +import org.apache.texera.amber.engine.architecture.coordinator.execution.WorkflowExecution import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn import org.apache.texera.amber.engine.architecture.scheduling.RegionExecutionManagerTestSupport._ import org.apache.texera.amber.engine.common.AmberRuntime @@ -117,35 +117,35 @@ class WorkflowExecutionManagerSpec // First region's worker holds endWorker pending until we explicitly fulfill it; the second // region's worker terminates immediately. This lets us assert the second region cannot start // until termination of the first finishes. - val rpcProbe = new ControllerRpcProbe( + val rpcProbe = new CoordinatorRpcProbe( endWorkerResponse = call => if (call.receiver == firstWorkerId) None else Some(EmptyReturn()) ) - val controller = createControllerHarness() - registerLiveWorker(controller.actorRefService, firstWorkerId) - registerLiveWorker(controller.actorRefService, secondWorkerId) + val coordinator = createCoordinatorHarness() + registerLiveWorker(coordinator.actorRefService, firstWorkerId) + registerLiveWorker(coordinator.actorRefService, secondWorkerId) val workflowManager = new WorkflowExecutionManager( workflowExecution, - ControllerConfig(None, None, None, None), + CoordinatorConfig(None, None, None, None), rpcProbe.asyncRPCClient ) workflowManager.schedule = Schedule(Map(0 -> Set(firstRegion), 1 -> Set(secondRegion))) - workflowManager.setupActorRefService(controller.actorRefService) + workflowManager.setupActorRefService(coordinator.actorRefService) - await(workflowManager.advanceRegionExecutions(controller.actorService)) + await(workflowManager.advanceRegionExecutions(coordinator.actorService)) assert(rpcProbe.startedWorkers == Seq(firstWorkerId)) - val advanceFuture = workflowManager.advanceRegionExecutions(controller.actorService) + val advanceFuture = workflowManager.advanceRegionExecutions(coordinator.actorService) waitUntil(rpcProbe.endWorkerCalls.size == 1) assert(advanceFuture.poll.isEmpty) assert(!rpcProbe.initializedWorkers.contains(secondWorkerId)) - assert(controller.actorRefService.hasActorRef(firstWorkerId)) + assert(coordinator.actorRefService.hasActorRef(firstWorkerId)) rpcProbe.fulfill(rpcProbe.onlyEndWorkerCall, EmptyReturn()) await(advanceFuture) - assert(!controller.actorRefService.hasActorRef(firstWorkerId)) + assert(!coordinator.actorRefService.hasActorRef(firstWorkerId)) assert(rpcProbe.initializedWorkers.contains(secondWorkerId)) assert(rpcProbe.startedWorkers.contains(secondWorkerId)) } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorSpec.scala index 0f4840f9653..177b2f4af90 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorSpec.scala @@ -45,7 +45,7 @@ import org.apache.texera.amber.engine.architecture.worker.statistics.WorkerState import org.apache.texera.amber.engine.common.ambermessage.{DataFrame, WorkflowFIFOMessage} import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient.ControlInvocation import org.apache.texera.amber.engine.common.storage.SequentialRecordStorage -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.util.VirtualIdentityUtils import org.scalamock.scalatest.MockFactory import org.scalatest.BeforeAndAfterEach @@ -144,11 +144,11 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with BeforeAndAfter .setPortId(inputPortId) dp.outputManager.addPort(outputPortId, schema, None) dp.processDCM( - ChannelIdentity(CONTROLLER, testWorkerId, isControl = true), + ChannelIdentity(COORDINATOR, testWorkerId, isControl = true), ControlInvocation( METHOD_OPEN_EXECUTOR, EmptyRequest(), - AsyncRPCContext(CONTROLLER, testWorkerId), + AsyncRPCContext(COORDINATOR, testWorkerId), 0 ) ) @@ -205,11 +205,11 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with BeforeAndAfter .setPortId(inputPortId) dp.outputManager.addPort(outputPortId, schema, None) dp.processDCM( - ChannelIdentity(CONTROLLER, testWorkerId, isControl = true), + ChannelIdentity(COORDINATOR, testWorkerId, isControl = true), ControlInvocation( METHOD_OPEN_EXECUTOR, EmptyRequest(), - AsyncRPCContext(CONTROLLER, testWorkerId), + AsyncRPCContext(COORDINATOR, testWorkerId), 0 ) ) @@ -219,11 +219,11 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with BeforeAndAfter ) while (dp.inputManager.hasUnfinishedInput || dp.outputManager.hasUnfinishedOutput) { dp.processDCM( - ChannelIdentity(CONTROLLER, testWorkerId, isControl = true), + ChannelIdentity(COORDINATOR, testWorkerId, isControl = true), ControlInvocation( METHOD_FLUSH_NETWORK_BUFFER, EmptyRequest(), - AsyncRPCContext(CONTROLLER, testWorkerId), + AsyncRPCContext(COORDINATOR, testWorkerId), 1 ) ) diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/WorkerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/WorkerSpec.scala index df9cb086a60..95933a90b19 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/WorkerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/WorkerSpec.scala @@ -48,7 +48,7 @@ import org.apache.texera.amber.engine.common.ambermessage.{ WorkflowFIFOMessage } import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.scalamock.scalatest.MockFactory import org.scalatest.BeforeAndAfterAll import org.scalatest.flatspec.AnyFlatSpecLike @@ -116,7 +116,7 @@ class WorkerSpec controls.foreach { ctrl => worker ! NetworkMessage( seq, - WorkflowFIFOMessage(ChannelIdentity(CONTROLLER, identifier1, isControl = true), seq, ctrl) + WorkflowFIFOMessage(ChannelIdentity(COORDINATOR, identifier1, isControl = true), seq, ctrl) ) seq += 1 } @@ -168,19 +168,19 @@ class WorkerSpec val invocation = AsyncRPCClient.ControlInvocation( METHOD_ADD_PARTITIONING, AddPartitioningRequest(mockLink, mockPolicy), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 0 ) val addPort1 = AsyncRPCClient.ControlInvocation( METHOD_ASSIGN_PORT, AssignPortRequest(mockPortId, input = true, mkSchema(1).toRawSchema, List(""), List()), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 1 ) val addPort2 = AsyncRPCClient.ControlInvocation( METHOD_ASSIGN_PORT, AssignPortRequest(mockPortId, input = false, mkSchema(1).toRawSchema, List(""), List()), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 2 ) val addInputChannel = AsyncRPCClient.ControlInvocation( @@ -189,7 +189,7 @@ class WorkerSpec ChannelIdentity(identifier2, identifier1, isControl = false), mockLink.toPortId ), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 3 ) @@ -202,7 +202,7 @@ class WorkerSpec ), isSource = false ), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 4 ) sendControlToWorker( @@ -225,7 +225,7 @@ class WorkerSpec worker ! AsyncRPCClient.ControlInvocation( METHOD_FLUSH_NETWORK_BUFFER, EmptyRequest(), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 1 ) //wait test to finish diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThreadSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThreadSpec.scala index d7ab0c18314..9cc8216efab 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThreadSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/OutputPortStorageWriterThreadSpec.scala @@ -106,7 +106,7 @@ class OutputPortStorageWriterThreadSpec extends AnyFlatSpec { // install a writer thread whose close() has already failed. This pins the // contract that closeOutputStorageWriterIfNeeded re-throws the captured // failure, which is the bridge from the writer thread to the DP thread → - // worker actor → controller supervisor → FatalError to client. + // worker actor → coordinator supervisor → FatalError to client. private def installWriterThread( manager: OutputManager, portId: PortIdentity, diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/SerializationManagerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/SerializationManagerSpec.scala index 64af073ab7f..bbbb6eb1970 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/SerializationManagerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/managers/SerializationManagerSpec.scala @@ -50,7 +50,7 @@ class SerializationManagerSpec extends AnyFlatSpec { // A non-worker actor id (created via the plain string constructor, not the // worker-identity factory) — VirtualIdentityUtils.getWorkerIndex will // return None for this, triggering the IllegalStateException guard. - private val controllerActorId: ActorVirtualIdentity = ActorVirtualIdentity("controller") + private val coordinatorActorId: ActorVirtualIdentity = ActorVirtualIdentity("coordinator") private def mkRequest(info: OpExecInitInfo, totalWorkers: Int = 1): InitializeExecutorRequest = InitializeExecutorRequest( @@ -61,7 +61,7 @@ class SerializationManagerSpec extends AnyFlatSpec { "SerializationManager.restoreExecutorState" should "throw IllegalStateException when actorId is not a worker identity" in { - val mgr = new SerializationManager(controllerActorId) + val mgr = new SerializationManager(coordinatorActorId) mgr.setOpInitialization( mkRequest( OpExecWithClassName( diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandlerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandlerSpec.scala index bdfd5598310..3a3c0ecb9ed 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandlerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndHandlerSpec.scala @@ -40,20 +40,20 @@ import org.apache.texera.amber.engine.architecture.worker.{ import org.apache.texera.amber.engine.common.actormessage.Backpressure import org.apache.texera.amber.engine.common.ambermessage.WorkflowFIFOMessage import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient.ControlInvocation -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.scalatest.flatspec.AnyFlatSpec import java.util.concurrent.LinkedBlockingQueue /** - * `endWorker` is the controller's acknowledgement point before it sends actor-level `gracefulStop`. + * `endWorker` is the coordinator's acknowledgement point before it sends actor-level `gracefulStop`. * * A successful reply means the worker has drained every queued workflow message. If the queue still contains work, * the handler must fail so the region execution manager can retry the kill instead of stopping the actor too early. */ class EndHandlerSpec extends AnyFlatSpec { private val workerId = ActorVirtualIdentity("Worker:WF1-test-op-main-0") - private val rpcContext = AsyncRPCContext(CONTROLLER, workerId) + private val rpcContext = AsyncRPCContext(COORDINATOR, workerId) private val awaitTimeout = Duration.fromSeconds(1) private def createEndHandlerForQueue( @@ -78,7 +78,7 @@ class EndHandlerSpec extends AnyFlatSpec { queue.put( FIFOMessageElement( WorkflowFIFOMessage( - ChannelIdentity(CONTROLLER, workerId, isControl = true), + ChannelIdentity(COORDINATOR, workerId, isControl = true), 0, ControlInvocation(METHOD_QUERY_STATISTICS, EmptyRequest(), rpcContext, 1) ) diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/common/CheckpointSubsystemSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/common/CheckpointSubsystemSpec.scala index 6b46030b6bc..bd3a9892f77 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/common/CheckpointSubsystemSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/common/CheckpointSubsystemSpec.scala @@ -33,7 +33,7 @@ class CheckpointSubsystemSpec extends AnyFlatSpec with BeforeAndAfterAll { // reflection so that CheckpointState.save/load (which hard-code // AmberRuntime.serde) reuse the same system. Both the suite-local system // and AmberRuntime's reference are torn down in afterAll, so no Pekko - // threads outlive the test (matching ControllerSpec/WorkerSpec hygiene). + // threads outlive the test (matching CoordinatorSpec/WorkerSpec hygiene). private val testSystem: ActorSystem = ActorSystem("CheckpointSubsystemSpec-test", AmberRuntime.pekkoConfig) private val testSerde: Serialization = SerializationExtension(testSystem) diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/common/ambermessage/AmberMessageEnvelopesSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/common/ambermessage/AmberMessageEnvelopesSpec.scala index fa03b5acef3..bcf32f1b2a4 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/common/ambermessage/AmberMessageEnvelopesSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/common/ambermessage/AmberMessageEnvelopesSpec.scala @@ -30,7 +30,7 @@ class AmberMessageEnvelopesSpec extends AnyFlatSpec with BeforeAndAfterAll { // Suite-local actor system used only by the ResendOutputTo test below; // shut down via TestKit.shutdownActorSystem in afterAll so threads do not - // outlive the test, matching the cleanup pattern in ControllerSpec / + // outlive the test, matching the cleanup pattern in CoordinatorSpec / // WorkerSpec. private val pekkoSystem: ActorSystem = ActorSystem("amber-message-envelopes-test") diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/BatchSizePropagationSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/BatchSizePropagationSpec.scala index e9b830bdfdc..3c819319934 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/BatchSizePropagationSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/BatchSizePropagationSpec.scala @@ -24,9 +24,9 @@ import org.apache.pekko.testkit.{ImplicitSender, TestKit} import org.apache.pekko.util.Timeout import org.apache.texera.amber.clustering.SingleNodeListener import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext, WorkflowSettings} -import org.apache.texera.amber.engine.architecture.controller._ +import org.apache.texera.amber.engine.architecture.coordinator._ import org.apache.texera.amber.engine.architecture.sendsemantics.partitionings._ -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.apache.texera.amber.engine.e2e.TestUtils.buildWorkflow import org.apache.texera.amber.operator.TestOperators import org.apache.texera.amber.operator.aggregate.AggregationFunction @@ -132,7 +132,7 @@ class BatchSizePropagationSpec context ) - val workflowScheduler = new WorkflowScheduler(context, CONTROLLER) + val workflowScheduler = new WorkflowScheduler(context, COORDINATOR) workflowScheduler.updateSchedule(workflow.physicalPlan) verifyBatchSizeInPartitioning(workflowScheduler, 1) @@ -162,7 +162,7 @@ class BatchSizePropagationSpec context ) - val workflowScheduler = new WorkflowScheduler(context, CONTROLLER) + val workflowScheduler = new WorkflowScheduler(context, COORDINATOR) workflowScheduler.updateSchedule(workflow.physicalPlan) verifyBatchSizeInPartitioning(workflowScheduler, 500) @@ -200,7 +200,7 @@ class BatchSizePropagationSpec context ) - val workflowScheduler = new WorkflowScheduler(context, CONTROLLER) + val workflowScheduler = new WorkflowScheduler(context, COORDINATOR) workflowScheduler.updateSchedule(workflow.physicalPlan) verifyBatchSizeInPartitioning(workflowScheduler, 100) @@ -241,7 +241,7 @@ class BatchSizePropagationSpec context ) - val workflowScheduler = new WorkflowScheduler(context, CONTROLLER) + val workflowScheduler = new WorkflowScheduler(context, COORDINATOR) workflowScheduler.updateSchedule(workflow.physicalPlan) verifyBatchSizeInPartitioning(workflowScheduler, 300) @@ -282,7 +282,7 @@ class BatchSizePropagationSpec context ) - val workflowScheduler = new WorkflowScheduler(context, CONTROLLER) + val workflowScheduler = new WorkflowScheduler(context, COORDINATOR) workflowScheduler.updateSchedule(workflow.physicalPlan) verifyBatchSizeInPartitioning(workflowScheduler, 1) diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/DataProcessingSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/DataProcessingSpec.scala index 6dde4eff9f1..6e0eb6e2305 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/DataProcessingSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/DataProcessingSpec.scala @@ -31,7 +31,7 @@ import org.apache.texera.amber.core.workflow.{ WorkflowContext, WorkflowSettings } -import org.apache.texera.amber.engine.architecture.controller._ +import org.apache.texera.amber.engine.architecture.coordinator._ import org.apache.texera.amber.engine.common.AmberRuntime import org.apache.texera.amber.engine.e2e.TestUtils.{ buildWorkflow, diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/PauseSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/PauseSpec.scala index 036caefa0a3..81438525510 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/PauseSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/PauseSpec.scala @@ -26,8 +26,8 @@ import com.twitter.util.{Await, Duration, Promise} import com.typesafe.scalalogging.Logger import org.apache.texera.amber.clustering.SingleNodeListener import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerConfig, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorConfig, ExecutionStateUpdate } import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmptyRequest @@ -103,7 +103,7 @@ class PauseSpec system, workflow.context, workflow.physicalPlan, - ControllerConfig.default, + CoordinatorConfig.default, error => {} ) val completion = Promise[Unit]() @@ -114,15 +114,15 @@ class PauseSpec } }) val stateWaitTimeout = Duration.fromSeconds(10) - Await.result(client.controllerInterface.startWorkflow(EmptyRequest(), ())) + Await.result(client.coordinatorInterface.startWorkflow(EmptyRequest(), ())) val firstPaused = stateReached(client, PAUSED) - Await.result(client.controllerInterface.pauseWorkflow(EmptyRequest(), ())) + Await.result(client.coordinatorInterface.pauseWorkflow(EmptyRequest(), ())) Await.result(firstPaused, stateWaitTimeout) - Await.result(client.controllerInterface.resumeWorkflow(EmptyRequest(), ())) + Await.result(client.coordinatorInterface.resumeWorkflow(EmptyRequest(), ())) val secondPaused = stateReached(client, PAUSED) - Await.result(client.controllerInterface.pauseWorkflow(EmptyRequest(), ())) + Await.result(client.coordinatorInterface.pauseWorkflow(EmptyRequest(), ())) Await.result(secondPaused, stateWaitTimeout) - Await.result(client.controllerInterface.resumeWorkflow(EmptyRequest(), ())) + Await.result(client.coordinatorInterface.resumeWorkflow(EmptyRequest(), ())) Await.result(completion, Duration.fromMinutes(1)) } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala index d001c4471c1..be40d332607 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala @@ -32,8 +32,8 @@ import org.apache.texera.amber.core.virtualidentity.{ WorkflowIdentity } import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext, WorkflowSettings} -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerConfig, +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorConfig, ExecutionStateUpdate, FatalError, Workflow @@ -157,7 +157,7 @@ object TestUtils { system, workflow.context, workflow.physicalPlan, - ControllerConfig.default, + CoordinatorConfig.default, e => completion.updateIfEmpty(Throw(e)) ) try { @@ -169,7 +169,7 @@ object TestUtils { ) } }) - Await.result(client.controllerInterface.startWorkflow(EmptyRequest(), ())) + Await.result(client.coordinatorInterface.startWorkflow(EmptyRequest(), ())) Await.result(completion, completionTimeout) } finally { client.shutdown() @@ -298,7 +298,7 @@ object TestUtils { system, workflow.context, workflow.physicalPlan, - ControllerConfig.default, + CoordinatorConfig.default, error => {} ) // Timeout for control-command acks (start/pause/reconfigure/resume). @@ -312,12 +312,12 @@ object TestUtils { } }) Await.result( - client.controllerInterface.startWorkflow(EmptyRequest(), ()), + client.coordinatorInterface.startWorkflow(EmptyRequest(), ()), commandTimeout ) val pausedReached = stateReached(client, PAUSED) Await.result( - client.controllerInterface.pauseWorkflow(EmptyRequest(), ()), + client.coordinatorInterface.pauseWorkflow(EmptyRequest(), ()), commandTimeout ) Await.result(pausedReached, commandTimeout) @@ -325,7 +325,7 @@ object TestUtils { workflow.physicalPlan.getPhysicalOpsOfLogicalOp(op.operatorIdentifier) ) Await.result( - client.controllerInterface.reconfigureWorkflow( + client.coordinatorInterface.reconfigureWorkflow( WorkflowReconfigureRequest( reconfiguration = physicalOps.map(op => UpdateExecutorRequest(op.id, newOpExecInitInfo)), reconfigurationId = "test-reconfigure-1" @@ -335,7 +335,7 @@ object TestUtils { commandTimeout ) Await.result( - client.controllerInterface.resumeWorkflow(EmptyRequest(), ()), + client.coordinatorInterface.resumeWorkflow(EmptyRequest(), ()), commandTimeout ) Await.result(completion, Duration.fromMinutes(1)) diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/CheckpointSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/CheckpointSpec.scala index 3d207fd23b3..145b1b0bc8b 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/CheckpointSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/CheckpointSpec.scala @@ -22,14 +22,14 @@ package org.apache.texera.amber.engine.faulttolerance import org.apache.pekko.actor.{ActorSystem, Props} import org.apache.texera.amber.clustering.SingleNodeListener import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} -import org.apache.texera.amber.engine.architecture.controller.{ - ControllerConfig, - ControllerProcessor +import org.apache.texera.amber.engine.architecture.coordinator.{ + CoordinatorConfig, + CoordinatorProcessor } import org.apache.texera.amber.engine.architecture.worker.DataProcessor import org.apache.texera.amber.engine.architecture.worker.WorkflowWorker.DPInputQueueElement import org.apache.texera.amber.engine.common.SerializedState.{CP_STATE_KEY, DP_STATE_KEY} -import org.apache.texera.amber.engine.common.virtualidentity.util.{CONTROLLER, SELF} +import org.apache.texera.amber.engine.common.virtualidentity.util.{COORDINATOR, SELF} import org.apache.texera.amber.engine.common.{AmberRuntime, CheckpointState} import org.apache.texera.amber.engine.e2e.TestUtils.buildWorkflow import org.apache.texera.amber.operator.TestOperators @@ -63,17 +63,17 @@ class CheckpointSpec extends AnyFlatSpecLike with BeforeAndAfterAll { system.actorOf(Props[SingleNodeListener](), "cluster-info") } - "Default controller state" should "round-trip through CheckpointState" in { + "Default coordinator state" should "round-trip through CheckpointState" in { val cp = - new ControllerProcessor( + new CoordinatorProcessor( workflow.context, - ControllerConfig.default, - CONTROLLER, + CoordinatorConfig.default, + COORDINATOR, msg => {} ) val chkpt = new CheckpointState() chkpt.save(CP_STATE_KEY, cp) - val restored: ControllerProcessor = chkpt.load(CP_STATE_KEY) + val restored: CoordinatorProcessor = chkpt.load(CP_STATE_KEY) assert(restored.actorId == cp.actorId) } @@ -131,16 +131,16 @@ class CheckpointSpec extends AnyFlatSpecLike with BeforeAndAfterAll { // workflow.context, // workflow.physicalPlan, // resultStorage, -// ControllerConfig.default, +// CoordinatorConfig.default, // error => {} // ) -// Await.result(client1.controllerInterface.startWorkflow(EmptyRequest(), ())) +// Await.result(client1.coordinatorInterface.startWorkflow(EmptyRequest(), ())) // Thread.sleep(100) -// Await.result(client1.controllerInterface.pauseWorkflow(EmptyRequest(), ())) +// Await.result(client1.coordinatorInterface.pauseWorkflow(EmptyRequest(), ())) // val checkpointId = EmbeddedControlMessageIdentity(s"Checkpoint_test_1") // val uri = new URI("ram:///recovery-logs/tmp/") // Await.result( -// client1.controllerInterface.takeGlobalCheckpoint( +// client1.coordinatorInterface.takeGlobalCheckpoint( // TakeGlobalCheckpointRequest(estimationOnly = false, checkpointId, uri.toString), // () // ), @@ -148,16 +148,16 @@ class CheckpointSpec extends AnyFlatSpecLike with BeforeAndAfterAll { // ) // client1.shutdown() // Thread.sleep(100) -// var controllerConfig = ControllerConfig.default -// controllerConfig = -// controllerConfig.copy(stateRestoreConfOpt = Some(StateRestoreConfig(uri, checkpointId))) +// var coordinatorConfig = CoordinatorConfig.default +// coordinatorConfig = +// coordinatorConfig.copy(stateRestoreConfOpt = Some(StateRestoreConfig(uri, checkpointId))) // val completableFuture = new CompletableFuture[Unit]() // val client2 = new AmberClient( // system, // workflow.context, // workflow.physicalPlan, // resultStorage, -// controllerConfig, +// coordinatorConfig, // error => {} // ) // client2.registerCallback[ExecutionStateUpdate] { evt => @@ -168,11 +168,11 @@ class CheckpointSpec extends AnyFlatSpecLike with BeforeAndAfterAll { // Thread.sleep(1000) // assert( // Await -// .result(client2.controllerInterface.startWorkflow(EmptyRequest(), ())) +// .result(client2.coordinatorInterface.startWorkflow(EmptyRequest(), ())) // .workflowState == PAUSED // ) // Thread.sleep(5000) -// Await.result(client2.controllerInterface.resumeWorkflow(EmptyRequest(), ())) +// Await.result(client2.coordinatorInterface.resumeWorkflow(EmptyRequest(), ())) // completableFuture.get(30000, TimeUnit.MILLISECONDS) // } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/LoggingSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/LoggingSpec.scala index 9a388119158..e090642f3c3 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/LoggingSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/LoggingSpec.scala @@ -35,7 +35,7 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, EmptyRequest } -import org.apache.texera.amber.engine.architecture.rpc.controllerservice.ControllerServiceGrpc.METHOD_WORKER_EXECUTION_COMPLETED +import org.apache.texera.amber.engine.architecture.rpc.coordinatorservice.CoordinatorServiceGrpc.METHOD_WORKER_EXECUTION_COMPLETED import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceGrpc.{ METHOD_ADD_PARTITIONING, METHOD_PAUSE_WORKER, @@ -51,7 +51,7 @@ import org.apache.texera.amber.engine.common.ambermessage.{ } import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient.ControlInvocation import org.apache.texera.amber.engine.common.storage.SequentialRecordStorage -import org.apache.texera.amber.engine.common.virtualidentity.util.{CONTROLLER, SELF} +import org.apache.texera.amber.engine.common.virtualidentity.util.{COORDINATOR, SELF} import org.scalatest.BeforeAndAfterAll import org.scalatest.concurrent.TimeLimitedTests import org.scalatest.flatspec.AnyFlatSpecLike @@ -80,25 +80,25 @@ class LoggingSpec ControlInvocation( METHOD_START_WORKER, EmptyRequest(), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 0 ), ControlInvocation( METHOD_ADD_PARTITIONING, AddPartitioningRequest(mockLink, mockPolicy), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 0 ), ControlInvocation( METHOD_PAUSE_WORKER, EmptyRequest(), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 0 ), ControlInvocation( METHOD_RESUME_WORKER, EmptyRequest(), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 0 ), DataFrame( @@ -116,13 +116,13 @@ class LoggingSpec ControlInvocation( METHOD_START_WORKER, EmptyRequest(), - AsyncRPCContext(CONTROLLER, identifier1), + AsyncRPCContext(COORDINATOR, identifier1), 0 ), ControlInvocation( METHOD_WORKER_EXECUTION_COMPLETED, EmptyRequest(), - AsyncRPCContext(identifier1, CONTROLLER), + AsyncRPCContext(identifier1, COORDINATOR), 0 ) ) @@ -135,7 +135,7 @@ class LoggingSpec logStorage.deleteStorage() val logManager = ReplayLogManager.createLogManager(logStorage, "tmpLog", x => {}) payloadToLog.foreach { payload => - val channel = ChannelIdentity(CONTROLLER, SELF, isControl = true) + val channel = ChannelIdentity(COORDINATOR, SELF, isControl = true) val msgOpt = Some(WorkflowFIFOMessage(channel, 0, payload)) logManager.withFaultTolerant(channel, msgOpt) { // do nothing diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/ReplaySpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/ReplaySpec.scala index 57c97d4d444..3fa198fb058 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/ReplaySpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/faulttolerance/ReplaySpec.scala @@ -38,7 +38,7 @@ import org.apache.texera.amber.engine.common.ambermessage.WorkflowFIFOMessage import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient.ControlInvocation import org.apache.texera.amber.engine.common.storage.SequentialRecordStorage import org.apache.texera.amber.engine.common.storage.SequentialRecordStorage.SequentialRecordReader -import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR import org.scalatest.BeforeAndAfterAll import org.scalatest.flatspec.AnyFlatSpecLike @@ -71,7 +71,7 @@ class ReplaySpec private val actorId = ActorVirtualIdentity("test") private val actorId2 = ActorVirtualIdentity("upstream1") private val actorId3 = ActorVirtualIdentity("upstream2") - private val channelId1 = ChannelIdentity(CONTROLLER, actorId, isControl = true) + private val channelId1 = ChannelIdentity(COORDINATOR, actorId, isControl = true) private val channelId2 = ChannelIdentity(actorId2, actorId, isControl = false) private val channelId3 = ChannelIdentity(actorId3, actorId, isControl = false) private val channelId4 = ChannelIdentity(actorId2, actorId, isControl = true) @@ -97,7 +97,7 @@ class ReplaySpec ControlInvocation( METHOD_START_WORKER, EmptyRequest(), - AsyncRPCContext(CONTROLLER, actorId), + AsyncRPCContext(COORDINATOR, actorId), 0 ) ) diff --git a/amber/src/test/scala/org/apache/texera/amber/error/ErrorUtilsSpec.scala b/amber/src/test/scala/org/apache/texera/amber/error/ErrorUtilsSpec.scala index 7390071e329..90952d06774 100644 --- a/amber/src/test/scala/org/apache/texera/amber/error/ErrorUtilsSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/error/ErrorUtilsSpec.scala @@ -199,9 +199,9 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers { } it should "fall back to the dummy operator id for actor names that do not match the pattern" in { - val actor = ActorVirtualIdentity("CONTROLLER") + val actor = ActorVirtualIdentity("COORDINATOR") val (operatorId, workerId) = ErrorUtils.getOperatorFromActorIdOpt(Some(actor)) operatorId shouldBe "__DummyOperator" - workerId shouldBe "CONTROLLER" + workerId shouldBe "COORDINATOR" } } diff --git a/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala b/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala index 99adb6bee9f..c460cfdb63d 100644 --- a/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala @@ -49,7 +49,7 @@ import scala.collection.mutable * when `executeWorkflow` fails -- surfaces as a `WorkflowErrorEvent` through the * normal websocket-event observable. * - * The unused `controllerConfig` / `resultService` are passed as `null` on + * The unused `coordinatorConfig` / `resultService` are passed as `null` on * purpose: construction must stay side-effect-free (all throwing work is in * `executeWorkflow`), so a future change that dereferences them during * construction would fail here. diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/LocationPreference.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/LocationPreference.scala index a8308d3f3fd..df05fbd3c7c 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/LocationPreference.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/LocationPreference.scala @@ -22,9 +22,9 @@ package org.apache.texera.amber.core.workflow // LocationPreference defines where operators should run. sealed trait LocationPreference extends Serializable -// PreferController: Run on the controller node. +// PreferCoordinator: Run on the coordinator node. // Example: For scan operators reading files. -object PreferController extends LocationPreference +object PreferCoordinator extends LocationPreference // RoundRobinPreference: Distribute across worker nodes, per operator. // Example: diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/PhysicalOp.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/PhysicalOp.scala index 1fdda49a6af..9e55fc503a7 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/PhysicalOp.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/PhysicalOp.scala @@ -51,7 +51,7 @@ class SchemaNotAvailableException(message: String) extends Exception(message) object PhysicalOp { /** all source operators should use sourcePhysicalOp to give the following configs: - * 1) it initializes at the controller jvm. + * 1) it initializes at the coordinator jvm. * 2) it only has 1 worker actor. * 3) it has no input ports. */ @@ -80,7 +80,7 @@ object PhysicalOp { executionId, opExecInitInfo, parallelizable = false, - locationPreference = Some(PreferController) + locationPreference = Some(PreferCoordinator) ) def oneToOnePhysicalOp( @@ -154,7 +154,7 @@ object PhysicalOp { opExecInitInfo: OpExecInitInfo ): PhysicalOp = { manyToOnePhysicalOp(physicalOpId, workflowId, executionId, opExecInitInfo) - .withLocationPreference(Some(PreferController)) + .withLocationPreference(Some(PreferCoordinator)) } } diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/VirtualIdentityUtils.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/VirtualIdentityUtils.scala index f91cca6af3a..b4dc09a8a68 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/VirtualIdentityUtils.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/VirtualIdentityUtils.scala @@ -66,7 +66,7 @@ object VirtualIdentityUtils { case workerNamePattern(_, operator, layerName, _) => PhysicalOpIdentity(OperatorIdentity(operator), layerName) case other => - // for special actorId such as SELF, CONTROLLER + // for special actorId such as SELF, COORDINATOR PhysicalOpIdentity(OperatorIdentity("__DummyOperator"), "__DummyLayer") } } @@ -96,7 +96,7 @@ object VirtualIdentityUtils { case workerNamePattern(_, _, _, idx) => Some(idx.toInt) case _ => - // for special actorId such as SELF, CONTROLLER + // for special actorId such as SELF, COORDINATOR None } } diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/PhysicalOpSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/PhysicalOpSpec.scala index 35eb0bc3388..1e8d247e452 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/PhysicalOpSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/PhysicalOpSpec.scala @@ -56,7 +56,7 @@ class PhysicalOpSpec extends AnyFlatSpec { // ----- factory methods ----- - "PhysicalOp factories" should "build source ops preferring the controller" in { + "PhysicalOp factories" should "build source ops preferring the coordinator" in { val op = PhysicalOp.sourcePhysicalOp( workflowId, executionId, @@ -65,11 +65,11 @@ class PhysicalOpSpec extends AnyFlatSpec { ) assert(op.id == opId("src")) assert(!op.parallelizable) - assert(op.locationPreference.contains(PreferController)) + assert(op.locationPreference.contains(PreferCoordinator)) assert(op.isSourceOperator) val op2 = PhysicalOp.sourcePhysicalOp(opId("s2"), workflowId, executionId, OpExecInitInfo.Empty) assert(op2.id == opId("s2")) - assert(op2.locationPreference.contains(PreferController)) + assert(op2.locationPreference.contains(PreferCoordinator)) } it should "build one-to-one ops as parallelizable with no location preference" in { @@ -99,7 +99,7 @@ class PhysicalOpSpec extends AnyFlatSpec { assert(op2.partitionRequirement == List(Some(SinglePartition()))) } - it should "build local ops on the controller requiring a single partition" in { + it should "build local ops on the coordinator requiring a single partition" in { val op = PhysicalOp.localPhysicalOp( workflowId, executionId, @@ -108,9 +108,9 @@ class PhysicalOpSpec extends AnyFlatSpec { ) assert(!op.parallelizable) assert(op.partitionRequirement == List(Some(SinglePartition()))) - assert(op.locationPreference.contains(PreferController)) + assert(op.locationPreference.contains(PreferCoordinator)) val op2 = PhysicalOp.localPhysicalOp(opId("l2"), workflowId, executionId, OpExecInitInfo.Empty) - assert(op2.locationPreference.contains(PreferController)) + assert(op2.locationPreference.contains(PreferCoordinator)) } // ----- dependee inputs ----- diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowCoreTypesSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowCoreTypesSpec.scala index 7c876d26f64..897f381c0a8 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowCoreTypesSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowCoreTypesSpec.scala @@ -35,16 +35,16 @@ class WorkflowCoreTypesSpec extends AnyFlatSpec { // LocationPreference // --------------------------------------------------------------------------- - "LocationPreference" should "have PreferController and RoundRobinPreference as singleton subtypes" in { - val a: LocationPreference = PreferController + "LocationPreference" should "have PreferCoordinator and RoundRobinPreference as singleton subtypes" in { + val a: LocationPreference = PreferCoordinator val b: LocationPreference = RoundRobinPreference - assert(a eq PreferController) + assert(a eq PreferCoordinator) assert(b eq RoundRobinPreference) assert(a != b) } it should "be Serializable on every subtype" in { - val all: Seq[LocationPreference] = Seq(PreferController, RoundRobinPreference) + val all: Seq[LocationPreference] = Seq(PreferCoordinator, RoundRobinPreference) all.foreach(p => assert(p.isInstanceOf[Serializable])) } @@ -131,8 +131,8 @@ class WorkflowCoreTypesSpec extends AnyFlatSpec { } "PhysicalOp.withLocationPreference" should "store the location preference" in { - val op = newPhysicalOp("a").withLocationPreference(Some(PreferController)) - assert(op.locationPreference.contains(PreferController)) + val op = newPhysicalOp("a").withLocationPreference(Some(PreferCoordinator)) + assert(op.locationPreference.contains(PreferCoordinator)) } "PhysicalOp.withParallelizable" should "set the parallelizable flag and round-trip through copy" in { diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/util/VirtualIdentityUtilsSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/VirtualIdentityUtilsSpec.scala index 58e58f68e4b..a72471df6e4 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/util/VirtualIdentityUtilsSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/VirtualIdentityUtilsSpec.scala @@ -62,8 +62,8 @@ class VirtualIdentityUtilsSpec extends AnyFlatSpec with Matchers { } it should "fall back to __DummyOperator/__DummyLayer for non-worker actor names" in { - val controller = ActorVirtualIdentity("CONTROLLER") - val opId = VirtualIdentityUtils.getPhysicalOpId(controller) + val coordinator = ActorVirtualIdentity("COORDINATOR") + val opId = VirtualIdentityUtils.getPhysicalOpId(coordinator) opId.logicalOpId.id shouldBe "__DummyOperator" opId.layerName shouldBe "__DummyLayer" } @@ -116,8 +116,8 @@ class VirtualIdentityUtilsSpec extends AnyFlatSpec with Matchers { // helper preserves the existing __DummyOperator sentinel so it // stays a drop-in replacement for the inline pattern at call sites // (see VirtualIdentityUtils.getLogicalOpId docstring). - val controller = ActorVirtualIdentity("CONTROLLER") - VirtualIdentityUtils.getLogicalOpId(controller) shouldBe "__DummyOperator" + val coordinator = ActorVirtualIdentity("COORDINATOR") + VirtualIdentityUtils.getLogicalOpId(coordinator) shouldBe "__DummyOperator" val self = ActorVirtualIdentity("SELF") VirtualIdentityUtils.getLogicalOpId(self) shouldBe "__DummyOperator" } @@ -130,14 +130,14 @@ class VirtualIdentityUtilsSpec extends AnyFlatSpec with Matchers { } it should "return None for non-worker actor names" in { - // Special ActorVirtualIdentity values like CONTROLLER or SELF do not + // Special ActorVirtualIdentity values like COORDINATOR or SELF do not // match workerNamePattern. getWorkerIndex returns None rather than // throwing scala.MatchError, mirroring the graceful handling in // getPhysicalOpId and toShorterString. Returning Option forces each // caller to explicitly acknowledge the non-worker case rather than // silently propagating a sentinel value. - val controller = ActorVirtualIdentity("CONTROLLER") - VirtualIdentityUtils.getWorkerIndex(controller) shouldBe None + val coordinator = ActorVirtualIdentity("COORDINATOR") + VirtualIdentityUtils.getWorkerIndex(coordinator) shouldBe None val self = ActorVirtualIdentity("SELF") VirtualIdentityUtils.getWorkerIndex(self) shouldBe None } @@ -177,8 +177,8 @@ class VirtualIdentityUtilsSpec extends AnyFlatSpec with Matchers { } it should "return the actor name unchanged when it does not match the worker pattern" in { - val controller = ActorVirtualIdentity("CONTROLLER") - VirtualIdentityUtils.toShorterString(controller) shouldBe "CONTROLLER" + val coordinator = ActorVirtualIdentity("COORDINATOR") + VirtualIdentityUtils.toShorterString(coordinator) shouldBe "COORDINATOR" } // ----- getFromActorIdForInputPortStorage ----- diff --git a/common/workflow-core/src/test/scala/org/apache/texera/service/util/LargeBinaryManagerSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/service/util/LargeBinaryManagerSpec.scala index a0436e682e2..0f2cce8b301 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/service/util/LargeBinaryManagerSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/service/util/LargeBinaryManagerSpec.scala @@ -28,7 +28,7 @@ class LargeBinaryManagerSpec extends AnyFunSuite with S3StorageTestBase with Bef /** Execution id used by the bulk of the tests. */ private val TestExecutionId: Long = 9999L - /** Seeds the thread's base URI for an execution, as the controller does in production. */ + /** Seeds the thread's base URI for an execution, as the coordinator does in production. */ private def setExecutionContext(eid: Long): Unit = LargeBinaryManager.setCurrentBaseUri(LargeBinaryManager.baseUriForExecution(eid)) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SpecialPhysicalOpFactorySpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SpecialPhysicalOpFactorySpec.scala index d676dfc30b1..e5ceb379672 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SpecialPhysicalOpFactorySpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SpecialPhysicalOpFactorySpec.scala @@ -27,7 +27,7 @@ import org.apache.texera.amber.core.virtualidentity.{ PhysicalOpIdentity, WorkflowIdentity } -import org.apache.texera.amber.core.workflow.{GlobalPortIdentity, PortIdentity, PreferController} +import org.apache.texera.amber.core.workflow.{GlobalPortIdentity, PortIdentity, PreferCoordinator} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -67,7 +67,7 @@ class SpecialPhysicalOpFactorySpec extends AnyFlatSpec with Matchers { op.isSourceOperator shouldBe true op.inputPorts shouldBe empty op.outputPorts.keySet shouldBe Set(PortIdentity(0)) - op.locationPreference shouldBe Some(PreferController) + op.locationPreference shouldBe Some(PreferCoordinator) op.parallelizable shouldBe false op.outputPorts(PortIdentity(0))._3 shouldBe Right(schema) }