From baf302b63d84059d472004a500d162f4d10fc46d Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sun, 16 Mar 2025 19:47:45 +0000 Subject: [PATCH 01/32] Start event-driven example --- examples/tutorials/005_events/.gitignore | 1 + examples/tutorials/005_events/hello_events.py | 146 ++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 examples/tutorials/005_events/.gitignore create mode 100644 examples/tutorials/005_events/hello_events.py diff --git a/examples/tutorials/005_events/.gitignore b/examples/tutorials/005_events/.gitignore new file mode 100644 index 00000000..16f2dc5f --- /dev/null +++ b/examples/tutorials/005_events/.gitignore @@ -0,0 +1 @@ +*.csv \ No newline at end of file diff --git a/examples/tutorials/005_events/hello_events.py b/examples/tutorials/005_events/hello_events.py new file mode 100644 index 00000000..629131b6 --- /dev/null +++ b/examples/tutorials/005_events/hello_events.py @@ -0,0 +1,146 @@ +"""Event-based model example.""" + +import asyncio +import random +import typing as _t + +from pydantic import BaseModel + +from plugboard.component import Component, IOController +from plugboard.connector import AsyncioConnector, ConnectorBuilder +from plugboard.events import Event, EventConnectorBuilder, StopEvent +from plugboard.library import FileWriter +from plugboard.process import LocalProcess +from plugboard.schemas import ConnectorSpec, ComponentArgsDict + + +class ExtremeValue(BaseModel): + """Data for event_A.""" + + value: float + extreme_type: _t.Literal["high", "low"] + + +class HighEvent(Event): + """High value event type.""" + + type: _t.ClassVar[str] = "high_event" + data: ExtremeValue + + +class LowEvent(Event): + """Low value event type.""" + + type: _t.ClassVar[str] = "low_event" + data: ExtremeValue + + +class Random(Component): + """Generates random numbers.""" + + io = IOController(outputs=["value"]) + + def __init__(self, iters: int = 50, **kwargs: _t.Unpack[ComponentArgsDict]) -> None: + super().__init__(**kwargs) + self.max_iters = iters + self.completed_iters = 0 + + async def step(self) -> None: + self.completed_iters += 1 + self.value = random.random() + if self.completed_iters >= self.max_iters: + self.io.queue_event(StopEvent(source=self.name, data={})) + + +class FindHighLowValues(Component): + """Raises an event on high or low values.""" + + io = IOController(inputs=["value"], output_events=[LowEvent, HighEvent]) + + def __init__( + self, + low_limit: float = 0.2, + high_limit: float = 0.8, + **kwargs: _t.Unpack[ComponentArgsDict], + ) -> None: + super().__init__(**kwargs) + self.low_limit = low_limit + self.high_limit = high_limit + + async def step(self) -> None: + if self.value >= self.high_limit: + self.io.queue_event( + HighEvent( + source=self.name, data=ExtremeValue(value=self.value, extreme_type="high") + ) + ) + if self.value <= self.low_limit: + self.io.queue_event( + LowEvent(source=self.name, data=ExtremeValue(value=self.value, extreme_type="low")) + ) + + +class CollectHigh(Component): + """Collects values from high events.""" + + io = IOController(input_events=[HighEvent], outputs=["value"]) + + def __init__(self, **kwargs: _t.Unpack[ComponentArgsDict]) -> None: + super().__init__(**kwargs) + self.latest_event: _t.Optional[ExtremeValue] = None + + async def step(self) -> None: + self.value = self.latest_event.value if self.latest_event else None + + @HighEvent.handler + async def handle_event(self, event: HighEvent) -> None: + self.latest_event = event.data + + +class CollectLow(Component): + """Collects values from low events.""" + + io = IOController(input_events=[LowEvent], outputs=["value"]) + + def __init__(self, **kwargs: _t.Unpack[ComponentArgsDict]) -> None: + super().__init__(**kwargs) + self.latest_event: _t.Optional[ExtremeValue] = None + + async def step(self) -> None: + self.value = self.latest_event.value if self.latest_event else float("nan") + + @LowEvent.handler + async def handle_event(self, event: LowEvent) -> None: + self.latest_event = event.data + + +async def main() -> None: + components = [ + Random(name="random-generator"), + FindHighLowValues(name="find-high-low", low_limit=0.2, high_limit=0.8), + CollectHigh(name="collect-high"), + CollectLow(name="collect-low"), + FileWriter(name="save-high", path="high.csv", field_names=["value"]), + FileWriter(name="save-low", path="low.csv", field_names=["value"]), + ] + connect = lambda in_, out_: AsyncioConnector(spec=ConnectorSpec(source=in_, target=out_)) + connectors = [ + connect("random-generator.value", "find-high-low.value"), + connect("collect-high.value", "save-high.value"), + connect("collect-low.value", "save-low.value"), + ] + connector_builder = ConnectorBuilder(connector_cls=AsyncioConnector) + event_connector_builder = EventConnectorBuilder(connector_builder=connector_builder) + event_connectors = list(event_connector_builder.build(components).values()) + + process = LocalProcess( + components=components, + connectors=connectors + event_connectors, + ) + + async with process: + await process.run() + + +if __name__ == "__main__": + asyncio.run(main()) From b29dd82783c5ccf7c9b9e52a9b6307040050c69f Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sun, 16 Mar 2025 20:08:18 +0000 Subject: [PATCH 02/32] None --- examples/tutorials/005_events/hello_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tutorials/005_events/hello_events.py b/examples/tutorials/005_events/hello_events.py index 629131b6..282cb496 100644 --- a/examples/tutorials/005_events/hello_events.py +++ b/examples/tutorials/005_events/hello_events.py @@ -107,7 +107,7 @@ def __init__(self, **kwargs: _t.Unpack[ComponentArgsDict]) -> None: self.latest_event: _t.Optional[ExtremeValue] = None async def step(self) -> None: - self.value = self.latest_event.value if self.latest_event else float("nan") + self.value = self.latest_event.value if self.latest_event else None @LowEvent.handler async def handle_event(self, event: LowEvent) -> None: From bdd4675e93b02708d8b00fc8fbcf938aab03be0f Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sun, 16 Mar 2025 20:47:39 +0000 Subject: [PATCH 03/32] WIP diagram for event-driven example --- docs/examples/tutorials/event-driven-models.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/examples/tutorials/event-driven-models.md b/docs/examples/tutorials/event-driven-models.md index 4047a74c..81ea5855 100644 --- a/docs/examples/tutorials/event-driven-models.md +++ b/docs/examples/tutorials/event-driven-models.md @@ -1 +1,12 @@ Tutorial coming soon. + +```mermaid +graph LR; + Random(random-generator)-->FindHighLowValues(find-high-low); + FindHighLowValues(find-high-low)-.->HighEvent{{high-event}}; + FindHighLowValues(find-high-low)-.->LowEvent{{low-event}}; + HighEvent{{high-event}}-.->CollectHigh(collect-high); + LowEvent{{low-event}}-.->CollectLow(collect-low); + CollectHigh(collect-high)-->SaveHigh(save-high); + CollectLow(collect-low)-->SaveLow(save-low); +``` \ No newline at end of file From a03c94a1d8cb49edcc81642e81a829ea6e8090ef Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sun, 16 Mar 2025 21:21:04 +0000 Subject: [PATCH 04/32] Prepare snippets on code --- examples/tutorials/005_events/hello_events.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/examples/tutorials/005_events/hello_events.py b/examples/tutorials/005_events/hello_events.py index 282cb496..a92fbd49 100644 --- a/examples/tutorials/005_events/hello_events.py +++ b/examples/tutorials/005_events/hello_events.py @@ -1,5 +1,6 @@ """Event-based model example.""" +# fmt: off import asyncio import random import typing as _t @@ -14,6 +15,7 @@ from plugboard.schemas import ConnectorSpec, ComponentArgsDict +# --8<-- [start:events] class ExtremeValue(BaseModel): """Data for event_A.""" @@ -33,8 +35,10 @@ class LowEvent(Event): type: _t.ClassVar[str] = "low_event" data: ExtremeValue +# --8<-- [end:events] +# --8<-- [start:source-component] class Random(Component): """Generates random numbers.""" @@ -49,9 +53,11 @@ async def step(self) -> None: self.completed_iters += 1 self.value = random.random() if self.completed_iters >= self.max_iters: - self.io.queue_event(StopEvent(source=self.name, data={})) + self.io.queue_event(StopEvent(source=self.name, data={})) # (1)! +# --8<-- [end:source-component] +# --8<-- [start:event-publisher] class FindHighLowValues(Component): """Raises an event on high or low values.""" @@ -69,7 +75,7 @@ def __init__( async def step(self) -> None: if self.value >= self.high_limit: - self.io.queue_event( + self.io.queue_event( # (1)! HighEvent( source=self.name, data=ExtremeValue(value=self.value, extreme_type="high") ) @@ -78,8 +84,10 @@ async def step(self) -> None: self.io.queue_event( LowEvent(source=self.name, data=ExtremeValue(value=self.value, extreme_type="low")) ) +# --8<-- [end:event-publisher] +# --8<-- [start:event-consumers] class CollectHigh(Component): """Collects values from high events.""" @@ -92,7 +100,7 @@ def __init__(self, **kwargs: _t.Unpack[ComponentArgsDict]) -> None: async def step(self) -> None: self.value = self.latest_event.value if self.latest_event else None - @HighEvent.handler + @HighEvent.handler # (1)! async def handle_event(self, event: HighEvent) -> None: self.latest_event = event.data @@ -112,9 +120,11 @@ async def step(self) -> None: @LowEvent.handler async def handle_event(self, event: LowEvent) -> None: self.latest_event = event.data +# --8<-- [end:event-consumers] async def main() -> None: + # --8<-- [start:main] components = [ Random(name="random-generator"), FindHighLowValues(name="find-high-low", low_limit=0.2, high_limit=0.8), @@ -124,12 +134,12 @@ async def main() -> None: FileWriter(name="save-low", path="low.csv", field_names=["value"]), ] connect = lambda in_, out_: AsyncioConnector(spec=ConnectorSpec(source=in_, target=out_)) - connectors = [ + connectors = [ # (1)! connect("random-generator.value", "find-high-low.value"), connect("collect-high.value", "save-high.value"), connect("collect-low.value", "save-low.value"), ] - connector_builder = ConnectorBuilder(connector_cls=AsyncioConnector) + connector_builder = ConnectorBuilder(connector_cls=AsyncioConnector) # (2)! event_connector_builder = EventConnectorBuilder(connector_builder=connector_builder) event_connectors = list(event_connector_builder.build(components).values()) @@ -140,6 +150,7 @@ async def main() -> None: async with process: await process.run() + # --8<-- [end:main] if __name__ == "__main__": From d5c218fa6c88622313b9d45f3aab2ce2a24d739c Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sun, 16 Mar 2025 21:22:35 +0000 Subject: [PATCH 05/32] Fix typo in docstring --- examples/tutorials/003_more_components/hello_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tutorials/003_more_components/hello_llm.py b/examples/tutorials/003_more_components/hello_llm.py index 8a59204b..10c41292 100644 --- a/examples/tutorials/003_more_components/hello_llm.py +++ b/examples/tutorials/003_more_components/hello_llm.py @@ -1,4 +1,4 @@ -"""Simple hello world example.""" +"""Simple LLM example.""" # fmt: off import asyncio From c43dba911821f3edff49da790e6fd71426d9108b Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 19:22:47 +0000 Subject: [PATCH 06/32] Writeup event-driven tutorial --- .../examples/tutorials/event-driven-models.md | 68 ++++++++++++++++++- examples/tutorials/005_events/hello_events.py | 12 ++-- 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/docs/examples/tutorials/event-driven-models.md b/docs/examples/tutorials/event-driven-models.md index 81ea5855..f66491cd 100644 --- a/docs/examples/tutorials/event-driven-models.md +++ b/docs/examples/tutorials/event-driven-models.md @@ -1,4 +1,10 @@ -Tutorial coming soon. +So far everything we have built in Plugboard has been a **discrete-time model**. This means that the whole model advances step-wise, i.e. `step` gets called on each [`Component`][plugboard.component.Component], calculating all of their outputs before advancing the simulation on. + +In this tutorial we're going to introduce an **event-driven model**, where data can be passed around between components based on triggers that you can define. Event-based models can be useful in a variety of scenarios, for example when modelling parts moving around a production line, or passengers arriving at a transport hub. + +## Event-based model + +Here's the model that we're going to build. Given a stream of random numbers, we'll trigger `HighEvent` whenever the value is above `0.8` and `LowEvent` whenever the value is below `0.2`. This allows us to funnel data into different parts of the model: in this case we'll just save the latest high/low values to a file at each step. In the diagram the _dotted lines_ represent the flow of event data: `FindHighLowValues` will publish events, while `CollectHigh` and `CollectLow` will subscribe to receive high and low events respectively. ```mermaid graph LR; @@ -9,4 +15,62 @@ graph LR; LowEvent{{low-event}}-.->CollectLow(collect-low); CollectHigh(collect-high)-->SaveHigh(save-high); CollectLow(collect-low)-->SaveLow(save-low); -``` \ No newline at end of file +``` + +## Defining events + +First we need to define the events that are going to get used in the model. Each event needs a name, in this case `"high_event"` and `"low_event"` and a `data` type associated with it. Use a [Pydantic](https://docs.pydantic.dev/latest/) model to define the format of this `data` field. + +```python +--8<-- "examples/tutorials/005_events/hello_events.py:events" +``` + +## Building components to create and consume events + +So far all of our process models have run step-by-step until completion. When a model contains event-driven components, we need a way to tell them to stop at the end of the simulation, otherwise they will stay running and listening for events forever. + +In this example, our `Random` component will drive the process by generating input random values. When it has completed `iters` iterations, we can use it to stop the model by sending a [`StopEvent`][plugboard.events.StopEvent], causing other event-driven components in the model to shutdown. + +```python +--8<-- "examples/tutorials/005_events/hello_events.py:source-component" +``` + +1. Use `self.io.queue_event` to send an event from a [`Component`][plugboard.component.Component]. Here we are sending [`StopEvent`][plugboard.events.StopEvent] to stop the process once all of the random values have been generated. + +Next, we will define `FindHighLowValues` to identify high and low values in the stream of random numbers and publish `HighEvent` and `LowEvent` respectively. + +```python +--8<-- "examples/tutorials/005_events/hello_events.py:event-publisher" +``` + +1. See how we use the [`IOController`][plugboard.component.IOController] to declare that this [`Component`][plugboard.component.Component] will publish events. +2. Call `self.io.queue_event` to send an event of the correct type. + +Finally, we need components to subscribe to these events and process them. Use the `Event.handler` decorator to identify the method on each [`Component`][plugboard.component.Component] that will do this processing. + +```python +--8<-- "examples/tutorials/005_events/hello_events.py:event-consumers" +``` + +1. Specify the events that this [`Component`][plugboard.component.Component] will subscribe to. +2. Use this decorator to indicate that we handle `HighEvent` here... +3. ...and we handle `LowEvent` here. + +!!! note + In a real model you could define whatever logic you need inside your event handler, e.g. create a file, publish another event, etc. Here we just store the event on an attribute so that its value can be output on the next call to `step()`. + +## Putting it all together + +Now we can create a [`Process`][plugboard.process.Process] from all these components. The outputs from `CollectLow` and `CollectHigh` are connected to separate [`FileWriter`][plugboard.library.FileWriter] components so that we'll get a CSV file containing the latest high and low values at each step of the simulation. + +!!! info + We need a few extra lines of code to create connectors for the event-based parts of the model. If you define your process in YAML this will be done automatically for you, but if you are defining the process in code then you will need to use the [`EventConnectorBuilder`][plugboard.events.EventConnectorBuilder] to do this. + +```python hl_lines="15-17" +--8<-- "examples/tutorials/005_events/hello_events.py:main" +``` + +1. These connectors are for the normal, non-event driven parts of the model and connect [`Component`][plugboard.component.Component]` inputs and outputs. +2. These lines will set up connectors for the events in the model. + +Take a look at the `high.csv` and `low.csv` files: the first few rows will usually be empty, and then as soon as high or low values are identified they will start to appear in the CSVs. diff --git a/examples/tutorials/005_events/hello_events.py b/examples/tutorials/005_events/hello_events.py index a92fbd49..002a0f10 100644 --- a/examples/tutorials/005_events/hello_events.py +++ b/examples/tutorials/005_events/hello_events.py @@ -59,9 +59,9 @@ async def step(self) -> None: # --8<-- [start:event-publisher] class FindHighLowValues(Component): - """Raises an event on high or low values.""" + """Publishes an event on high or low values.""" - io = IOController(inputs=["value"], output_events=[LowEvent, HighEvent]) + io = IOController(inputs=["value"], output_events=[LowEvent, HighEvent]) # (1)! def __init__( self, @@ -75,7 +75,7 @@ def __init__( async def step(self) -> None: if self.value >= self.high_limit: - self.io.queue_event( # (1)! + self.io.queue_event( # (2)! HighEvent( source=self.name, data=ExtremeValue(value=self.value, extreme_type="high") ) @@ -91,7 +91,7 @@ async def step(self) -> None: class CollectHigh(Component): """Collects values from high events.""" - io = IOController(input_events=[HighEvent], outputs=["value"]) + io = IOController(input_events=[HighEvent], outputs=["value"]) # (1)! def __init__(self, **kwargs: _t.Unpack[ComponentArgsDict]) -> None: super().__init__(**kwargs) @@ -100,7 +100,7 @@ def __init__(self, **kwargs: _t.Unpack[ComponentArgsDict]) -> None: async def step(self) -> None: self.value = self.latest_event.value if self.latest_event else None - @HighEvent.handler # (1)! + @HighEvent.handler # (2)! async def handle_event(self, event: HighEvent) -> None: self.latest_event = event.data @@ -117,7 +117,7 @@ def __init__(self, **kwargs: _t.Unpack[ComponentArgsDict]) -> None: async def step(self) -> None: self.value = self.latest_event.value if self.latest_event else None - @LowEvent.handler + @LowEvent.handler # (3)! async def handle_event(self, event: LowEvent) -> None: self.latest_event = event.data # --8<-- [end:event-consumers] From be278a69e2a70eae5f01c8a170092eec78a1cc10 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 19:24:10 +0000 Subject: [PATCH 07/32] Fix broken links --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cdd3153c..8041a973 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ plugboard process run my-model.yaml ## 📖 Documentation -For more information including a detailed API reference and step-by-step usage examples, refer to the [documentation site](https://docs.plugboard.dev). We recommend diving into the [tutorials](https://docs.plugboard.dev/examples/tutorials/hello-world/) for a step-by-step to getting started. +For more information including a detailed API reference and step-by-step usage examples, refer to the [documentation site](https://docs.plugboard.dev). We recommend diving into the [tutorials](https://docs.plugboard.dev/latest/examples/tutorials/hello-world/) for a step-by-step to getting started. ## 🐾 Roadmap @@ -175,7 +175,7 @@ Plugboard is under active development, with new features in the works: ## 👋 Contributions -Contributions are welcomed and warmly received! For bug fixes and smaller feature requests feel free to open an issue on this repo. For any larger changes please get in touch with us to discuss first. More information for developers can be found in [the contributing section](https://docs.plugboard.dev/contributing/) of the docs. +Contributions are welcomed and warmly received! For bug fixes and smaller feature requests feel free to open an issue on this repo. For any larger changes please get in touch with us to discuss first. More information for developers can be found in [the contributing section](https://docs.plugboard.dev/latest/contributing/) of the docs. ## ⚖️ Licence From a51f9a62928c0f2867b6232d4f0f59236c0fe15d Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 19:27:33 +0000 Subject: [PATCH 08/32] Update for implemented feature --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8041a973..e4c375c4 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,8 @@ Some examples of what you can build with Plugboard include: - A **command line interface** for executing models; - Built to handle the **data intensive simulation** requirements of industrial process applications; - Modern implementation with **Python 3.12 and above** based around **asyncio** with complete type annotation coverage; -- Built-in integrations for **loading/saving data** from cloud storage and SQL databases. +- Built-in integrations for **loading/saving data** from cloud storage and SQL databases; +- **Detailed logging** of component inputs, outputs and state for monitoring and process mining or surrogate modelling use-cases. ## 🔌 Installation @@ -167,7 +168,6 @@ For more information including a detailed API reference and step-by-step usage e Plugboard is under active development, with new features in the works: -- Detailed logging of component inputs, outputs and state for monitoring and process mining or surrogate modelling use-cases. - Support for strongly typed data messages and validation based on pydantic. - Support for different parallelisation patterns such as: single-threaded with coroutines, single-host multi process, or distributed with Ray in Kubernetes. - Data exchange between components with popular messaging technologies like RabbitMQ and Google Pub/Sub. From c9a203611231da3603aa891e937660e5646c408b Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 19:39:14 +0000 Subject: [PATCH 09/32] Update links --- docs/usage/key-concepts.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage/key-concepts.md b/docs/usage/key-concepts.md index 3c953149..06980fdc 100644 --- a/docs/usage/key-concepts.md +++ b/docs/usage/key-concepts.md @@ -19,7 +19,7 @@ When implementing your own components, you will need to: * Specify its inputs and ouputs using an [`IOController`][plugboard.component.IOController]; * Define a `step()` method the executes the main logic of your component for a single step; and * Optionally define an `init()` method to do any required preparatory steps before the model in run. -* In the case of event based models, define custom `Event` subclasses and corresponding event handler methods decorated with `Event.handler`. +* In the case of event based models, define custom [`Event`][plugboard.events.Event] subclasses and corresponding event handler methods decorated with `Event.handler`. ### Connectors @@ -42,7 +42,7 @@ graph LR; A(Load data)-->D(Record output); ``` -For models with explicitly declared input and output fields, connectors for each input-output pair must be defined explicitly using one of the `Connector` implementations. Connectors required for any events used in the model will be created for you automatically. +For models with explicitly declared input and output fields, connectors for each input-output pair must be defined explicitly using one of the [`Connector`][plugboard.connector.Connector] implementations. Connectors required for any events used in the model will be created for you automatically. ### Processes From 258bd3345dcc167526ba0979b40b57462a332c68 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 19:56:45 +0000 Subject: [PATCH 10/32] Add YAML version --- .../examples/tutorials/event-driven-models.md | 2 +- examples/tutorials/005_events/model.yaml | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 examples/tutorials/005_events/model.yaml diff --git a/docs/examples/tutorials/event-driven-models.md b/docs/examples/tutorials/event-driven-models.md index f66491cd..99b95d6d 100644 --- a/docs/examples/tutorials/event-driven-models.md +++ b/docs/examples/tutorials/event-driven-models.md @@ -73,4 +73,4 @@ Now we can create a [`Process`][plugboard.process.Process] from all these compon 1. These connectors are for the normal, non-event driven parts of the model and connect [`Component`][plugboard.component.Component]` inputs and outputs. 2. These lines will set up connectors for the events in the model. -Take a look at the `high.csv` and `low.csv` files: the first few rows will usually be empty, and then as soon as high or low values are identified they will start to appear in the CSVs. +Take a look at the `high.csv` and `low.csv` files: the first few rows will usually be empty, and then as soon as high or low values are identified they will start to appear in the CSVs. As usual, you can run this model from the CLI using `plugboard process run model.yaml`. \ No newline at end of file diff --git a/examples/tutorials/005_events/model.yaml b/examples/tutorials/005_events/model.yaml new file mode 100644 index 00000000..c3f68cd2 --- /dev/null +++ b/examples/tutorials/005_events/model.yaml @@ -0,0 +1,38 @@ +plugboard: + process: + args: + components: + - type: hello_events.Random + args: + name: "random-generator" + iters: 50 + - type: hello_events.FindHighLowValues + args: + name: "find-high-low" + low_limit: 0.2 + high_limit: 0.8 + - type: hello_events.CollectLow + args: + name: "collect-low" + - type: hello_events.CollectHigh + args: + name: "collect-high" + - type: plugboard.library.file_io.FileWriter + args: + name: "save-low" + path: "low.csv" + field_names: + - value + - type: plugboard.library.file_io.FileWriter + args: + name: "save-high" + path: "high.csv" + field_names: + - value + connectors: + - source: "random-generator.value" + target: "find-high-low.value" + - source: "collect-low.value" + target: "save-low.value" + - source: "collect-high.value" + target: "save-high.value" From cb19233b51b98dfb2726554ae551d6a35d1d7429 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 20:21:22 +0000 Subject: [PATCH 11/32] Add info on logging to tutorial --- docs/examples/tutorials/more-components.md | 3 +++ examples/tutorials/003_more_components/hello_llm.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/examples/tutorials/more-components.md b/docs/examples/tutorials/more-components.md index ab0fc82e..1a467b9b 100644 --- a/docs/examples/tutorials/more-components.md +++ b/docs/examples/tutorials/more-components.md @@ -54,6 +54,9 @@ We can now define a component to query a weather API and get temperature and win --8<-- "examples/tutorials/003_more_components/hello_llm.py:weather" ``` +!!! info + See how we used `self._logger` to record log messages. All Plugboard [`Component`][plugboard.component.Component] objects have a [structlog](https://www.structlog.org/) logger on the `_logger` attribute. See [configuration](../../../usage/configuration/) for more information on configuring the logging. + ### Putting it all together As usual, we can link all our components together in a [`LocalProcess`][plugboard.process.Process] and run them as follows: diff --git a/examples/tutorials/003_more_components/hello_llm.py b/examples/tutorials/003_more_components/hello_llm.py index 10c41292..b595c5fd 100644 --- a/examples/tutorials/003_more_components/hello_llm.py +++ b/examples/tutorials/003_more_components/hello_llm.py @@ -36,8 +36,12 @@ async def step(self) -> None: ) try: response.raise_for_status() - except httpx.HTTPStatusError as e: - print(f"Error querying weather API: {e}") + except httpx.HTTPStatusError: + self._logger.error( + "Error querying weather API", + code=response.status_code, + message=response.text, + ) return data = response.json() self.temperature = data["current"]["temperature_2m"] From 956148d2e62d2a9b9b08ad73b58306b6cd213985 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 20:28:37 +0000 Subject: [PATCH 12/32] Add tags --- docs/examples/tutorials/event-driven-models.md | 5 +++++ docs/examples/tutorials/hello-world.md | 4 ++++ docs/examples/tutorials/more-complex-process.md | 4 ++++ docs/examples/tutorials/more-components.md | 7 +++++++ docs/examples/tutorials/running-in-parallel.md | 4 ++++ 5 files changed, 24 insertions(+) diff --git a/docs/examples/tutorials/event-driven-models.md b/docs/examples/tutorials/event-driven-models.md index 99b95d6d..fd1c4fd5 100644 --- a/docs/examples/tutorials/event-driven-models.md +++ b/docs/examples/tutorials/event-driven-models.md @@ -1,3 +1,8 @@ +--- +tags: + - tutorial + - events +--- So far everything we have built in Plugboard has been a **discrete-time model**. This means that the whole model advances step-wise, i.e. `step` gets called on each [`Component`][plugboard.component.Component], calculating all of their outputs before advancing the simulation on. In this tutorial we're going to introduce an **event-driven model**, where data can be passed around between components based on triggers that you can define. Event-based models can be useful in a variety of scenarios, for example when modelling parts moving around a production line, or passengers arriving at a transport hub. diff --git a/docs/examples/tutorials/hello-world.md b/docs/examples/tutorials/hello-world.md index 4825c6d9..7783c404 100644 --- a/docs/examples/tutorials/hello-world.md +++ b/docs/examples/tutorials/hello-world.md @@ -1,3 +1,7 @@ +--- +tags: + - tutorial +--- Plugboard is built to help you with two things: **defining process models**, and **executing those models**. There are two main ways to interact with Plugboard: via the Python API; or, via the CLI using model definitions saved in yaml format. In this introductory tutorial we'll do both, before building up to more complex models in later tutorials. ## Building models with the Python API diff --git a/docs/examples/tutorials/more-complex-process.md b/docs/examples/tutorials/more-complex-process.md index aeabdb45..af669ee1 100644 --- a/docs/examples/tutorials/more-complex-process.md +++ b/docs/examples/tutorials/more-complex-process.md @@ -1,3 +1,7 @@ +--- +tags: + - tutorial +--- In the last example our [`Process`][plugboard.process.Process] consisted of just two components. Usually we use many more components, allowing you to break down your model into separate parts that you can build/test individually. Plugboard allows for **branching** and **looping** connections between your components. In this tutorial we'll also demonstrate how to make components reusable between different processes. diff --git a/docs/examples/tutorials/more-components.md b/docs/examples/tutorials/more-components.md index 1a467b9b..c6e713cb 100644 --- a/docs/examples/tutorials/more-components.md +++ b/docs/examples/tutorials/more-components.md @@ -1,3 +1,10 @@ +--- +tags: + - tutorial + - logging + - llm + - io +--- Plugboard's [`Component`][plugboard.component.Component] objects can run anything you can code in Python. This includes: * Using your own or third-party Python packages; diff --git a/docs/examples/tutorials/running-in-parallel.md b/docs/examples/tutorials/running-in-parallel.md index c4163f1b..7a5635e9 100644 --- a/docs/examples/tutorials/running-in-parallel.md +++ b/docs/examples/tutorials/running-in-parallel.md @@ -1,3 +1,7 @@ +--- +tags: + - ray +--- Up until now we have running all our models in a single computational process. This is perfectly sufficient for simple models, or when your components can make use of [Python's asyncio](https://docs.python.org/3/library/asyncio.html) to avoid blocking. As your models get larger and more computationally intensive you may benefit from running parts of the model in parallel. Plugboard integrates with the [Ray](https://docs.ray.io/) framework, allowing you to split your computation across multiple CPU cores, or even across nodes in a [Ray cluster](https://docs.ray.io/en/latest/cluster/getting-started.html). From 6814620cb71279d230d29ee1fb9a9b60b4724d3f Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 20:39:06 +0000 Subject: [PATCH 13/32] Add tags --- docs/examples/tutorials/.meta.yml | 2 ++ docs/examples/tutorials/event-driven-models.md | 1 - docs/examples/tutorials/hello-world.md | 4 ---- docs/examples/tutorials/more-complex-process.md | 4 ---- docs/examples/tutorials/more-components.md | 1 - mkdocs.yaml | 2 ++ 6 files changed, 4 insertions(+), 10 deletions(-) create mode 100644 docs/examples/tutorials/.meta.yml diff --git a/docs/examples/tutorials/.meta.yml b/docs/examples/tutorials/.meta.yml new file mode 100644 index 00000000..c239a305 --- /dev/null +++ b/docs/examples/tutorials/.meta.yml @@ -0,0 +1,2 @@ +tags: + - tutorial diff --git a/docs/examples/tutorials/event-driven-models.md b/docs/examples/tutorials/event-driven-models.md index fd1c4fd5..c52c8d9e 100644 --- a/docs/examples/tutorials/event-driven-models.md +++ b/docs/examples/tutorials/event-driven-models.md @@ -1,6 +1,5 @@ --- tags: - - tutorial - events --- So far everything we have built in Plugboard has been a **discrete-time model**. This means that the whole model advances step-wise, i.e. `step` gets called on each [`Component`][plugboard.component.Component], calculating all of their outputs before advancing the simulation on. diff --git a/docs/examples/tutorials/hello-world.md b/docs/examples/tutorials/hello-world.md index 7783c404..4825c6d9 100644 --- a/docs/examples/tutorials/hello-world.md +++ b/docs/examples/tutorials/hello-world.md @@ -1,7 +1,3 @@ ---- -tags: - - tutorial ---- Plugboard is built to help you with two things: **defining process models**, and **executing those models**. There are two main ways to interact with Plugboard: via the Python API; or, via the CLI using model definitions saved in yaml format. In this introductory tutorial we'll do both, before building up to more complex models in later tutorials. ## Building models with the Python API diff --git a/docs/examples/tutorials/more-complex-process.md b/docs/examples/tutorials/more-complex-process.md index af669ee1..aeabdb45 100644 --- a/docs/examples/tutorials/more-complex-process.md +++ b/docs/examples/tutorials/more-complex-process.md @@ -1,7 +1,3 @@ ---- -tags: - - tutorial ---- In the last example our [`Process`][plugboard.process.Process] consisted of just two components. Usually we use many more components, allowing you to break down your model into separate parts that you can build/test individually. Plugboard allows for **branching** and **looping** connections between your components. In this tutorial we'll also demonstrate how to make components reusable between different processes. diff --git a/docs/examples/tutorials/more-components.md b/docs/examples/tutorials/more-components.md index c6e713cb..dba9d1b5 100644 --- a/docs/examples/tutorials/more-components.md +++ b/docs/examples/tutorials/more-components.md @@ -1,6 +1,5 @@ --- tags: - - tutorial - logging - llm - io diff --git a/mkdocs.yaml b/mkdocs.yaml index 7bd03225..88fd3882 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -34,6 +34,8 @@ plugins: - mike: alias_type: symlink canonical_version: latest +- meta +- tags markdown_extensions: - admonition From 0928c57db6463eb02079366f66b9b874faec934d Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 20:45:03 +0000 Subject: [PATCH 14/32] Add tags to demos --- examples/demos/llm/.meta.yml | 2 ++ examples/demos/llm/002_bluesky_websocket/.meta.yml | 3 +++ examples/demos/physics-models/.meta.yml | 2 ++ 3 files changed, 7 insertions(+) create mode 100644 examples/demos/llm/.meta.yml create mode 100644 examples/demos/llm/002_bluesky_websocket/.meta.yml create mode 100644 examples/demos/physics-models/.meta.yml diff --git a/examples/demos/llm/.meta.yml b/examples/demos/llm/.meta.yml new file mode 100644 index 00000000..e0d6d44d --- /dev/null +++ b/examples/demos/llm/.meta.yml @@ -0,0 +1,2 @@ +tags: + - llm \ No newline at end of file diff --git a/examples/demos/llm/002_bluesky_websocket/.meta.yml b/examples/demos/llm/002_bluesky_websocket/.meta.yml new file mode 100644 index 00000000..b6ee8f08 --- /dev/null +++ b/examples/demos/llm/002_bluesky_websocket/.meta.yml @@ -0,0 +1,3 @@ +tags: + - io + - streaming \ No newline at end of file diff --git a/examples/demos/physics-models/.meta.yml b/examples/demos/physics-models/.meta.yml new file mode 100644 index 00000000..6ff3114a --- /dev/null +++ b/examples/demos/physics-models/.meta.yml @@ -0,0 +1,2 @@ +tags: + - physics-models \ No newline at end of file From 34f9b04fe49f2cc6058daad697b92a95d96775a9 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 21 Mar 2025 21:00:47 +0000 Subject: [PATCH 15/32] Add tag index --- docs/usage/topics.md | 5 +++++ mkdocs.yaml | 1 + 2 files changed, 6 insertions(+) create mode 100644 docs/usage/topics.md diff --git a/docs/usage/topics.md b/docs/usage/topics.md new file mode 100644 index 00000000..e4822cbf --- /dev/null +++ b/docs/usage/topics.md @@ -0,0 +1,5 @@ +# Topic index + +To find information on a specific topic, you can look for pages under one of the tags below. + + \ No newline at end of file diff --git a/mkdocs.yaml b/mkdocs.yaml index 88fd3882..d093f680 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -114,6 +114,7 @@ nav: - Running in parallel: examples/tutorials/running-in-parallel.md - Event-driven models: examples/tutorials/event-driven-models.md - Configuration: usage/configuration.md + - Topics: usage/topics.md - Demos: - Fundamentals: - Simple model: examples/demos/fundamentals/001_simple_model/simple-model.ipynb From 57cb0652c92b1a8196e3cb527262a8677ba737ed Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sat, 22 Mar 2025 15:46:35 +0000 Subject: [PATCH 16/32] Add car wash demo --- .../002_car_wash_queue/.gitignore | 2 + .../fundamentals/002_car_wash_queue/.meta.yml | 2 + .../002_car_wash_queue/car-arrivals.csv | 242 +++++++++++ .../002_car_wash_queue/car-wash.ipynb | 375 ++++++++++++++++++ mkdocs.yaml | 1 + 5 files changed, 622 insertions(+) create mode 100644 examples/demos/fundamentals/002_car_wash_queue/.gitignore create mode 100644 examples/demos/fundamentals/002_car_wash_queue/.meta.yml create mode 100644 examples/demos/fundamentals/002_car_wash_queue/car-arrivals.csv create mode 100644 examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb diff --git a/examples/demos/fundamentals/002_car_wash_queue/.gitignore b/examples/demos/fundamentals/002_car_wash_queue/.gitignore new file mode 100644 index 00000000..de5b4daa --- /dev/null +++ b/examples/demos/fundamentals/002_car_wash_queue/.gitignore @@ -0,0 +1,2 @@ +*.csv +!car-arrivals.csv \ No newline at end of file diff --git a/examples/demos/fundamentals/002_car_wash_queue/.meta.yml b/examples/demos/fundamentals/002_car_wash_queue/.meta.yml new file mode 100644 index 00000000..9e58b17e --- /dev/null +++ b/examples/demos/fundamentals/002_car_wash_queue/.meta.yml @@ -0,0 +1,2 @@ +tags: + - events \ No newline at end of file diff --git a/examples/demos/fundamentals/002_car_wash_queue/car-arrivals.csv b/examples/demos/fundamentals/002_car_wash_queue/car-arrivals.csv new file mode 100644 index 00000000..824f57bf --- /dev/null +++ b/examples/demos/fundamentals/002_car_wash_queue/car-arrivals.csv @@ -0,0 +1,242 @@ +time_stamp,cars +2025-03-01 08:00:00,0 +2025-03-01 08:01:00,0 +2025-03-01 08:02:00,0 +2025-03-01 08:03:00,0 +2025-03-01 08:04:00,0 +2025-03-01 08:05:00,0 +2025-03-01 08:06:00,0 +2025-03-01 08:07:00,0 +2025-03-01 08:08:00,1 +2025-03-01 08:09:00,0 +2025-03-01 08:10:00,0 +2025-03-01 08:11:00,1 +2025-03-01 08:12:00,0 +2025-03-01 08:13:00,0 +2025-03-01 08:14:00,0 +2025-03-01 08:15:00,0 +2025-03-01 08:16:00,0 +2025-03-01 08:17:00,0 +2025-03-01 08:18:00,0 +2025-03-01 08:19:00,0 +2025-03-01 08:20:00,1 +2025-03-01 08:21:00,0 +2025-03-01 08:22:00,0 +2025-03-01 08:23:00,0 +2025-03-01 08:24:00,2 +2025-03-01 08:25:00,0 +2025-03-01 08:26:00,1 +2025-03-01 08:27:00,0 +2025-03-01 08:28:00,0 +2025-03-01 08:29:00,0 +2025-03-01 08:30:00,0 +2025-03-01 08:31:00,0 +2025-03-01 08:32:00,0 +2025-03-01 08:33:00,0 +2025-03-01 08:34:00,0 +2025-03-01 08:35:00,0 +2025-03-01 08:36:00,0 +2025-03-01 08:37:00,0 +2025-03-01 08:38:00,1 +2025-03-01 08:39:00,0 +2025-03-01 08:40:00,1 +2025-03-01 08:41:00,0 +2025-03-01 08:42:00,0 +2025-03-01 08:43:00,2 +2025-03-01 08:44:00,0 +2025-03-01 08:45:00,0 +2025-03-01 08:46:00,3 +2025-03-01 08:47:00,2 +2025-03-01 08:48:00,0 +2025-03-01 08:49:00,2 +2025-03-01 08:50:00,0 +2025-03-01 08:51:00,3 +2025-03-01 08:52:00,2 +2025-03-01 08:53:00,0 +2025-03-01 08:54:00,1 +2025-03-01 08:55:00,0 +2025-03-01 08:56:00,0 +2025-03-01 08:57:00,1 +2025-03-01 08:58:00,0 +2025-03-01 08:59:00,0 +2025-03-01 09:00:00,0 +2025-03-01 09:01:00,0 +2025-03-01 09:02:00,0 +2025-03-01 09:03:00,0 +2025-03-01 09:04:00,0 +2025-03-01 09:05:00,0 +2025-03-01 09:06:00,0 +2025-03-01 09:07:00,0 +2025-03-01 09:08:00,1 +2025-03-01 09:09:00,0 +2025-03-01 09:10:00,0 +2025-03-01 09:11:00,0 +2025-03-01 09:12:00,0 +2025-03-01 09:13:00,0 +2025-03-01 09:14:00,0 +2025-03-01 09:15:00,0 +2025-03-01 09:16:00,0 +2025-03-01 09:17:00,0 +2025-03-01 09:18:00,0 +2025-03-01 09:19:00,0 +2025-03-01 09:20:00,0 +2025-03-01 09:21:00,0 +2025-03-01 09:22:00,1 +2025-03-01 09:23:00,0 +2025-03-01 09:24:00,0 +2025-03-01 09:25:00,0 +2025-03-01 09:26:00,0 +2025-03-01 09:27:00,0 +2025-03-01 09:28:00,0 +2025-03-01 09:29:00,0 +2025-03-01 09:30:00,0 +2025-03-01 09:31:00,0 +2025-03-01 09:32:00,0 +2025-03-01 09:33:00,0 +2025-03-01 09:34:00,0 +2025-03-01 09:35:00,0 +2025-03-01 09:36:00,0 +2025-03-01 09:37:00,0 +2025-03-01 09:38:00,0 +2025-03-01 09:39:00,0 +2025-03-01 09:40:00,0 +2025-03-01 09:41:00,0 +2025-03-01 09:42:00,0 +2025-03-01 09:43:00,2 +2025-03-01 09:44:00,0 +2025-03-01 09:45:00,0 +2025-03-01 09:46:00,0 +2025-03-01 09:47:00,0 +2025-03-01 09:48:00,2 +2025-03-01 09:49:00,0 +2025-03-01 09:50:00,2 +2025-03-01 09:51:00,0 +2025-03-01 09:52:00,0 +2025-03-01 09:53:00,0 +2025-03-01 09:54:00,0 +2025-03-01 09:55:00,0 +2025-03-01 09:56:00,0 +2025-03-01 09:57:00,0 +2025-03-01 09:58:00,0 +2025-03-01 09:59:00,0 +2025-03-01 10:00:00,0 +2025-03-01 10:01:00,0 +2025-03-01 10:02:00,0 +2025-03-01 10:03:00,0 +2025-03-01 10:04:00,0 +2025-03-01 10:05:00,0 +2025-03-01 10:06:00,0 +2025-03-01 10:07:00,0 +2025-03-01 10:08:00,0 +2025-03-01 10:09:00,0 +2025-03-01 10:10:00,0 +2025-03-01 10:11:00,0 +2025-03-01 10:12:00,0 +2025-03-01 10:13:00,0 +2025-03-01 10:14:00,0 +2025-03-01 10:15:00,0 +2025-03-01 10:16:00,0 +2025-03-01 10:17:00,0 +2025-03-01 10:18:00,0 +2025-03-01 10:19:00,0 +2025-03-01 10:20:00,0 +2025-03-01 10:21:00,0 +2025-03-01 10:22:00,0 +2025-03-01 10:23:00,0 +2025-03-01 10:24:00,0 +2025-03-01 10:25:00,0 +2025-03-01 10:26:00,0 +2025-03-01 10:27:00,0 +2025-03-01 10:28:00,0 +2025-03-01 10:29:00,0 +2025-03-01 10:30:00,0 +2025-03-01 10:31:00,1 +2025-03-01 10:32:00,0 +2025-03-01 10:33:00,0 +2025-03-01 10:34:00,0 +2025-03-01 10:35:00,0 +2025-03-01 10:36:00,0 +2025-03-01 10:37:00,0 +2025-03-01 10:38:00,0 +2025-03-01 10:39:00,0 +2025-03-01 10:40:00,0 +2025-03-01 10:41:00,0 +2025-03-01 10:42:00,0 +2025-03-01 10:43:00,0 +2025-03-01 10:44:00,0 +2025-03-01 10:45:00,0 +2025-03-01 10:46:00,0 +2025-03-01 10:47:00,0 +2025-03-01 10:48:00,0 +2025-03-01 10:49:00,0 +2025-03-01 10:50:00,0 +2025-03-01 10:51:00,0 +2025-03-01 10:52:00,0 +2025-03-01 10:53:00,0 +2025-03-01 10:54:00,0 +2025-03-01 10:55:00,0 +2025-03-01 10:56:00,0 +2025-03-01 10:57:00,0 +2025-03-01 10:58:00,1 +2025-03-01 10:59:00,0 +2025-03-01 11:00:00,0 +2025-03-01 11:01:00,0 +2025-03-01 11:02:00,0 +2025-03-01 11:03:00,5 +2025-03-01 11:04:00,0 +2025-03-01 11:05:00,0 +2025-03-01 11:06:00,0 +2025-03-01 11:07:00,0 +2025-03-01 11:08:00,0 +2025-03-01 11:09:00,0 +2025-03-01 11:10:00,1 +2025-03-01 11:11:00,0 +2025-03-01 11:12:00,0 +2025-03-01 11:13:00,0 +2025-03-01 11:14:00,0 +2025-03-01 11:15:00,0 +2025-03-01 11:16:00,0 +2025-03-01 11:17:00,0 +2025-03-01 11:18:00,0 +2025-03-01 11:19:00,0 +2025-03-01 11:20:00,0 +2025-03-01 11:21:00,0 +2025-03-01 11:22:00,0 +2025-03-01 11:23:00,0 +2025-03-01 11:24:00,0 +2025-03-01 11:25:00,0 +2025-03-01 11:26:00,0 +2025-03-01 11:27:00,0 +2025-03-01 11:28:00,0 +2025-03-01 11:29:00,1 +2025-03-01 11:30:00,1 +2025-03-01 11:31:00,0 +2025-03-01 11:32:00,0 +2025-03-01 11:33:00,0 +2025-03-01 11:34:00,0 +2025-03-01 11:35:00,0 +2025-03-01 11:36:00,0 +2025-03-01 11:37:00,0 +2025-03-01 11:38:00,0 +2025-03-01 11:39:00,0 +2025-03-01 11:40:00,0 +2025-03-01 11:41:00,0 +2025-03-01 11:42:00,0 +2025-03-01 11:43:00,0 +2025-03-01 11:44:00,0 +2025-03-01 11:45:00,0 +2025-03-01 11:46:00,0 +2025-03-01 11:47:00,0 +2025-03-01 11:48:00,0 +2025-03-01 11:49:00,0 +2025-03-01 11:50:00,0 +2025-03-01 11:51:00,0 +2025-03-01 11:52:00,0 +2025-03-01 11:53:00,0 +2025-03-01 11:54:00,0 +2025-03-01 11:55:00,0 +2025-03-01 11:56:00,0 +2025-03-01 11:57:00,0 +2025-03-01 11:58:00,0 +2025-03-01 11:59:00,0 +2025-03-01 12:00:00,0 diff --git a/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb b/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb new file mode 100644 index 00000000..8fad8564 --- /dev/null +++ b/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb @@ -0,0 +1,375 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Car wash queing model\n", + "\n", + "[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/plugboard-dev/plugboard)\n", + "\n", + "This model uses event-driven components to model cars arriving and queueing at a carwash. We can use the model to understand how many cars are in the queue at a given moment.\n", + "\n", + "Cars arrive at the carwash and enter a queue to go into one of three washing machines. If any one of the machines is empty then the car at the front of the queue moves into it. The washing process takes a random amount of time (configurable on each machine).\n", + "\n", + "!!! tip\n", + " Install `pandas` and `plotly` to run this demo." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "import random\n", + "import typing as _t\n", + "\n", + "import pandas as pd\n", + "from pydantic import BaseModel\n", + "\n", + "from plugboard.connector import AsyncioConnector\n", + "from plugboard.component import Component, IOController\n", + "from plugboard.connector import AsyncioConnector, ConnectorBuilder\n", + "from plugboard.events import Event, EventConnectorBuilder, StopEvent\n", + "from plugboard.schemas import ComponentArgsDict, ConnectorSpec\n", + "from plugboard.process import LocalProcess\n", + "from plugboard.library import FileWriter" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class CarData(BaseModel):\n", + " \"\"\"Data for a car at the carwash.\"\"\"\n", + "\n", + " car_id: int\n", + " machine_id: _t.Optional[int] = None\n", + " arrival_time: _t.Optional[datetime] = None\n", + " leave_time: _t.Optional[datetime] = None\n", + "\n", + "\n", + "class CarArrived(Event):\n", + " \"\"\"Event for when a car arrives at the carwash.\"\"\"\n", + "\n", + " type: _t.ClassVar[str] = \"car_arrived\"\n", + " data: CarData\n", + "\n", + "\n", + "class CarEntersWash(Event):\n", + " \"\"\"Event for when a car enters the wash.\"\"\"\n", + "\n", + " type: _t.ClassVar[str] = \"car_enters_wash\"\n", + " data: CarData\n", + "\n", + "\n", + "class CarLeavesWash(Event):\n", + " \"\"\"Event for when a car leaves the wash.\"\"\"\n", + "\n", + " type: _t.ClassVar[str] = \"car_leaves_wash\"\n", + " data: CarData" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class CarArrivals(Component):\n", + " \"\"\"This component emits an event for each new car that arrives.\"\"\"\n", + "\n", + " io = IOController(outputs=[\"time_stamp\"], output_events=[CarArrived])\n", + "\n", + " def __init__(self, path: str, **kwargs):\n", + " super().__init__(**kwargs)\n", + " self._path = path\n", + " self._car_id = 0\n", + "\n", + " async def init(self) -> None:\n", + " df = pd.read_csv(self._path, parse_dates=[\"time_stamp\"])\n", + " self._iterator = df.iterrows()\n", + "\n", + " async def step(self) -> None:\n", + " try:\n", + " _, row = next(self._iterator)\n", + " except StopIteration:\n", + " self.io.queue_event(StopEvent(source=self.name, data={}))\n", + " return\n", + " self.time_stamp = row[\"time_stamp\"].to_pydatetime()\n", + " # Emit an event for each car that arrives at this time\n", + " for _ in range(row[\"cars\"]):\n", + " self._car_id += 1\n", + " car = CarData(car_id=self._car_id, arrival_time=self.time_stamp)\n", + " self.io.queue_event(CarArrived(source=self.name, data=car.model_dump()))\n", + " self._logger.info(\"Car arrived\", car=car.model_dump())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class CarWashQueue(Component):\n", + " \"\"\"This component manages the queue of cars waiting for the wash.\"\"\"\n", + "\n", + " io = IOController(\n", + " outputs=[\"queue_length\"],\n", + " input_events=[CarArrived, CarLeavesWash],\n", + " output_events=[CarEntersWash],\n", + " )\n", + "\n", + " def __init__(self, n_machines: int = 3, **kwargs):\n", + " super().__init__(**kwargs)\n", + " self._queue = []\n", + " # Use this dictionary to keep track of which machines are available\n", + " self._machine_status = {idx: False for idx in range(1, n_machines + 1)}\n", + "\n", + " async def step(self) -> None:\n", + " # Check if there are any cars waiting for the wash\n", + " while self._queue:\n", + " # Check if any of the machines are available\n", + " for idx, machine in self._machine_status.items():\n", + " if not machine:\n", + " # Machine is not busy - send car to wash\n", + " car = self._queue.pop(0)\n", + " car.machine_id = idx\n", + " self.io.queue_event(CarEntersWash(source=self.name, data=car))\n", + " self._logger.info(\"Car enters wash\", car=car.model_dump())\n", + " # Mark machine as busy\n", + " self._machine_status[idx] = True\n", + " break\n", + " else:\n", + " # No machines available\n", + " break\n", + " self.queue_length = len(self._queue)\n", + "\n", + " @CarArrived.handler\n", + " async def car_arrived(self, event: CarArrived) -> None:\n", + " # Add the car to the queue\n", + " self._queue.append(event.data)\n", + "\n", + " @CarLeavesWash.handler\n", + " async def car_leaves_wash(self, event: CarLeavesWash) -> None:\n", + " # Mark the machine as available\n", + " self._machine_status[event.data.machine_id] = False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class CarWashMachine(Component):\n", + " \"\"\"This component simulates the car wash machine.\"\"\"\n", + "\n", + " io = IOController(outputs=[\"busy\"], input_events=[CarEntersWash], output_events=[CarLeavesWash])\n", + "\n", + " def __init__(self, machine_id: int, wash_time_range: tuple[int, int] = (4, 6), **kwargs):\n", + " super().__init__(**kwargs)\n", + " self._machine_id = machine_id\n", + " self._wash_time_range = wash_time_range\n", + " self._time_remaining = 0\n", + " self._car = None\n", + " self.busy = False\n", + "\n", + " @CarEntersWash.handler\n", + " async def car_enters_wash(self, event: CarEntersWash) -> None:\n", + " # Check if car is going into this machine\n", + " if event.data.machine_id != self._machine_id:\n", + " return\n", + " # Start the wash\n", + " self._time_remaining = random.randint(*self._wash_time_range)\n", + " self.busy = True\n", + " self._car = event.data\n", + " self._logger.info(\"Car enters wash\", car=self._car.model_dump())\n", + "\n", + " async def step(self) -> None:\n", + " # Check if the wash is complete\n", + " if self.busy:\n", + " self._time_remaining -= 1\n", + " if self._time_remaining <= 0:\n", + " self.busy = False\n", + " self._car.leave_time = self.time_stamp\n", + " self.io.queue_event(CarLeavesWash(source=self.name, data=self._car))\n", + " self._logger.info(\"Car leaves wash\", car=self._car.model_dump())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class CaptureData(Component):\n", + " \"\"\"This component captures the data for each car that leaves the wash and saves to CSV.\"\"\"\n", + "\n", + " io = IOController(inputs=[\"time_stamp\"], input_events=[CarLeavesWash])\n", + "\n", + " def __init__(self, path: str, **kwargs):\n", + " super().__init__(**kwargs)\n", + " self._path = path\n", + " self._data = []\n", + "\n", + " async def step(self) -> None:\n", + " pass\n", + "\n", + " @CarLeavesWash.handler\n", + " async def car_leaves_wash(self, event: CarLeavesWash) -> None:\n", + " car = event.data\n", + " car.leave_time = self.time_stamp\n", + " self._data.append(car.model_dump())\n", + " self._logger.info(\"Car data captured\", car=car.model_dump())\n", + "\n", + " async def destroy(self):\n", + " pd.DataFrame(self._data).to_csv(self._path, index=False)\n", + " return await super().destroy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "components = [\n", + " CarArrivals(name=\"car-arrivals\", path=\"car-arrivals.csv\"),\n", + " CarWashQueue(name=\"car-wash-queue\", n_machines=3),\n", + " CarWashMachine(name=\"car-wash-machine-1\", machine_id=1, wash_time_range=(3, 5)),\n", + " CarWashMachine(name=\"car-wash-machine-2\", machine_id=2, wash_time_range=(3, 5)),\n", + " CarWashMachine(name=\"car-wash-machine-3\", machine_id=3, wash_time_range=(5, 9)),\n", + " CaptureData(name=\"capture-data\", path=\"car-wash-data.csv\"),\n", + " FileWriter(\n", + " name=\"write-data\",\n", + " path=\"queue-length.csv\",\n", + " chunk_size=1,\n", + " field_names=[\n", + " \"time_stamp\",\n", + " \"queue_length\",\n", + " \"machine_1_busy\",\n", + " \"machine_2_busy\",\n", + " \"machine_3_busy\",\n", + " ],\n", + " ),\n", + "]\n", + "connect = lambda in_, out_: AsyncioConnector(spec=ConnectorSpec(source=in_, target=out_))\n", + "connectors = [\n", + " connect(\"car-arrivals.time_stamp\", \"write-data.time_stamp\"),\n", + " connect(\"car-arrivals.time_stamp\", \"capture-data.time_stamp\"),\n", + " connect(\"car-wash-queue.queue_length\", \"write-data.queue_length\"),\n", + " connect(\"car-wash-machine-1.busy\", \"write-data.machine_1_busy\"),\n", + " connect(\"car-wash-machine-2.busy\", \"write-data.machine_2_busy\"),\n", + " connect(\"car-wash-machine-3.busy\", \"write-data.machine_3_busy\"),\n", + "]\n", + "connector_builder = ConnectorBuilder(connector_cls=AsyncioConnector)\n", + "event_connector_builder = EventConnectorBuilder(connector_builder=connector_builder)\n", + "event_connectors = list(event_connector_builder.build(components).values())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "process = LocalProcess(\n", + " components=components,\n", + " connectors=connectors + event_connectors,\n", + ")\n", + "async with process:\n", + " await process.run()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_queue = pd.read_csv(\"queue-length.csv\")\n", + "df_data = pd.read_csv(\"car-wash-data.csv\", parse_dates=[\"arrival_time\", \"leave_time\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_queue[[\"machine_1_busy\", \"machine_2_busy\", \"machine_3_busy\"]].mean().to_frame(\n", + " \"busy_percent\"\n", + ") * 100" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " fig = df_queue.plot(\n", + " backend=\"plotly\",\n", + " x=\"time_stamp\",\n", + " y=[\"queue_length\"],\n", + " title=\"Queue length at car wash\",\n", + " labels={\"index\": \"Time\", \"value\": \"Number of cars\"},\n", + " )\n", + "except (ImportError, ValueError):\n", + " print(\"Please install plotly to run this cell.\")\n", + " fig = None\n", + "fig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data = df_data.assign(\n", + " total_time=(df_data[\"leave_time\"] - df_data[\"arrival_time\"]).dt.total_seconds() / 60\n", + ")\n", + "try:\n", + " fig = df_data.plot(\n", + " backend=\"plotly\",\n", + " kind=\"hist\",\n", + " x=\"total_time\",\n", + " title=\"Time spent at car wash\",\n", + " labels={\"index\": \"Time\", \"value\": \"Number of cars\"},\n", + " )\n", + "except (ImportError, ValueError):\n", + " print(\"Please install plotly to run this cell.\")\n", + " fig = None\n", + "fig" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/mkdocs.yaml b/mkdocs.yaml index d093f680..3375c44b 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -118,6 +118,7 @@ nav: - Demos: - Fundamentals: - Simple model: examples/demos/fundamentals/001_simple_model/simple-model.ipynb + - Car wash: examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb - LLMs: - Data filtering: examples/demos/llm/001_data_filter/llm-filtering.ipynb - Websocket streaming: examples/demos/llm/002_bluesky_websocket/bluesky-websocket.ipynb From 400a22a23c3eb065986bdc54b7b06c4e7caa7fec Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sat, 22 Mar 2025 15:58:56 +0000 Subject: [PATCH 17/32] Update with text descriptions --- .../002_car_wash_queue/car-wash.ipynb | 78 ++++++++++++++++--- 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb b/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb index 8fad8564..fff069c4 100644 --- a/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb +++ b/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb @@ -4,16 +4,20 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Car wash queing model\n", + "# Car wash queueing model\n", "\n", "[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/plugboard-dev/plugboard)\n", "\n", "This model uses event-driven components to model cars arriving and queueing at a carwash. We can use the model to understand how many cars are in the queue at a given moment.\n", "\n", - "Cars arrive at the carwash and enter a queue to go into one of three washing machines. If any one of the machines is empty then the car at the front of the queue moves into it. The washing process takes a random amount of time (configurable on each machine).\n", + "Cars arrive at the carwash and enter a queue to go into one of three washing machines. If any one of the machines is empty then the car at the front of the queue moves into it, otherwise they will wait in the queue. The washing process takes a random amount of time (configurable on each machine).\n", "\n", - "!!! tip\n", - " Install `pandas` and `plotly` to run this demo." + "
\n", + "

Note

\n", + "

\n", + " Install pandas and plotly to run this demo.\n", + "

\n", + "
" ] }, { @@ -38,6 +42,16 @@ "from plugboard.library import FileWriter" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start be defining the events that are required for:\n", + "* A car arriving and entering the queue;\n", + "* A car moving from the queue into a washing machine;\n", + "* A car leaving the car-wash after washing is completed." + ] + }, { "cell_type": "code", "execution_count": null, @@ -74,6 +88,13 @@ " data: CarData" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The CSV file in`car-arrivals.csv` contains minute-by-minute data on the number of cars that arrive at the carwash. We need a component to read this data and publish an event for each car that arrives." + ] + }, { "cell_type": "code", "execution_count": null, @@ -85,7 +106,7 @@ "\n", " io = IOController(outputs=[\"time_stamp\"], output_events=[CarArrived])\n", "\n", - " def __init__(self, path: str, **kwargs):\n", + " def __init__(self, path: str, **kwargs: _t.Unpack[ComponentArgsDict]):\n", " super().__init__(**kwargs)\n", " self._path = path\n", " self._car_id = 0\n", @@ -109,6 +130,13 @@ " self._logger.info(\"Car arrived\", car=car.model_dump())" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This next component implements a queue. It monitors the status of each washing machine, and publishes events when cars move from the queue to a machine." + ] + }, { "cell_type": "code", "execution_count": null, @@ -124,7 +152,7 @@ " output_events=[CarEntersWash],\n", " )\n", "\n", - " def __init__(self, n_machines: int = 3, **kwargs):\n", + " def __init__(self, n_machines: int = 3, **kwargs: _t.Unpack[ComponentArgsDict]):\n", " super().__init__(**kwargs)\n", " self._queue = []\n", " # Use this dictionary to keep track of which machines are available\n", @@ -160,6 +188,13 @@ " self._machine_status[event.data.machine_id] = False" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next model the washing machines. Each one will listen for an event telling it to start washing a car." + ] + }, { "cell_type": "code", "execution_count": null, @@ -171,7 +206,12 @@ "\n", " io = IOController(outputs=[\"busy\"], input_events=[CarEntersWash], output_events=[CarLeavesWash])\n", "\n", - " def __init__(self, machine_id: int, wash_time_range: tuple[int, int] = (4, 6), **kwargs):\n", + " def __init__(\n", + " self,\n", + " machine_id: int,\n", + " wash_time_range: tuple[int, int] = (4, 6),\n", + " **kwargs: _t.Unpack[ComponentArgsDict],\n", + " ):\n", " super().__init__(**kwargs)\n", " self._machine_id = machine_id\n", " self._wash_time_range = wash_time_range\n", @@ -201,6 +241,13 @@ " self._logger.info(\"Car leaves wash\", car=self._car.model_dump())" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use this component to capture data as each car leaves the washing machines, so that we can record how long the process took for each vehicle." + ] + }, { "cell_type": "code", "execution_count": null, @@ -212,7 +259,7 @@ "\n", " io = IOController(inputs=[\"time_stamp\"], input_events=[CarLeavesWash])\n", "\n", - " def __init__(self, path: str, **kwargs):\n", + " def __init__(self, path: str, **kwargs: _t.Unpack[ComponentArgsDict]):\n", " super().__init__(**kwargs)\n", " self._path = path\n", " self._data = []\n", @@ -243,6 +290,7 @@ " CarWashQueue(name=\"car-wash-queue\", n_machines=3),\n", " CarWashMachine(name=\"car-wash-machine-1\", machine_id=1, wash_time_range=(3, 5)),\n", " CarWashMachine(name=\"car-wash-machine-2\", machine_id=2, wash_time_range=(3, 5)),\n", + " # The third machine takes longer to wash cars\n", " CarWashMachine(name=\"car-wash-machine-3\", machine_id=3, wash_time_range=(5, 9)),\n", " CaptureData(name=\"capture-data\", path=\"car-wash-data.csv\"),\n", " FileWriter(\n", @@ -286,6 +334,15 @@ " await process.run()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Results analysis\n", + "\n", + "Now load the output CSV files and analyse the data. Try going back and adjusting parameters to see their effect." + ] + }, { "cell_type": "code", "execution_count": null, @@ -302,8 +359,9 @@ "metadata": {}, "outputs": [], "source": [ + "# Print the average utilisation of each machine\n", "df_queue[[\"machine_1_busy\", \"machine_2_busy\", \"machine_3_busy\"]].mean().to_frame(\n", - " \"busy_percent\"\n", + " \"utilisation_percent\"\n", ") * 100" ] }, @@ -313,6 +371,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Plot the queue length over time\n", "try:\n", " fig = df_queue.plot(\n", " backend=\"plotly\",\n", @@ -333,6 +392,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Plot a histogram of the time spent at the car wash for each vehicle\n", "df_data = df_data.assign(\n", " total_time=(df_data[\"leave_time\"] - df_data[\"arrival_time\"]).dt.total_seconds() / 60\n", ")\n", From 3942f8609d31521a26ec2ed5eb373f5819651f15 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sat, 22 Mar 2025 16:15:08 +0000 Subject: [PATCH 18/32] README fixes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e4c375c4..873fc2d3 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Support for parallelisation can be installed using `plugboard[ray]`. ## 🚀 Usage -Plugboard is built to help you with two things: defining process models, and executing those models. There are two main ways to interact with plugboard: via the Python API; or, via the CLI using model definitions saved in yaml or json format. +Plugboard is built to help you with two things: defining process models, and executing those models. There are two main ways to interact with plugboard: via the Python API; or, via the CLI using model definitions saved in yaml format. ### Building models with the Python API @@ -135,7 +135,7 @@ graph LR; ### Executing pre-defined models on the CLI -In many cases, we want to define components once, with suitable parameters, and then use them repeatedly in different simulations. Plugboard enables this workflow with model specification files in yaml or json format. Once the components have been defined, the simple model above can be represented with a yaml file like so. +In many cases, we want to define components once, with suitable parameters, and then use them repeatedly in different simulations. Plugboard enables this workflow with model specification files in yaml format. Once the components have been defined, the simple model above can be represented as follows. ```yaml # my-model.yaml plugboard: From 924475829ac553af2ba562713c5ebb417fb67d9a Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sat, 22 Mar 2025 16:17:19 +0000 Subject: [PATCH 19/32] Minor updates --- .../demos/fundamentals/002_car_wash_queue/car-wash.ipynb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb b/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb index fff069c4..d2db261c 100644 --- a/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb +++ b/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb @@ -279,6 +279,13 @@ " return await super().destroy()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When building the `Process` to run the model, we need to create connectors both for the normal data inputs/outputs, and also for the events as shown below." + ] + }, { "cell_type": "code", "execution_count": null, @@ -296,7 +303,6 @@ " FileWriter(\n", " name=\"write-data\",\n", " path=\"queue-length.csv\",\n", - " chunk_size=1,\n", " field_names=[\n", " \"time_stamp\",\n", " \"queue_length\",\n", From 7fb6e237c5219a9d6425ebf7f960cb9f9ae1608d Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sat, 22 Mar 2025 18:42:39 +0000 Subject: [PATCH 20/32] Fix logger for types not supported by msgspec --- plugboard/utils/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugboard/utils/logging.py b/plugboard/utils/logging.py index a7f03a54..0c9fde88 100644 --- a/plugboard/utils/logging.py +++ b/plugboard/utils/logging.py @@ -18,7 +18,7 @@ def _is_ipython() -> bool: def _serialiser(obj: _t.Any, default: _t.Callable | None) -> bytes: - return json.encode(obj) + return json.encode(obj, enc_hook=default) def configure_logging(settings: Settings) -> None: From 78cac0760f6e3fb832b2a2aeed05f815e1373ad5 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sat, 22 Mar 2025 18:42:59 +0000 Subject: [PATCH 21/32] Remove logging conversions --- .../fundamentals/002_car_wash_queue/car-wash.ipynb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb b/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb index d2db261c..89005011 100644 --- a/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb +++ b/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb @@ -121,13 +121,13 @@ " except StopIteration:\n", " self.io.queue_event(StopEvent(source=self.name, data={}))\n", " return\n", - " self.time_stamp = row[\"time_stamp\"].to_pydatetime()\n", + " self.time_stamp = row[\"time_stamp\"]\n", " # Emit an event for each car that arrives at this time\n", " for _ in range(row[\"cars\"]):\n", " self._car_id += 1\n", " car = CarData(car_id=self._car_id, arrival_time=self.time_stamp)\n", - " self.io.queue_event(CarArrived(source=self.name, data=car.model_dump()))\n", - " self._logger.info(\"Car arrived\", car=car.model_dump())" + " self.io.queue_event(CarArrived(source=self.name, data=car))\n", + " self._logger.info(\"Car arrived\", car=car)" ] }, { @@ -168,7 +168,7 @@ " car = self._queue.pop(0)\n", " car.machine_id = idx\n", " self.io.queue_event(CarEntersWash(source=self.name, data=car))\n", - " self._logger.info(\"Car enters wash\", car=car.model_dump())\n", + " self._logger.info(\"Car enters wash\", car=car)\n", " # Mark machine as busy\n", " self._machine_status[idx] = True\n", " break\n", @@ -228,7 +228,7 @@ " self._time_remaining = random.randint(*self._wash_time_range)\n", " self.busy = True\n", " self._car = event.data\n", - " self._logger.info(\"Car enters wash\", car=self._car.model_dump())\n", + " self._logger.info(\"Car enters wash\", car=self._car)\n", "\n", " async def step(self) -> None:\n", " # Check if the wash is complete\n", @@ -238,7 +238,7 @@ " self.busy = False\n", " self._car.leave_time = self.time_stamp\n", " self.io.queue_event(CarLeavesWash(source=self.name, data=self._car))\n", - " self._logger.info(\"Car leaves wash\", car=self._car.model_dump())" + " self._logger.info(\"Car leaves wash\", car=self._car)" ] }, { @@ -272,7 +272,7 @@ " car = event.data\n", " car.leave_time = self.time_stamp\n", " self._data.append(car.model_dump())\n", - " self._logger.info(\"Car data captured\", car=car.model_dump())\n", + " self._logger.info(\"Car data captured\", car=car)\n", "\n", " async def destroy(self):\n", " pd.DataFrame(self._data).to_csv(self._path, index=False)\n", From f75bf3ab2a17667aac80d1d144ae042c51f1b1e6 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sat, 22 Mar 2025 18:47:35 +0000 Subject: [PATCH 22/32] Install ipywidgets to remove warnings in notebooks --- pyproject.toml | 1 + uv.lock | 46 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ae0922ce..20caa3b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ websockets = ["websockets>=14.2"] [dependency-groups] dev = [ "ipython~=8.26", + "ipywidgets>=8.1.5", "jupyterlab~=4.2", "mypy~=1.11", "nbstripout~=0.8", diff --git a/uv.lock b/uv.lock index ceadecd4..6571d992 100644 --- a/uv.lock +++ b/uv.lock @@ -552,7 +552,7 @@ name = "click" version = "8.1.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } wheels = [ @@ -1172,7 +1172,7 @@ name = "ipykernel" version = "6.29.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "appnope", marker = "platform_system == 'Darwin'" }, { name = "comm" }, { name = "debugpy" }, { name = "ipython" }, @@ -1211,6 +1211,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/e1/f4474a7ecdb7745a820f6f6039dc43c66add40f1bcc66485607d93571af6/ipython-8.32.0-py3-none-any.whl", hash = "sha256:cae85b0c61eff1fc48b0a8002de5958b6528fa9c8defb1894da63f42613708aa", size = 825524 }, ] +[[package]] +name = "ipywidgets" +version = "8.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "comm" }, + { name = "ipython" }, + { name = "jupyterlab-widgets" }, + { name = "traitlets" }, + { name = "widgetsnbextension" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/4c/dab2a281b07596a5fc220d49827fe6c794c66f1493d7a74f1df0640f2cc5/ipywidgets-8.1.5.tar.gz", hash = "sha256:870e43b1a35656a80c18c9503bbf2d16802db1cb487eec6fab27d683381dde17", size = 116723 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/2d/9c0b76f2f9cc0ebede1b9371b6f317243028ed60b90705863d493bae622e/ipywidgets-8.1.5-py3-none-any.whl", hash = "sha256:3290f526f87ae6e77655555baba4f36681c555b8bdbbff430b70e52c34c86245", size = 139767 }, +] + [[package]] name = "isodate" version = "0.7.2" @@ -1581,6 +1597,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/09/2032e7d15c544a0e3cd831c51d77a8ca57f7555b2e1b2922142eddb02a84/jupyterlab_server-2.27.3-py3-none-any.whl", hash = "sha256:e697488f66c3db49df675158a77b3b017520d772c6e1548c7d9bcc5df7944ee4", size = 59700 }, ] +[[package]] +name = "jupyterlab-widgets" +version = "3.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/73/fa26bbb747a9ea4fca6b01453aa22990d52ab62dd61384f1ac0dc9d4e7ba/jupyterlab_widgets-3.0.13.tar.gz", hash = "sha256:a2966d385328c1942b683a8cd96b89b8dd82c8b8f81dda902bb2bc06d46f5bed", size = 203556 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/93/858e87edc634d628e5d752ba944c2833133a28fa87bb093e6832ced36a3e/jupyterlab_widgets-3.0.13-py3-none-any.whl", hash = "sha256:e3cda2c233ce144192f1e29914ad522b2f4c40e77214b0cc97377ca3d323db54", size = 214392 }, +] + [[package]] name = "jupytext" version = "1.16.6" @@ -2016,7 +2041,7 @@ version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, { name = "ghp-import" }, { name = "jinja2" }, { name = "markdown" }, @@ -2782,6 +2807,7 @@ websockets = [ [package.dev-dependencies] dev = [ { name = "ipython" }, + { name = "ipywidgets" }, { name = "jupyterlab" }, { name = "mypy" }, { name = "nbstripout" }, @@ -2844,6 +2870,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "ipython", specifier = "~=8.26" }, + { name = "ipywidgets", specifier = ">=8.1.5" }, { name = "jupyterlab", specifier = "~=4.2" }, { name = "mypy", specifier = "~=1.11" }, { name = "nbstripout", specifier = "~=0.8" }, @@ -2901,7 +2928,7 @@ name = "portalocker" version = "2.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ed/d3/c6c64067759e87af98cc668c1cc75171347d0f1577fab7ca3749134e3cd4/portalocker-2.10.1.tar.gz", hash = "sha256:ef1bf844e878ab08aee7e40184156e1151f228f103aa5c6bd0724cc330960f8f", size = 40891 } wheels = [ @@ -4001,7 +4028,7 @@ name = "tqdm" version = "4.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } wheels = [ @@ -4225,6 +4252,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498 }, ] +[[package]] +name = "widgetsnbextension" +version = "4.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/fc/238c424fd7f4ebb25f8b1da9a934a3ad7c848286732ae04263661eb0fc03/widgetsnbextension-4.0.13.tar.gz", hash = "sha256:ffcb67bc9febd10234a362795f643927f4e0c05d9342c727b65d2384f8feacb6", size = 1164730 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/02/88b65cc394961a60c43c70517066b6b679738caf78506a5da7b88ffcb643/widgetsnbextension-4.0.13-py3-none-any.whl", hash = "sha256:74b2692e8500525cc38c2b877236ba51d34541e6385eeed5aec15a70f88a6c71", size = 2335872 }, +] + [[package]] name = "wrapt" version = "1.17.2" From b7692b0dc4b4e949c3498805d09cdb7d64a40a01 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Tue, 1 Apr 2025 20:56:49 +0100 Subject: [PATCH 23/32] Update diagram --- docs/examples/tutorials/event-driven-models.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/examples/tutorials/event-driven-models.md b/docs/examples/tutorials/event-driven-models.md index c52c8d9e..a9be7df6 100644 --- a/docs/examples/tutorials/event-driven-models.md +++ b/docs/examples/tutorials/event-driven-models.md @@ -11,14 +11,14 @@ In this tutorial we're going to introduce an **event-driven model**, where data Here's the model that we're going to build. Given a stream of random numbers, we'll trigger `HighEvent` whenever the value is above `0.8` and `LowEvent` whenever the value is below `0.2`. This allows us to funnel data into different parts of the model: in this case we'll just save the latest high/low values to a file at each step. In the diagram the _dotted lines_ represent the flow of event data: `FindHighLowValues` will publish events, while `CollectHigh` and `CollectLow` will subscribe to receive high and low events respectively. ```mermaid -graph LR; - Random(random-generator)-->FindHighLowValues(find-high-low); - FindHighLowValues(find-high-low)-.->HighEvent{{high-event}}; - FindHighLowValues(find-high-low)-.->LowEvent{{low-event}}; - HighEvent{{high-event}}-.->CollectHigh(collect-high); - LowEvent{{low-event}}-.->CollectLow(collect-low); - CollectHigh(collect-high)-->SaveHigh(save-high); - CollectLow(collect-low)-->SaveLow(save-low); +flowchart LR + collect-high@{ shape: rounded, label: CollectHigh
**collect-high** } --> save-high@{ shape: rounded, label: FileWriter
**save-high** } + collect-low@{ shape: rounded, label: CollectLow
**collect-low** } --> save-low@{ shape: rounded, label: FileWriter
**save-low** } + random-generator@{ shape: rounded, label: Random
**random-generator** } --> find-high-low@{ shape: rounded, label: FindHighLowValues
**find-high-low** } + low_event@{ shape: hex, label: LowEvent } -.-> collect-low@{ shape: rounded, label: CollectLow
**collect-low** } + high_event@{ shape: hex, label: HighEvent } -.-> collect-high@{ shape: rounded, label: CollectHigh
**collect-high** } + find-high-low@{ shape: rounded, label: FindHighLowValues
**find-high-low** } -.-> high_event@{ shape: hex, label: HighEvent } + find-high-low@{ shape: rounded, label: FindHighLowValues
**find-high-low** } -.-> low_event@{ shape: hex, label: LowEvent } ``` ## Defining events From 971a22c7c233fa2dc820b347a965326f49fb735e Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Mon, 15 Sep 2025 21:27:52 +0100 Subject: [PATCH 24/32] Rename folder --- .../{002_car_wash_queue => 003_car_wash_queue}/.gitignore | 0 .../{002_car_wash_queue => 003_car_wash_queue}/.meta.yml | 0 .../{002_car_wash_queue => 003_car_wash_queue}/car-arrivals.csv | 0 .../{002_car_wash_queue => 003_car_wash_queue}/car-wash.ipynb | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename examples/demos/fundamentals/{002_car_wash_queue => 003_car_wash_queue}/.gitignore (100%) rename examples/demos/fundamentals/{002_car_wash_queue => 003_car_wash_queue}/.meta.yml (100%) rename examples/demos/fundamentals/{002_car_wash_queue => 003_car_wash_queue}/car-arrivals.csv (100%) rename examples/demos/fundamentals/{002_car_wash_queue => 003_car_wash_queue}/car-wash.ipynb (100%) diff --git a/examples/demos/fundamentals/002_car_wash_queue/.gitignore b/examples/demos/fundamentals/003_car_wash_queue/.gitignore similarity index 100% rename from examples/demos/fundamentals/002_car_wash_queue/.gitignore rename to examples/demos/fundamentals/003_car_wash_queue/.gitignore diff --git a/examples/demos/fundamentals/002_car_wash_queue/.meta.yml b/examples/demos/fundamentals/003_car_wash_queue/.meta.yml similarity index 100% rename from examples/demos/fundamentals/002_car_wash_queue/.meta.yml rename to examples/demos/fundamentals/003_car_wash_queue/.meta.yml diff --git a/examples/demos/fundamentals/002_car_wash_queue/car-arrivals.csv b/examples/demos/fundamentals/003_car_wash_queue/car-arrivals.csv similarity index 100% rename from examples/demos/fundamentals/002_car_wash_queue/car-arrivals.csv rename to examples/demos/fundamentals/003_car_wash_queue/car-arrivals.csv diff --git a/examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb b/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb similarity index 100% rename from examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb rename to examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb From 642086a6657898affaa200842ca40b416cf1e4c1 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Tue, 16 Sep 2025 20:20:14 +0100 Subject: [PATCH 25/32] Fixup event tutorial --- docs/examples/tutorials/event-driven-models.md | 8 +++----- examples/tutorials/005_events/hello_events.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/examples/tutorials/event-driven-models.md b/docs/examples/tutorials/event-driven-models.md index a9be7df6..d3fe681d 100644 --- a/docs/examples/tutorials/event-driven-models.md +++ b/docs/examples/tutorials/event-driven-models.md @@ -33,14 +33,12 @@ First we need to define the events that are going to get used in the model. Each So far all of our process models have run step-by-step until completion. When a model contains event-driven components, we need a way to tell them to stop at the end of the simulation, otherwise they will stay running and listening for events forever. -In this example, our `Random` component will drive the process by generating input random values. When it has completed `iters` iterations, we can use it to stop the model by sending a [`StopEvent`][plugboard.events.StopEvent], causing other event-driven components in the model to shutdown. +In this example, our `Random` component will drive the process by generating input random values. When it has completed `iters` iterations, we call `self.io.close()` to stop the model, causing other components in the model to shutdown. ```python --8<-- "examples/tutorials/005_events/hello_events.py:source-component" ``` -1. Use `self.io.queue_event` to send an event from a [`Component`][plugboard.component.Component]. Here we are sending [`StopEvent`][plugboard.events.StopEvent] to stop the process once all of the random values have been generated. - Next, we will define `FindHighLowValues` to identify high and low values in the stream of random numbers and publish `HighEvent` and `LowEvent` respectively. ```python @@ -48,7 +46,7 @@ Next, we will define `FindHighLowValues` to identify high and low values in the ``` 1. See how we use the [`IOController`][plugboard.component.IOController] to declare that this [`Component`][plugboard.component.Component] will publish events. -2. Call `self.io.queue_event` to send an event of the correct type. +2. Use `self.io.queue_event` to send an event from a [`Component`][plugboard.component.Component]. Here we are senging the `HighEvent` or `LowEvent` depending on the input value. Finally, we need components to subscribe to these events and process them. Use the `Event.handler` decorator to identify the method on each [`Component`][plugboard.component.Component] that will do this processing. @@ -61,7 +59,7 @@ Finally, we need components to subscribe to these events and process them. Use t 3. ...and we handle `LowEvent` here. !!! note - In a real model you could define whatever logic you need inside your event handler, e.g. create a file, publish another event, etc. Here we just store the event on an attribute so that its value can be output on the next call to `step()`. + In a real model you could define whatever logic you need inside your event handler, e.g. create a file, publish another event, etc. Here we just store the event on an attribute so that its value can be output via the `step()` method. ## Putting it all together diff --git a/examples/tutorials/005_events/hello_events.py b/examples/tutorials/005_events/hello_events.py index 002a0f10..c2a03f2f 100644 --- a/examples/tutorials/005_events/hello_events.py +++ b/examples/tutorials/005_events/hello_events.py @@ -53,7 +53,7 @@ async def step(self) -> None: self.completed_iters += 1 self.value = random.random() if self.completed_iters >= self.max_iters: - self.io.queue_event(StopEvent(source=self.name, data={})) # (1)! + await self.io.close() # --8<-- [end:source-component] From 9ffa0a017e01943fc58607ccfb3239a301a5c5c2 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Tue, 16 Sep 2025 20:43:49 +0100 Subject: [PATCH 26/32] Fixup carwash examples --- .../003_car_wash_queue/car-wash.ipynb | 17 ++++++++++++----- mkdocs.yaml | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb b/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb index 89005011..cf3f6406 100644 --- a/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb +++ b/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb @@ -119,7 +119,7 @@ " try:\n", " _, row = next(self._iterator)\n", " except StopIteration:\n", - " self.io.queue_event(StopEvent(source=self.name, data={}))\n", + " await self.io.close()\n", " return\n", " self.time_stamp = row[\"time_stamp\"]\n", " # Emit an event for each car that arrives at this time\n", @@ -147,6 +147,7 @@ " \"\"\"This component manages the queue of cars waiting for the wash.\"\"\"\n", "\n", " io = IOController(\n", + " inputs=[\"time_stamp\"],\n", " outputs=[\"queue_length\"],\n", " input_events=[CarArrived, CarLeavesWash],\n", " output_events=[CarEntersWash],\n", @@ -204,7 +205,12 @@ "class CarWashMachine(Component):\n", " \"\"\"This component simulates the car wash machine.\"\"\"\n", "\n", - " io = IOController(outputs=[\"busy\"], input_events=[CarEntersWash], output_events=[CarLeavesWash])\n", + " io = IOController(\n", + " inputs=[\"time_stamp\"],\n", + " outputs=[\"busy\"],\n", + " input_events=[CarEntersWash],\n", + " output_events=[CarLeavesWash],\n", + " )\n", "\n", " def __init__(\n", " self,\n", @@ -264,9 +270,6 @@ " self._path = path\n", " self._data = []\n", "\n", - " async def step(self) -> None:\n", - " pass\n", - "\n", " @CarLeavesWash.handler\n", " async def car_leaves_wash(self, event: CarLeavesWash) -> None:\n", " car = event.data\n", @@ -316,6 +319,10 @@ "connectors = [\n", " connect(\"car-arrivals.time_stamp\", \"write-data.time_stamp\"),\n", " connect(\"car-arrivals.time_stamp\", \"capture-data.time_stamp\"),\n", + " connect(\"car-arrivals.time_stamp\", \"car-wash-queue.time_stamp\"),\n", + " connect(\"car-arrivals.time_stamp\", \"car-wash-machine-1.time_stamp\"),\n", + " connect(\"car-arrivals.time_stamp\", \"car-wash-machine-2.time_stamp\"),\n", + " connect(\"car-arrivals.time_stamp\", \"car-wash-machine-3.time_stamp\"),\n", " connect(\"car-wash-queue.queue_length\", \"write-data.queue_length\"),\n", " connect(\"car-wash-machine-1.busy\", \"write-data.machine_1_busy\"),\n", " connect(\"car-wash-machine-2.busy\", \"write-data.machine_2_busy\"),\n", diff --git a/mkdocs.yaml b/mkdocs.yaml index a6ec5882..c521cf11 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -119,7 +119,7 @@ nav: - Demos: - Fundamentals: - Simple model: examples/demos/fundamentals/001_simple_model/simple-model.ipynb - - Car wash: examples/demos/fundamentals/002_car_wash_queue/car-wash.ipynb + - Car wash: examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb - LLMs: - Data filtering: examples/demos/llm/001_data_filter/llm-filtering.ipynb - Websocket streaming: examples/demos/llm/002_bluesky_websocket/bluesky-websocket.ipynb From 729806e61cf13c871cc9bb4a092913476b57bc21 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Wed, 17 Sep 2025 20:01:08 +0100 Subject: [PATCH 27/32] Update LLM prompt --- .github/instructions/models.instructions.md | 84 ++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/.github/instructions/models.instructions.md b/.github/instructions/models.instructions.md index 49c05273..d8ac9432 100644 --- a/.github/instructions/models.instructions.md +++ b/.github/instructions/models.instructions.md @@ -95,4 +95,86 @@ import asyncio async with process: await process.run() -``` \ No newline at end of file +``` + +## Event-driven models + +You can help users to implement event-driven models using Plugboard's event system. Components can emit and handle events to communicate with each other. + +Examples of where you might want to use events include: +* A component that monitors a data stream and emits an event when a threshold is crossed; +* A component that listens for events and triggers actions in response, e.g. sending an alert; +* A trading algorithm that uses events to signal buy/sell decisions; +* A simulation of a taxi rank where events are used to signal the arrival and departure of taxis and passengers. + +Events must be defined by inheriting from the `plugboard.events.Event` class. Each event class should define the data it carries using a Pydantic `BaseModel`. For example: + +```python +from pydantic import BaseModel +from plugboard.events import Event + +class MyEventData(BaseModel): + some_value: int + another_value: str + +class MyEvent(Event): + data: MyEventData +``` + +Components can emit events using the `self.io.queue_event()` method or by returning them from an event handler. Event handlers are defined using methods decorated with `@EventClass.handler`. For example: + +```python +from plugboard.component import Component, IOController as IO + +class MyEventPublisher(Component): + io = IO(inputs=["some_input"], output_events=[MyEvent]) + + async def step(self) -> None: + # Emit an event + event_data = MyEventData(some_value=42, another_value=f"received {self.some_input}") + self.io.queue_event(MyEvent(source=self.name, data=event_data)) + +class MyEventSubscriber(Component): + io = IO(input_events=[MyEvent], output_events=[MyEvent]) + + @MyEvent.handler + async def handle_my_event(self, event: MyEvent) -> MyEvent: + # Handle the event + print(f"Received event: {event.data}") + output_event_data = MyEventData(some_value=event.data.some_value + 1, another_value="handled") + return MyEvent(source=self.name, data=output_event_data) +``` + +To assemble a process with event-driven components, you can use the same approach as for non-event-driven components. You will need to create connectors for event-driven components using `plugboard.events.event_connector_builder.EventConnectorBuilder`. For example: + +```python +from plugboard.connector import AsyncioConnector, ConnectorBuilder +from plugboard.events.event_connector_builder import EventConnectorBuilder +from plugboard.process import LocalProcess + +# Define components.... +component_1 = ... +component_2 = ... + +# Define connectors for non-event components as before +connect = lambda in_, out_: AsyncioConnector(spec=ConnectorSpec(source=in_, target=out_)) +connectors = [ + connect("component_1.output", "component_2.input"), + ... +] + +connector_builder = ConnectorBuilder(connector_cls=AsyncioConnector) +event_connector_builder = EventConnectorBuilder(connector_builder=connector_builder) +event_connectors = list(event_connector_builder.build(components).values()) + +process = LocalProcess( + components=[ + component_1, component_2, ... + ], + connectors=connectors + event_connectors, +) +``` + +## Exporting models + +If the user wants to export their model you use in the CLI, you can do this by calling `process.dump("path/to/file.yaml")`. From ad955e567423f377487c8e327d11df98f222b82c Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Wed, 17 Sep 2025 20:02:12 +0100 Subject: [PATCH 28/32] Unused import --- examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb b/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb index cf3f6406..03b4af0f 100644 --- a/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb +++ b/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb @@ -36,7 +36,7 @@ "from plugboard.connector import AsyncioConnector\n", "from plugboard.component import Component, IOController\n", "from plugboard.connector import AsyncioConnector, ConnectorBuilder\n", - "from plugboard.events import Event, EventConnectorBuilder, StopEvent\n", + "from plugboard.events import Event, EventConnectorBuilder\n", "from plugboard.schemas import ComponentArgsDict, ConnectorSpec\n", "from plugboard.process import LocalProcess\n", "from plugboard.library import FileWriter" From 65f85d46c2688c401cc883cfadeb04fa02efd5c6 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Tue, 23 Sep 2025 20:20:31 +0100 Subject: [PATCH 29/32] Delete car wash example --- .../003_car_wash_queue/.gitignore | 2 - .../fundamentals/003_car_wash_queue/.meta.yml | 2 - .../003_car_wash_queue/car-arrivals.csv | 242 ---------- .../003_car_wash_queue/car-wash.ipynb | 448 ------------------ 4 files changed, 694 deletions(-) delete mode 100644 examples/demos/fundamentals/003_car_wash_queue/.gitignore delete mode 100644 examples/demos/fundamentals/003_car_wash_queue/.meta.yml delete mode 100644 examples/demos/fundamentals/003_car_wash_queue/car-arrivals.csv delete mode 100644 examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb diff --git a/examples/demos/fundamentals/003_car_wash_queue/.gitignore b/examples/demos/fundamentals/003_car_wash_queue/.gitignore deleted file mode 100644 index de5b4daa..00000000 --- a/examples/demos/fundamentals/003_car_wash_queue/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.csv -!car-arrivals.csv \ No newline at end of file diff --git a/examples/demos/fundamentals/003_car_wash_queue/.meta.yml b/examples/demos/fundamentals/003_car_wash_queue/.meta.yml deleted file mode 100644 index 9e58b17e..00000000 --- a/examples/demos/fundamentals/003_car_wash_queue/.meta.yml +++ /dev/null @@ -1,2 +0,0 @@ -tags: - - events \ No newline at end of file diff --git a/examples/demos/fundamentals/003_car_wash_queue/car-arrivals.csv b/examples/demos/fundamentals/003_car_wash_queue/car-arrivals.csv deleted file mode 100644 index 824f57bf..00000000 --- a/examples/demos/fundamentals/003_car_wash_queue/car-arrivals.csv +++ /dev/null @@ -1,242 +0,0 @@ -time_stamp,cars -2025-03-01 08:00:00,0 -2025-03-01 08:01:00,0 -2025-03-01 08:02:00,0 -2025-03-01 08:03:00,0 -2025-03-01 08:04:00,0 -2025-03-01 08:05:00,0 -2025-03-01 08:06:00,0 -2025-03-01 08:07:00,0 -2025-03-01 08:08:00,1 -2025-03-01 08:09:00,0 -2025-03-01 08:10:00,0 -2025-03-01 08:11:00,1 -2025-03-01 08:12:00,0 -2025-03-01 08:13:00,0 -2025-03-01 08:14:00,0 -2025-03-01 08:15:00,0 -2025-03-01 08:16:00,0 -2025-03-01 08:17:00,0 -2025-03-01 08:18:00,0 -2025-03-01 08:19:00,0 -2025-03-01 08:20:00,1 -2025-03-01 08:21:00,0 -2025-03-01 08:22:00,0 -2025-03-01 08:23:00,0 -2025-03-01 08:24:00,2 -2025-03-01 08:25:00,0 -2025-03-01 08:26:00,1 -2025-03-01 08:27:00,0 -2025-03-01 08:28:00,0 -2025-03-01 08:29:00,0 -2025-03-01 08:30:00,0 -2025-03-01 08:31:00,0 -2025-03-01 08:32:00,0 -2025-03-01 08:33:00,0 -2025-03-01 08:34:00,0 -2025-03-01 08:35:00,0 -2025-03-01 08:36:00,0 -2025-03-01 08:37:00,0 -2025-03-01 08:38:00,1 -2025-03-01 08:39:00,0 -2025-03-01 08:40:00,1 -2025-03-01 08:41:00,0 -2025-03-01 08:42:00,0 -2025-03-01 08:43:00,2 -2025-03-01 08:44:00,0 -2025-03-01 08:45:00,0 -2025-03-01 08:46:00,3 -2025-03-01 08:47:00,2 -2025-03-01 08:48:00,0 -2025-03-01 08:49:00,2 -2025-03-01 08:50:00,0 -2025-03-01 08:51:00,3 -2025-03-01 08:52:00,2 -2025-03-01 08:53:00,0 -2025-03-01 08:54:00,1 -2025-03-01 08:55:00,0 -2025-03-01 08:56:00,0 -2025-03-01 08:57:00,1 -2025-03-01 08:58:00,0 -2025-03-01 08:59:00,0 -2025-03-01 09:00:00,0 -2025-03-01 09:01:00,0 -2025-03-01 09:02:00,0 -2025-03-01 09:03:00,0 -2025-03-01 09:04:00,0 -2025-03-01 09:05:00,0 -2025-03-01 09:06:00,0 -2025-03-01 09:07:00,0 -2025-03-01 09:08:00,1 -2025-03-01 09:09:00,0 -2025-03-01 09:10:00,0 -2025-03-01 09:11:00,0 -2025-03-01 09:12:00,0 -2025-03-01 09:13:00,0 -2025-03-01 09:14:00,0 -2025-03-01 09:15:00,0 -2025-03-01 09:16:00,0 -2025-03-01 09:17:00,0 -2025-03-01 09:18:00,0 -2025-03-01 09:19:00,0 -2025-03-01 09:20:00,0 -2025-03-01 09:21:00,0 -2025-03-01 09:22:00,1 -2025-03-01 09:23:00,0 -2025-03-01 09:24:00,0 -2025-03-01 09:25:00,0 -2025-03-01 09:26:00,0 -2025-03-01 09:27:00,0 -2025-03-01 09:28:00,0 -2025-03-01 09:29:00,0 -2025-03-01 09:30:00,0 -2025-03-01 09:31:00,0 -2025-03-01 09:32:00,0 -2025-03-01 09:33:00,0 -2025-03-01 09:34:00,0 -2025-03-01 09:35:00,0 -2025-03-01 09:36:00,0 -2025-03-01 09:37:00,0 -2025-03-01 09:38:00,0 -2025-03-01 09:39:00,0 -2025-03-01 09:40:00,0 -2025-03-01 09:41:00,0 -2025-03-01 09:42:00,0 -2025-03-01 09:43:00,2 -2025-03-01 09:44:00,0 -2025-03-01 09:45:00,0 -2025-03-01 09:46:00,0 -2025-03-01 09:47:00,0 -2025-03-01 09:48:00,2 -2025-03-01 09:49:00,0 -2025-03-01 09:50:00,2 -2025-03-01 09:51:00,0 -2025-03-01 09:52:00,0 -2025-03-01 09:53:00,0 -2025-03-01 09:54:00,0 -2025-03-01 09:55:00,0 -2025-03-01 09:56:00,0 -2025-03-01 09:57:00,0 -2025-03-01 09:58:00,0 -2025-03-01 09:59:00,0 -2025-03-01 10:00:00,0 -2025-03-01 10:01:00,0 -2025-03-01 10:02:00,0 -2025-03-01 10:03:00,0 -2025-03-01 10:04:00,0 -2025-03-01 10:05:00,0 -2025-03-01 10:06:00,0 -2025-03-01 10:07:00,0 -2025-03-01 10:08:00,0 -2025-03-01 10:09:00,0 -2025-03-01 10:10:00,0 -2025-03-01 10:11:00,0 -2025-03-01 10:12:00,0 -2025-03-01 10:13:00,0 -2025-03-01 10:14:00,0 -2025-03-01 10:15:00,0 -2025-03-01 10:16:00,0 -2025-03-01 10:17:00,0 -2025-03-01 10:18:00,0 -2025-03-01 10:19:00,0 -2025-03-01 10:20:00,0 -2025-03-01 10:21:00,0 -2025-03-01 10:22:00,0 -2025-03-01 10:23:00,0 -2025-03-01 10:24:00,0 -2025-03-01 10:25:00,0 -2025-03-01 10:26:00,0 -2025-03-01 10:27:00,0 -2025-03-01 10:28:00,0 -2025-03-01 10:29:00,0 -2025-03-01 10:30:00,0 -2025-03-01 10:31:00,1 -2025-03-01 10:32:00,0 -2025-03-01 10:33:00,0 -2025-03-01 10:34:00,0 -2025-03-01 10:35:00,0 -2025-03-01 10:36:00,0 -2025-03-01 10:37:00,0 -2025-03-01 10:38:00,0 -2025-03-01 10:39:00,0 -2025-03-01 10:40:00,0 -2025-03-01 10:41:00,0 -2025-03-01 10:42:00,0 -2025-03-01 10:43:00,0 -2025-03-01 10:44:00,0 -2025-03-01 10:45:00,0 -2025-03-01 10:46:00,0 -2025-03-01 10:47:00,0 -2025-03-01 10:48:00,0 -2025-03-01 10:49:00,0 -2025-03-01 10:50:00,0 -2025-03-01 10:51:00,0 -2025-03-01 10:52:00,0 -2025-03-01 10:53:00,0 -2025-03-01 10:54:00,0 -2025-03-01 10:55:00,0 -2025-03-01 10:56:00,0 -2025-03-01 10:57:00,0 -2025-03-01 10:58:00,1 -2025-03-01 10:59:00,0 -2025-03-01 11:00:00,0 -2025-03-01 11:01:00,0 -2025-03-01 11:02:00,0 -2025-03-01 11:03:00,5 -2025-03-01 11:04:00,0 -2025-03-01 11:05:00,0 -2025-03-01 11:06:00,0 -2025-03-01 11:07:00,0 -2025-03-01 11:08:00,0 -2025-03-01 11:09:00,0 -2025-03-01 11:10:00,1 -2025-03-01 11:11:00,0 -2025-03-01 11:12:00,0 -2025-03-01 11:13:00,0 -2025-03-01 11:14:00,0 -2025-03-01 11:15:00,0 -2025-03-01 11:16:00,0 -2025-03-01 11:17:00,0 -2025-03-01 11:18:00,0 -2025-03-01 11:19:00,0 -2025-03-01 11:20:00,0 -2025-03-01 11:21:00,0 -2025-03-01 11:22:00,0 -2025-03-01 11:23:00,0 -2025-03-01 11:24:00,0 -2025-03-01 11:25:00,0 -2025-03-01 11:26:00,0 -2025-03-01 11:27:00,0 -2025-03-01 11:28:00,0 -2025-03-01 11:29:00,1 -2025-03-01 11:30:00,1 -2025-03-01 11:31:00,0 -2025-03-01 11:32:00,0 -2025-03-01 11:33:00,0 -2025-03-01 11:34:00,0 -2025-03-01 11:35:00,0 -2025-03-01 11:36:00,0 -2025-03-01 11:37:00,0 -2025-03-01 11:38:00,0 -2025-03-01 11:39:00,0 -2025-03-01 11:40:00,0 -2025-03-01 11:41:00,0 -2025-03-01 11:42:00,0 -2025-03-01 11:43:00,0 -2025-03-01 11:44:00,0 -2025-03-01 11:45:00,0 -2025-03-01 11:46:00,0 -2025-03-01 11:47:00,0 -2025-03-01 11:48:00,0 -2025-03-01 11:49:00,0 -2025-03-01 11:50:00,0 -2025-03-01 11:51:00,0 -2025-03-01 11:52:00,0 -2025-03-01 11:53:00,0 -2025-03-01 11:54:00,0 -2025-03-01 11:55:00,0 -2025-03-01 11:56:00,0 -2025-03-01 11:57:00,0 -2025-03-01 11:58:00,0 -2025-03-01 11:59:00,0 -2025-03-01 12:00:00,0 diff --git a/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb b/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb deleted file mode 100644 index 03b4af0f..00000000 --- a/examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb +++ /dev/null @@ -1,448 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Car wash queueing model\n", - "\n", - "[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/plugboard-dev/plugboard)\n", - "\n", - "This model uses event-driven components to model cars arriving and queueing at a carwash. We can use the model to understand how many cars are in the queue at a given moment.\n", - "\n", - "Cars arrive at the carwash and enter a queue to go into one of three washing machines. If any one of the machines is empty then the car at the front of the queue moves into it, otherwise they will wait in the queue. The washing process takes a random amount of time (configurable on each machine).\n", - "\n", - "
\n", - "

Note

\n", - "

\n", - " Install pandas and plotly to run this demo.\n", - "

\n", - "
" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime\n", - "import random\n", - "import typing as _t\n", - "\n", - "import pandas as pd\n", - "from pydantic import BaseModel\n", - "\n", - "from plugboard.connector import AsyncioConnector\n", - "from plugboard.component import Component, IOController\n", - "from plugboard.connector import AsyncioConnector, ConnectorBuilder\n", - "from plugboard.events import Event, EventConnectorBuilder\n", - "from plugboard.schemas import ComponentArgsDict, ConnectorSpec\n", - "from plugboard.process import LocalProcess\n", - "from plugboard.library import FileWriter" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Start be defining the events that are required for:\n", - "* A car arriving and entering the queue;\n", - "* A car moving from the queue into a washing machine;\n", - "* A car leaving the car-wash after washing is completed." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class CarData(BaseModel):\n", - " \"\"\"Data for a car at the carwash.\"\"\"\n", - "\n", - " car_id: int\n", - " machine_id: _t.Optional[int] = None\n", - " arrival_time: _t.Optional[datetime] = None\n", - " leave_time: _t.Optional[datetime] = None\n", - "\n", - "\n", - "class CarArrived(Event):\n", - " \"\"\"Event for when a car arrives at the carwash.\"\"\"\n", - "\n", - " type: _t.ClassVar[str] = \"car_arrived\"\n", - " data: CarData\n", - "\n", - "\n", - "class CarEntersWash(Event):\n", - " \"\"\"Event for when a car enters the wash.\"\"\"\n", - "\n", - " type: _t.ClassVar[str] = \"car_enters_wash\"\n", - " data: CarData\n", - "\n", - "\n", - "class CarLeavesWash(Event):\n", - " \"\"\"Event for when a car leaves the wash.\"\"\"\n", - "\n", - " type: _t.ClassVar[str] = \"car_leaves_wash\"\n", - " data: CarData" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The CSV file in`car-arrivals.csv` contains minute-by-minute data on the number of cars that arrive at the carwash. We need a component to read this data and publish an event for each car that arrives." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class CarArrivals(Component):\n", - " \"\"\"This component emits an event for each new car that arrives.\"\"\"\n", - "\n", - " io = IOController(outputs=[\"time_stamp\"], output_events=[CarArrived])\n", - "\n", - " def __init__(self, path: str, **kwargs: _t.Unpack[ComponentArgsDict]):\n", - " super().__init__(**kwargs)\n", - " self._path = path\n", - " self._car_id = 0\n", - "\n", - " async def init(self) -> None:\n", - " df = pd.read_csv(self._path, parse_dates=[\"time_stamp\"])\n", - " self._iterator = df.iterrows()\n", - "\n", - " async def step(self) -> None:\n", - " try:\n", - " _, row = next(self._iterator)\n", - " except StopIteration:\n", - " await self.io.close()\n", - " return\n", - " self.time_stamp = row[\"time_stamp\"]\n", - " # Emit an event for each car that arrives at this time\n", - " for _ in range(row[\"cars\"]):\n", - " self._car_id += 1\n", - " car = CarData(car_id=self._car_id, arrival_time=self.time_stamp)\n", - " self.io.queue_event(CarArrived(source=self.name, data=car))\n", - " self._logger.info(\"Car arrived\", car=car)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This next component implements a queue. It monitors the status of each washing machine, and publishes events when cars move from the queue to a machine." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class CarWashQueue(Component):\n", - " \"\"\"This component manages the queue of cars waiting for the wash.\"\"\"\n", - "\n", - " io = IOController(\n", - " inputs=[\"time_stamp\"],\n", - " outputs=[\"queue_length\"],\n", - " input_events=[CarArrived, CarLeavesWash],\n", - " output_events=[CarEntersWash],\n", - " )\n", - "\n", - " def __init__(self, n_machines: int = 3, **kwargs: _t.Unpack[ComponentArgsDict]):\n", - " super().__init__(**kwargs)\n", - " self._queue = []\n", - " # Use this dictionary to keep track of which machines are available\n", - " self._machine_status = {idx: False for idx in range(1, n_machines + 1)}\n", - "\n", - " async def step(self) -> None:\n", - " # Check if there are any cars waiting for the wash\n", - " while self._queue:\n", - " # Check if any of the machines are available\n", - " for idx, machine in self._machine_status.items():\n", - " if not machine:\n", - " # Machine is not busy - send car to wash\n", - " car = self._queue.pop(0)\n", - " car.machine_id = idx\n", - " self.io.queue_event(CarEntersWash(source=self.name, data=car))\n", - " self._logger.info(\"Car enters wash\", car=car)\n", - " # Mark machine as busy\n", - " self._machine_status[idx] = True\n", - " break\n", - " else:\n", - " # No machines available\n", - " break\n", - " self.queue_length = len(self._queue)\n", - "\n", - " @CarArrived.handler\n", - " async def car_arrived(self, event: CarArrived) -> None:\n", - " # Add the car to the queue\n", - " self._queue.append(event.data)\n", - "\n", - " @CarLeavesWash.handler\n", - " async def car_leaves_wash(self, event: CarLeavesWash) -> None:\n", - " # Mark the machine as available\n", - " self._machine_status[event.data.machine_id] = False" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next model the washing machines. Each one will listen for an event telling it to start washing a car." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class CarWashMachine(Component):\n", - " \"\"\"This component simulates the car wash machine.\"\"\"\n", - "\n", - " io = IOController(\n", - " inputs=[\"time_stamp\"],\n", - " outputs=[\"busy\"],\n", - " input_events=[CarEntersWash],\n", - " output_events=[CarLeavesWash],\n", - " )\n", - "\n", - " def __init__(\n", - " self,\n", - " machine_id: int,\n", - " wash_time_range: tuple[int, int] = (4, 6),\n", - " **kwargs: _t.Unpack[ComponentArgsDict],\n", - " ):\n", - " super().__init__(**kwargs)\n", - " self._machine_id = machine_id\n", - " self._wash_time_range = wash_time_range\n", - " self._time_remaining = 0\n", - " self._car = None\n", - " self.busy = False\n", - "\n", - " @CarEntersWash.handler\n", - " async def car_enters_wash(self, event: CarEntersWash) -> None:\n", - " # Check if car is going into this machine\n", - " if event.data.machine_id != self._machine_id:\n", - " return\n", - " # Start the wash\n", - " self._time_remaining = random.randint(*self._wash_time_range)\n", - " self.busy = True\n", - " self._car = event.data\n", - " self._logger.info(\"Car enters wash\", car=self._car)\n", - "\n", - " async def step(self) -> None:\n", - " # Check if the wash is complete\n", - " if self.busy:\n", - " self._time_remaining -= 1\n", - " if self._time_remaining <= 0:\n", - " self.busy = False\n", - " self._car.leave_time = self.time_stamp\n", - " self.io.queue_event(CarLeavesWash(source=self.name, data=self._car))\n", - " self._logger.info(\"Car leaves wash\", car=self._car)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use this component to capture data as each car leaves the washing machines, so that we can record how long the process took for each vehicle." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class CaptureData(Component):\n", - " \"\"\"This component captures the data for each car that leaves the wash and saves to CSV.\"\"\"\n", - "\n", - " io = IOController(inputs=[\"time_stamp\"], input_events=[CarLeavesWash])\n", - "\n", - " def __init__(self, path: str, **kwargs: _t.Unpack[ComponentArgsDict]):\n", - " super().__init__(**kwargs)\n", - " self._path = path\n", - " self._data = []\n", - "\n", - " @CarLeavesWash.handler\n", - " async def car_leaves_wash(self, event: CarLeavesWash) -> None:\n", - " car = event.data\n", - " car.leave_time = self.time_stamp\n", - " self._data.append(car.model_dump())\n", - " self._logger.info(\"Car data captured\", car=car)\n", - "\n", - " async def destroy(self):\n", - " pd.DataFrame(self._data).to_csv(self._path, index=False)\n", - " return await super().destroy()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When building the `Process` to run the model, we need to create connectors both for the normal data inputs/outputs, and also for the events as shown below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "components = [\n", - " CarArrivals(name=\"car-arrivals\", path=\"car-arrivals.csv\"),\n", - " CarWashQueue(name=\"car-wash-queue\", n_machines=3),\n", - " CarWashMachine(name=\"car-wash-machine-1\", machine_id=1, wash_time_range=(3, 5)),\n", - " CarWashMachine(name=\"car-wash-machine-2\", machine_id=2, wash_time_range=(3, 5)),\n", - " # The third machine takes longer to wash cars\n", - " CarWashMachine(name=\"car-wash-machine-3\", machine_id=3, wash_time_range=(5, 9)),\n", - " CaptureData(name=\"capture-data\", path=\"car-wash-data.csv\"),\n", - " FileWriter(\n", - " name=\"write-data\",\n", - " path=\"queue-length.csv\",\n", - " field_names=[\n", - " \"time_stamp\",\n", - " \"queue_length\",\n", - " \"machine_1_busy\",\n", - " \"machine_2_busy\",\n", - " \"machine_3_busy\",\n", - " ],\n", - " ),\n", - "]\n", - "connect = lambda in_, out_: AsyncioConnector(spec=ConnectorSpec(source=in_, target=out_))\n", - "connectors = [\n", - " connect(\"car-arrivals.time_stamp\", \"write-data.time_stamp\"),\n", - " connect(\"car-arrivals.time_stamp\", \"capture-data.time_stamp\"),\n", - " connect(\"car-arrivals.time_stamp\", \"car-wash-queue.time_stamp\"),\n", - " connect(\"car-arrivals.time_stamp\", \"car-wash-machine-1.time_stamp\"),\n", - " connect(\"car-arrivals.time_stamp\", \"car-wash-machine-2.time_stamp\"),\n", - " connect(\"car-arrivals.time_stamp\", \"car-wash-machine-3.time_stamp\"),\n", - " connect(\"car-wash-queue.queue_length\", \"write-data.queue_length\"),\n", - " connect(\"car-wash-machine-1.busy\", \"write-data.machine_1_busy\"),\n", - " connect(\"car-wash-machine-2.busy\", \"write-data.machine_2_busy\"),\n", - " connect(\"car-wash-machine-3.busy\", \"write-data.machine_3_busy\"),\n", - "]\n", - "connector_builder = ConnectorBuilder(connector_cls=AsyncioConnector)\n", - "event_connector_builder = EventConnectorBuilder(connector_builder=connector_builder)\n", - "event_connectors = list(event_connector_builder.build(components).values())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "process = LocalProcess(\n", - " components=components,\n", - " connectors=connectors + event_connectors,\n", - ")\n", - "async with process:\n", - " await process.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Results analysis\n", - "\n", - "Now load the output CSV files and analyse the data. Try going back and adjusting parameters to see their effect." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df_queue = pd.read_csv(\"queue-length.csv\")\n", - "df_data = pd.read_csv(\"car-wash-data.csv\", parse_dates=[\"arrival_time\", \"leave_time\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Print the average utilisation of each machine\n", - "df_queue[[\"machine_1_busy\", \"machine_2_busy\", \"machine_3_busy\"]].mean().to_frame(\n", - " \"utilisation_percent\"\n", - ") * 100" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Plot the queue length over time\n", - "try:\n", - " fig = df_queue.plot(\n", - " backend=\"plotly\",\n", - " x=\"time_stamp\",\n", - " y=[\"queue_length\"],\n", - " title=\"Queue length at car wash\",\n", - " labels={\"index\": \"Time\", \"value\": \"Number of cars\"},\n", - " )\n", - "except (ImportError, ValueError):\n", - " print(\"Please install plotly to run this cell.\")\n", - " fig = None\n", - "fig" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Plot a histogram of the time spent at the car wash for each vehicle\n", - "df_data = df_data.assign(\n", - " total_time=(df_data[\"leave_time\"] - df_data[\"arrival_time\"]).dt.total_seconds() / 60\n", - ")\n", - "try:\n", - " fig = df_data.plot(\n", - " backend=\"plotly\",\n", - " kind=\"hist\",\n", - " x=\"total_time\",\n", - " title=\"Time spent at car wash\",\n", - " labels={\"index\": \"Time\", \"value\": \"Number of cars\"},\n", - " )\n", - "except (ImportError, ValueError):\n", - " print(\"Please install plotly to run this cell.\")\n", - " fig = None\n", - "fig" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 80fcc85dfa9f55b630b93251d591aad312fad22d Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Tue, 23 Sep 2025 20:21:03 +0100 Subject: [PATCH 30/32] Update instructions --- .github/instructions/models.instructions.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/instructions/models.instructions.md b/.github/instructions/models.instructions.md index d8ac9432..a2a3cf8f 100644 --- a/.github/instructions/models.instructions.md +++ b/.github/instructions/models.instructions.md @@ -104,8 +104,7 @@ You can help users to implement event-driven models using Plugboard's event syst Examples of where you might want to use events include: * A component that monitors a data stream and emits an event when a threshold is crossed; * A component that listens for events and triggers actions in response, e.g. sending an alert; -* A trading algorithm that uses events to signal buy/sell decisions; -* A simulation of a taxi rank where events are used to signal the arrival and departure of taxis and passengers. +* A trading algorithm that uses events to signal buy/sell decisions. Events must be defined by inheriting from the `plugboard.events.Event` class. Each event class should define the data it carries using a Pydantic `BaseModel`. For example: From 8becf58fcbb9a7b7b74323db19f44afdd0666e8d Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Wed, 1 Oct 2025 20:05:23 +0100 Subject: [PATCH 31/32] Swap out for momentum signal example --- examples/demos/finance/.meta.yml | 2 + .../finance/001_momentum_signal/.gitignore | 1 + .../finance/001_momentum_signal/.meta.yml | 2 + .../001_momentum_signal/momentum-signal.ipynb | 616 ++++++++++++++++++ mkdocs.yaml | 4 +- 5 files changed, 624 insertions(+), 1 deletion(-) create mode 100644 examples/demos/finance/.meta.yml create mode 100644 examples/demos/finance/001_momentum_signal/.gitignore create mode 100644 examples/demos/finance/001_momentum_signal/.meta.yml create mode 100644 examples/demos/finance/001_momentum_signal/momentum-signal.ipynb diff --git a/examples/demos/finance/.meta.yml b/examples/demos/finance/.meta.yml new file mode 100644 index 00000000..f53b3e9a --- /dev/null +++ b/examples/demos/finance/.meta.yml @@ -0,0 +1,2 @@ +tags: + - finance \ No newline at end of file diff --git a/examples/demos/finance/001_momentum_signal/.gitignore b/examples/demos/finance/001_momentum_signal/.gitignore new file mode 100644 index 00000000..16f2dc5f --- /dev/null +++ b/examples/demos/finance/001_momentum_signal/.gitignore @@ -0,0 +1 @@ +*.csv \ No newline at end of file diff --git a/examples/demos/finance/001_momentum_signal/.meta.yml b/examples/demos/finance/001_momentum_signal/.meta.yml new file mode 100644 index 00000000..9e58b17e --- /dev/null +++ b/examples/demos/finance/001_momentum_signal/.meta.yml @@ -0,0 +1,2 @@ +tags: + - events \ No newline at end of file diff --git a/examples/demos/finance/001_momentum_signal/momentum-signal.ipynb b/examples/demos/finance/001_momentum_signal/momentum-signal.ipynb new file mode 100644 index 00000000..db33182d --- /dev/null +++ b/examples/demos/finance/001_momentum_signal/momentum-signal.ipynb @@ -0,0 +1,616 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Momentum trading\n", + "\n", + "[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/plugboard-dev/plugboard)\n", + "\n", + "This notebook implements a simple momentum strategy on the S&P 500 using Plugboard’s event-driven components:\n", + "\n", + "- Data source: streams S&P 500 prices from Yahoo! Finance;\n", + "- Indicators: three pairs of exponential moving averages (fast/medium/slow);\n", + "- Signals: compare EMAs to create buy/sell signals;\n", + "- Events: combine three signals into a TradeEvent (weak/strong buy/sell);\n", + "- Sink: write trades to a CSV file for inspection.\n", + "\n", + "You can run the process, then visualize trades on a price chart." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "Here's a diagram to illustrate the whole process:\n", + "\n", + "![](https://mermaid.ink/img/pako:eNrlll9LwzAUxb9KyGNZBftYZDhkPk0QFUQQJGtu20DaSJpuQul3d0lY7Zyb_ZP64uvNObe_3JyEVjgSFHCIYy62UUqkQquH1xwhLggFeV2hIiXvECIpypwCnSFO1sBD9EJSIe4li2BllFdrOfc86_I8VCPfnyPIiB-TQvmXpxst7xbW-yXWfvcMQR-GwDlDBpSVWedJ7OVTcQT9ONzPo9gFrvM0rHgahqAPQzOHEdE2ny9YYkqn3TdSFIXYgHxkSU647bS3HWH0T7drjBEBb1BscQCMNf6AMyTnU-AMjHuDoksDQLTtCKN_4l1iOIicwVFyd599kiQSEqLEmffgSSttr0Ujt12_N2lDOjj9vwEdeSjTQzp5sP85oam_wQby1s1J4eOw51Kva5yLhmcrmYLfWIzvlnF4Nuo2ivUfYEyxxxZy113iGc5AZoRRHFZYpZDpP1kKMSm5wnX9CenNtM8=)" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Imports and helpers\n", + "\n", + "We import Plugboard’s core building blocks and define a small helper to create connectors:\n", + "\n", + "- Components expose named inputs/outputs and can emit/receive events.\n", + "- Connectors move values between component fields.\n", + "- Event connectors route declared events between publishers and subscribers automatically." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import datetime as dt\n", + "import typing as _t\n", + "\n", + "from plugboard.connector import AsyncioConnector, ConnectorBuilder\n", + "from plugboard.events import EventConnectorBuilder\n", + "from plugboard.process import LocalProcess\n", + "from plugboard.schemas import ConnectorSpec\n", + "\n", + "import pandas as pd\n", + "import plotly.graph_objects as go\n", + "\n", + "import yfinance as yf\n", + "from pydantic import BaseModel\n", + "from plugboard.events import Event\n", + "\n", + "from plugboard.component import Component, IOController as IO\n", + "from plugboard.schemas import ComponentArgsDict\n", + "\n", + "# Helper to create field connectors\n", + "connect = lambda src, tgt: AsyncioConnector(spec=ConnectorSpec(source=src, target=tgt))" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "### Components: data and indicators\n", + "\n", + "- YahooPriceLoader streams price and timestamp row-by-row from Yahoo Finance for ^GSPC.\n", + "- EMA consumes `price` and emits an exponentially weighted moving average as `ema`.\n", + "\n", + "Components declare their I/O via `IOController`, giving Plugboard enough metadata to wire processes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "def _ensure_dt(val: _t.Any) -> dt.datetime:\n", + " if isinstance(val, dt.datetime):\n", + " return val\n", + " if isinstance(val, dt.date):\n", + " return dt.datetime.combine(val, dt.time())\n", + " return pd.to_datetime(val).to_pydatetime()\n", + "\n", + "\n", + "class YahooPriceLoader(Component):\n", + " \"\"\"Loads historical prices for a symbol from Yahoo Finance and streams them row by row.\n", + "\n", + " Outputs per step:\n", + " price: float - adjusted close price (or close if adj not present)\n", + " timestamp: datetime\n", + " \"\"\"\n", + "\n", + " io = IO(outputs=[\"price\", \"timestamp\"]) # stream out prices\n", + "\n", + " def __init__(\n", + " self,\n", + " symbol: str = \"^GSPC\",\n", + " period: str | None = None,\n", + " start: str | dt.date | None = None,\n", + " end: str | dt.date | None = None,\n", + " interval: str = \"1d\",\n", + " limit: int | None = None,\n", + " **kwargs: _t.Unpack[ComponentArgsDict],\n", + " ) -> None:\n", + " super().__init__(**kwargs)\n", + " self.symbol = symbol\n", + " self.period = period\n", + " self.start = start\n", + " self.end = end\n", + " self.interval = interval\n", + " self.limit = limit\n", + " self._data: pd.DataFrame | None = None\n", + " self._iter = 0\n", + "\n", + " async def _ensure_data(self) -> None:\n", + " if self._data is not None:\n", + " return\n", + " if yf is None: # pragma: no cover - runtime safeguard\n", + " raise RuntimeError(\"yfinance not installed. Please 'pip install yfinance'.\")\n", + " df = yf.download(\n", + " self.symbol,\n", + " period=self.period,\n", + " start=self.start,\n", + " end=self.end,\n", + " interval=self.interval,\n", + " progress=False,\n", + " )\n", + " if df.empty:\n", + " raise RuntimeError(f\"No data returned for symbol {self.symbol}\")\n", + " # Prefer Adj Close if exists\n", + " if \"Adj Close\" in df.columns:\n", + " df.rename(columns={\"Adj Close\": \"AdjClose\"}, inplace=True)\n", + " price_col = \"AdjClose\"\n", + " elif \"Close\" in df.columns:\n", + " price_col = \"Close\"\n", + " else:\n", + " price_col = df.columns[0]\n", + " df = df[[price_col]].rename(columns={price_col: \"price\"})\n", + " df.index.name = \"timestamp\"\n", + " df.reset_index(inplace=True)\n", + " if self.limit is not None:\n", + " df = df.head(self.limit)\n", + " # Remove column multi-index if present\n", + " self._data = df.droplevel(1, axis=\"columns\")\n", + "\n", + " @property\n", + " def df(self) -> pd.DataFrame:\n", + " \"\"\"The full DataFrame of loaded price data.\"\"\"\n", + " if self._data is None:\n", + " raise RuntimeError(\"Data not yet loaded. Call step() first.\")\n", + " return self._data\n", + "\n", + " async def step(self) -> None: # noqa: D401\n", + " await self._ensure_data()\n", + " if self._iter >= len(self._data):\n", + " await self.io.close()\n", + " return\n", + " row = self._data.iloc[self._iter]\n", + " self.price = float(row[\"price\"])\n", + " ts = row[\"timestamp\"]\n", + " self.timestamp = _ensure_dt(ts)\n", + " self._iter += 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "class EMA(Component):\n", + " \"\"\"Computes an exponential moving average of an input price stream.\n", + "\n", + " Inputs:\n", + " price: float\n", + " Outputs:\n", + " ema: float\n", + " \"\"\"\n", + "\n", + " io = IO(inputs=[\"price\"], outputs=[\"ema\"])\n", + "\n", + " def __init__(\n", + " self,\n", + " alpha: float | None = None,\n", + " span: int | None = None,\n", + " **kwargs: _t.Unpack[ComponentArgsDict],\n", + " ) -> None:\n", + " super().__init__(**kwargs)\n", + " if alpha is None and span is None:\n", + " raise ValueError(\"Provide either alpha or span\")\n", + " if alpha is not None and not (0 < alpha <= 1):\n", + " raise ValueError(\"alpha must be (0,1]\")\n", + " self.alpha = alpha if alpha is not None else 2 / (span + 1)\n", + " self._ema: float | None = None\n", + "\n", + " async def step(self) -> None:\n", + " price = float(self.price)\n", + " if self._ema is None:\n", + " self._ema = price\n", + " else:\n", + " self._ema = self.alpha * price + (1 - self.alpha) * self._ema\n", + " self.ema = self._ema" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### Components: signals and events\n", + "\n", + "- CrossoverSignal reads two EMAs (`fast`, `slow`) and emits a directional `signal`.\n", + "- TradeSignalAggregator takes three signals plus the current `price` and `timestamp`, and emits a `TradeEvent` with direction/size/strength.\n", + "- TradeEventFileWriter subscribes to `TradeEvent` and appends a CSV row per event." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "class CrossoverSignal(Component):\n", + " \"\"\"Generates buy/sell/hold signal from two moving averages.\n", + "\n", + " Inputs:\n", + " fast: float\n", + " slow: float\n", + " Outputs:\n", + " signal: int (1=buy, -1=sell)\n", + " \"\"\"\n", + "\n", + " io = IO(inputs=[\"fast\", \"slow\"], outputs=[\"signal\"])\n", + "\n", + " def __init__(self, **kwargs: _t.Unpack[ComponentArgsDict]) -> None:\n", + " super().__init__(**kwargs)\n", + "\n", + " async def step(self) -> None:\n", + " fast = float(self.fast)\n", + " slow = float(self.slow)\n", + " self.signal = 1 if fast >= slow else -1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "class TradeData(BaseModel):\n", + " \"\"\"Data for a trade decision.\n", + "\n", + " side: \"buy\" or \"sell\"\n", + " size: number of units\n", + " price: execution price\n", + " timestamp: event time\n", + " strength: \"strong\" | \"weak\"\n", + " count_buy: number of buy signals\n", + " count_sell: number of sell signals\n", + " \"\"\"\n", + "\n", + " side: _t.Literal[\"buy\", \"sell\"]\n", + " size: int\n", + " price: float\n", + " timestamp: dt.datetime\n", + " strength: _t.Literal[\"strong\", \"weak\"]\n", + " count_buy: int\n", + " count_sell: int\n", + "\n", + "\n", + "class TradeEvent(Event):\n", + " \"\"\"Event emitted when strategy decides to trade.\"\"\"\n", + "\n", + " type: _t.ClassVar[str] = \"trade_event\"\n", + " data: TradeData\n", + "\n", + "\n", + "class TradeSignalAggregator(Component):\n", + " \"\"\"Aggregates three crossover signals into trade events.\n", + "\n", + " Inputs:\n", + " s1, s2, s3 (int signals: 1 buy, -1 sell, 0 hold)\n", + " price (float)\n", + " timestamp (datetime)\n", + " Output events:\n", + " TradeEvent\n", + " Logic:\n", + " strong buy = 3 buys -> size 2\n", + " weak buy = 2 buys 1 sell -> size 1\n", + " strong sell = 3 sells -> size 2\n", + " weak sell = 2 sells 1 buy -> size 1\n", + " \"\"\"\n", + "\n", + " io = IO(\n", + " inputs=[\"s1\", \"s2\", \"s3\", \"price\", \"timestamp\"],\n", + " output_events=[TradeEvent],\n", + " )\n", + "\n", + " def __init__(self, **kwargs: _t.Unpack[ComponentArgsDict]) -> None:\n", + " super().__init__(**kwargs)\n", + " self._previous_signal: int | None = None\n", + "\n", + " async def step(self) -> None:\n", + " signals = [int(self.s1), int(self.s2), int(self.s3)]\n", + " count_buy = sum(1 for s in signals if s == 1)\n", + " count_sell = sum(1 for s in signals if s == -1)\n", + " net_signal = count_buy - count_sell\n", + " if net_signal >= 2:\n", + " decision, strength, size = \"buy\", \"strong\", 2\n", + " elif net_signal == 1:\n", + " decision, strength, size = \"buy\", \"weak\", 1\n", + " elif net_signal <= -2:\n", + " decision, strength, size = \"sell\", \"strong\", 2\n", + " elif net_signal == -1:\n", + " decision, strength, size = \"sell\", \"weak\", 1\n", + "\n", + " if net_signal != self._previous_signal:\n", + " self._logger.info(\n", + " f\"Emitting trade decision\", decision=decision, size=size, strength=strength\n", + " )\n", + " trade = TradeEvent(\n", + " source=self.name,\n", + " data=TradeData(\n", + " side=decision,\n", + " size=size,\n", + " price=float(self.price),\n", + " timestamp=_ensure_dt(self.timestamp),\n", + " strength=strength,\n", + " count_buy=count_buy,\n", + " count_sell=count_sell,\n", + " ),\n", + " )\n", + " self.io.queue_event(trade)\n", + " self._previous_signal = net_signal" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "class TradeEventFileWriter(Component):\n", + " \"\"\"Consumes trade events and writes them to a CSV file (append mode).\"\"\"\n", + "\n", + " io = IO(input_events=[TradeEvent])\n", + "\n", + " def __init__(self, path: str = \"trades.csv\", **kwargs: _t.Unpack[ComponentArgsDict]) -> None:\n", + " super().__init__(**kwargs)\n", + " self.path = path\n", + " # Write header\n", + " with open(self.path, \"w\", encoding=\"utf-8\") as f:\n", + " f.write(\"timestamp,side,size,price,strength,count_buy,count_sell,source\\n\")\n", + "\n", + " @TradeEvent.handler\n", + " async def handle_trade(self, event: TradeEvent) -> None: # noqa: D401\n", + " d = event.data\n", + " with open(self.path, \"a\", encoding=\"utf-8\") as f:\n", + " f.write(\n", + " f\"{d.timestamp.isoformat()},{d.side},{d.size},{d.price:.4f},{d.strength},{d.count_buy},{d.count_sell},{event.source}\\n\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### Wire the process\n", + "\n", + "Here we:\n", + "\n", + "- Instantiate the source, indicator, signal, aggregator, and writer components.\n", + "- Connect fields with `AsyncioConnector` (price→EMAs, EMAs→signals, signals→aggregator).\n", + "- Build event connectors so `TradeEvent` flows from the aggregator to the file writer.\n", + "- Create a `LocalProcess` to run everything in-process." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "# Build components\n", + "price_loader = YahooPriceLoader(name=\"loader\", period=\"10y\", interval=\"1d\")\n", + "\n", + "# Three EMAs with different speeds (adjust spans as desired)\n", + "ema_fast_1 = EMA(name=\"ema-fast-1\", span=8)\n", + "ema_fast_2 = EMA(name=\"ema-fast-2\", span=15)\n", + "ema_medium_1 = EMA(name=\"ema-medium-1\", span=30)\n", + "ema_medium_2 = EMA(name=\"ema-medium-2\", span=50)\n", + "ema_slow_1 = EMA(name=\"ema-slow-1\", span=80)\n", + "ema_slow_2 = EMA(name=\"ema-slow-2\", span=150)\n", + "\n", + "# Signals from different pairings\n", + "sig_fast = CrossoverSignal(name=\"sig-fast\")\n", + "sig_medium = CrossoverSignal(name=\"sig-medium\")\n", + "sig_slow = CrossoverSignal(name=\"sig-slow\")\n", + "\n", + "# Aggregator producing trade events\n", + "aggregator = TradeSignalAggregator(name=\"trade-aggregator\")\n", + "trade_writer = TradeEventFileWriter(name=\"trade-writer\", path=\"trades.csv\")\n", + "\n", + "components = [\n", + " price_loader,\n", + " ema_fast_1,\n", + " ema_fast_2,\n", + " ema_medium_1,\n", + " ema_medium_2,\n", + " ema_slow_1,\n", + " ema_slow_2,\n", + " sig_fast,\n", + " sig_medium,\n", + " sig_slow,\n", + " aggregator,\n", + " trade_writer,\n", + "]\n", + "\n", + "# Field connectors\n", + "connectors = [\n", + " # Price to EMAs\n", + " connect(\"loader.price\", \"ema-fast-1.price\"),\n", + " connect(\"loader.price\", \"ema-fast-2.price\"),\n", + " connect(\"loader.price\", \"ema-medium-1.price\"),\n", + " connect(\"loader.price\", \"ema-medium-2.price\"),\n", + " connect(\"loader.price\", \"ema-slow-1.price\"),\n", + " connect(\"loader.price\", \"ema-slow-2.price\"),\n", + " # Convert the three pairs of EMAs into signals\n", + " connect(\"ema-fast-1.ema\", \"sig-fast.fast\"),\n", + " connect(\"ema-fast-2.ema\", \"sig-fast.slow\"),\n", + " connect(\"ema-medium-1.ema\", \"sig-medium.fast\"),\n", + " connect(\"ema-medium-2.ema\", \"sig-medium.slow\"),\n", + " connect(\"ema-slow-1.ema\", \"sig-slow.fast\"),\n", + " connect(\"ema-slow-2.ema\", \"sig-slow.slow\"),\n", + " # Signals + price + timestamp into aggregator\n", + " connect(\"sig-fast.signal\", \"trade-aggregator.s1\"),\n", + " connect(\"sig-medium.signal\", \"trade-aggregator.s2\"),\n", + " connect(\"sig-slow.signal\", \"trade-aggregator.s3\"),\n", + " connect(\"loader.price\", \"trade-aggregator.price\"),\n", + " connect(\"loader.timestamp\", \"trade-aggregator.timestamp\"),\n", + "]\n", + "\n", + "# Event connectors\n", + "builder = ConnectorBuilder(connector_cls=AsyncioConnector)\n", + "event_builder = EventConnectorBuilder(connector_builder=builder)\n", + "event_connectors = list(event_builder.build(components).values())\n", + "\n", + "process = LocalProcess(components=components, connectors=connectors + event_connectors)" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "### Run the process\n", + "\n", + "Running the process iterates over the price history, updates indicators, produces signals, emits trade events, and writes them to `trades.csv`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "async with process:\n", + " await process.run()" + ] + }, + { + "cell_type": "markdown", + "id": "15", + "metadata": {}, + "source": [ + "### Visualize trades from CSV\n", + "\n", + "After the run, `trades.csv` contains one row per `TradeEvent`. We overlay buy/sell markers on the price series to see where the strategy acted." + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "df_prices = price_loader.df\n", + "df_trades = pd.read_csv(\"trades.csv\", parse_dates=[\"timestamp\"])\n", + "\n", + "traces = [\n", + " go.Scatter(x=df_prices[\"timestamp\"], y=df_prices[\"price\"], mode=\"lines\", name=\"Price\"),\n", + " go.Scatter(\n", + " x=df_trades[df_trades[\"side\"] == \"buy\"][\"timestamp\"],\n", + " y=df_trades[df_trades[\"side\"] == \"buy\"][\"price\"],\n", + " mode=\"markers\",\n", + " name=\"Buy\",\n", + " marker=dict(\n", + " color=\"green\",\n", + " symbol=\"triangle-up\",\n", + " size=df_trades[df_trades[\"side\"] == \"buy\"][\"strength\"].map(\n", + " lambda x: 18 if x == \"strong\" else 12\n", + " ),\n", + " ),\n", + " ),\n", + " go.Scatter(\n", + " x=df_trades[df_trades[\"side\"] == \"sell\"][\"timestamp\"],\n", + " y=df_trades[df_trades[\"side\"] == \"sell\"][\"price\"],\n", + " mode=\"markers\",\n", + " name=\"Sell\",\n", + " marker=dict(\n", + " color=\"red\",\n", + " symbol=\"triangle-down\",\n", + " size=df_trades[df_trades[\"side\"] == \"sell\"][\"strength\"].map(\n", + " lambda x: 18 if x == \"strong\" else 12\n", + " ),\n", + " ),\n", + " ),\n", + "]\n", + "fig = go.Figure(data=traces)\n", + "fig" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "### Visualize the process diagram\n", + "\n", + "We can render a Mermaid diagram of the Plugboard process, showing components, fields, and event flows. This helps debug and document the model wiring." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "# Visualise the process\n", + "from plugboard.diagram import MermaidDiagram\n", + "\n", + "diagram_md = MermaidDiagram.from_process(process)\n", + "diagram_md.url" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "### Next steps\n", + "\n", + "Potential enhancements to this example could include:\n", + "\n", + "- Adding a component to track PnL from the trades;\n", + "- Using `plugboard.tune` to choose the moving averages to optimise PnL." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mkdocs.yaml b/mkdocs.yaml index c521cf11..f754c2dd 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -119,12 +119,14 @@ nav: - Demos: - Fundamentals: - Simple model: examples/demos/fundamentals/001_simple_model/simple-model.ipynb - - Car wash: examples/demos/fundamentals/003_car_wash_queue/car-wash.ipynb + - Production line: examples/demos/fundamentals/002_production_line_optimisation/production-line.ipynb - LLMs: - Data filtering: examples/demos/llm/001_data_filter/llm-filtering.ipynb - Websocket streaming: examples/demos/llm/002_bluesky_websocket/bluesky-websocket.ipynb - Physics-based models: - Hot water tank: examples/demos/physics-models/001-hot-water-tank/hot-water-tank.ipynb + - Finance: + - Momentum trading signal: examples/demos/finance/001_momentum_signal/momentum-signal.ipynb - API Reference: - component: api/component/component.md - connector: api/connector/connector.md From 1c67def6cb46d0611d898259bc0a80a8e87cac0f Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Wed, 1 Oct 2025 20:08:43 +0100 Subject: [PATCH 32/32] Update text --- docs/examples/tutorials/event-driven-models.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/tutorials/event-driven-models.md b/docs/examples/tutorials/event-driven-models.md index d3fe681d..b0b99843 100644 --- a/docs/examples/tutorials/event-driven-models.md +++ b/docs/examples/tutorials/event-driven-models.md @@ -4,7 +4,7 @@ tags: --- So far everything we have built in Plugboard has been a **discrete-time model**. This means that the whole model advances step-wise, i.e. `step` gets called on each [`Component`][plugboard.component.Component], calculating all of their outputs before advancing the simulation on. -In this tutorial we're going to introduce an **event-driven model**, where data can be passed around between components based on triggers that you can define. Event-based models can be useful in a variety of scenarios, for example when modelling parts moving around a production line, or passengers arriving at a transport hub. +In this tutorial we're going to introduce an **event-driven model**, where data can be passed around between components based on triggers that you can define. Event-based models can be useful in a variety of scenarios, for example when modelling parts moving around a production line, or to trigger expensive computation only when certain conditions are met in the model. ## Event-based model