Merge branch 'refs/heads/main' into feat/async-ack

# Conflicts:
#	Cargo.lock

Author: chenquan

Cargo.lock

@@ -1,5 +1,5 @@
 [workspace.package]
-version = "0.4.0-rc1"
+version = "0.5.0"
 edition = "2021"
 description = "High-performance Rust flow processing engine"
 authors = ["chenquan <chenquan.dev@gmail.com>"]
@@ -46,7 +46,7 @@ flume = "=0.11"
 # Sql
 sqlx = { version = "0.8", features = ["mysql", "postgres", "runtime-tokio", "tls-native-tls"] }
 
-tempfile = "3.22.0"
+tempfile = "3.23.0"
 mockall = "0.12"
 arkflow-core = { path = "crates/arkflow-core" }
 arkflow-plugin = { path = "crates/arkflow-plugin" }

Cargo.toml

@@ -70,14 +70,14 @@ arkflow-core = { workspace = true }
 sqlx = { workspace = true }
 
 # Websocket
-tokio-tungstenite = { version = "0.27", features = ["native-tls"] }
+tokio-tungstenite = { version = "0.28", features = ["native-tls"] }
 
 # NATS
-async-nats = "0.43"
+async-nats = "0.44"
 
 # Pulsar
-pulsar = "6.3"
-rand = "0.8"
+pulsar = "6.4"
+rand = "0.9"
 
 
 # modbus

crates/arkflow-plugin/Cargo.toml

@@ -0,0 +1,219 @@
+---
+sidebar_position: 1
+logo: ./logo.svg
+---
+
+# Introduction
+
+ArkFlow is a high-performance Rust stream processing engine that provides powerful data stream processing capabilities, supporting various input/output sources and processors.
+
+:::tip
+Currently, ArkFlow is **stateless**, but it can still help you solve most data engineering problems. It implements transaction-based resilience and features backpressure. 
+
+Therefore, when connected to input and output sources that provide at-least-once semantics, it can guarantee at-least-once delivery without needing to retain messages in transit.
+
+In the future, we will gradually improve the functions of ArkFlow to enable it to have transactional and state management capabilities, so as to better meet various data processing needs.
+
+:::
+
+
+logo([Logo Usage Guidelines](./about-logo)）: 
+
+![ArkFlow logo](logo.svg)
+
+
+## Core Features
+
+- **High Performance**: Built on Rust and Tokio async runtime, delivering exceptional performance and low latency
+- **Multiple Data Sources**: Support for Kafka, MQTT, HTTP, files, and other input/output sources
+- **Powerful Processing**: Built-in SQL queries, JSON processing, Protobuf encoding/decoding, batch processing, and other processors
+- **Extensibility**: Modular design, easy to extend with new input, output, and processor components
+
+## Installation
+
+### Building from Source
+
+```bash
+# Clone repository
+git clone https://github.com/arkflow-rs/arkflow.git
+cd arkflow
+
+# Build project
+cargo build --release
+
+# Run tests
+cargo test
+```
+
+## Quick Start
+
+1. Create a configuration file `config.yaml`:
+
+```yaml
+logging:
+  level: info
+streams:
+  - input:
+      type: "generate"
+      context: '{ "timestamp": 1625000000000, "value": 10, "sensor": "temp_1" }'
+      interval: 1s
+      batch_size: 10
+    buffer:
+      type: "memory"
+      capacity: 10
+      timeout: 10s
+    pipeline:
+      thread_num: 4
+      processors:
+        - type: "json_to_arrow"
+        - type: "sql"
+          query: "SELECT * FROM flow WHERE value >= 10"
+
+    output:
+      type: "stdout"
+    error_output:
+      type: "stdout"
+```
+
+2. Run ArkFlow:
+
+```bash
+./target/release/arkflow --config config.yaml
+```
+
+## Configuration Guide
+
+ArkFlow uses YAML format configuration files and supports the following main configuration items:
+
+### Top-level Configuration
+
+```yaml
+logging:
+  level: info  # Log levels: debug, info, warn, error
+
+streams: # Stream definition list
+  - input:      # Input configuration
+    # ...
+    pipeline:   # Pipeline configuration
+    # ...
+    output:     # Output configuration
+    # ...
+    error_output: # Error output configuration
+    # ...
+    buffer:     # Buffer configuration
+    # ... 
+```
+
+
+### Input Components
+
+ArkFlow supports multiple input sources:
+
+- **Kafka**: Read data from Kafka topics
+- **MQTT**: Subscribe to messages from MQTT topics
+- **HTTP**: Receive data via HTTP
+- **File**: Reading data from files(Csv,Json, Parquet, Avro, Arrow) using SQL
+- **Generator**: Generate test data
+- **Database**: Query data from databases(MySQL, PostgreSQL, SQLite, Duckdb)
+- **Nats**: Subscribe to messages from Nats topics
+- **Redis**: Subscribe to messages from Redis channels or lists
+- **Websocket**: Subscribe to messages from WebSocket connections
+
+Example:
+
+```yaml
+input:
+  type: kafka
+  brokers:
+    - localhost:9092
+  topics:
+    - test-topic
+  consumer_group: test-group
+  client_id: arkflow
+  start_from_latest: true
+```
+
+### Processors
+
+ArkFlow provides multiple data processors:
+
+- **JSON**: JSON data processing and transformation
+- **SQL**: Process data using SQL queries
+- **Protobuf**: Protobuf encoding/decoding
+- **Batch Processing**: Process messages in batches
+- **Vrl**: Process data using [VRL](https://vector.dev/docs/reference/vrl/)
+
+Example:
+
+```yaml
+pipeline:
+  thread_num: 4
+  processors:
+    - type: json_to_arrow
+    - type: sql
+      query: "SELECT * FROM flow WHERE value >= 10"
+```
+
+### Output Components
+
+ArkFlow supports multiple output targets:
+
+- **Kafka**: Write data to Kafka topics
+- **MQTT**: Publish messages to MQTT topics
+- **HTTP**: Send data via HTTP
+- **Standard Output**: Output data to the console
+- **Drop**: Discard data
+- **Nats**: Publish messages to Nats topics
+
+Example:
+
+```yaml
+output:
+  type: kafka
+  brokers:
+    - localhost:9092
+  topic: 
+    type: value
+    value: test-topic
+  client_id: arkflow-producer
+```
+### Error Output Components
+ArkFlow supports multiple error output targets:
+- **Kafka**: Write error data to Kafka topics
+- **MQTT**: Publish error messages to MQTT topics
+- **HTTP**: Send error data via HTTP
+- **Standard Output**: Output error data to the console
+- **Drop**: Discard error data
+- **Nats**: Publish messages to Nats topics
+
+Example:
+
+```yaml
+error_output:
+  type: kafka
+  brokers:
+    - localhost:9092
+  topic: 
+    type: value
+    value: error-topic
+  client_id: error-arkflow-producer
+``` 
+
+
+### Buffer Components
+
+ArkFlow provides buffer capabilities to handle backpressure and temporary storage of messages:
+
+- **Memory Buffer**: Memory buffer, for high-throughput scenarios and window aggregation.
+- **Session Window**: The Session Window buffer component provides a session-based message grouping mechanism where messages are grouped based on activity gaps. It implements a session window that closes after a configurable period of inactivity.
+- **Sliding Window**: The Sliding Window buffer component provides a time-based windowing mechanism for processing message batches. It implements a sliding window algorithm with configurable window size, slide interval and slide size.
+- **Tumbling Window**: The Tumbling Window buffer component provides a fixed-size, non-overlapping windowing mechanism for processing message batches. It implements a tumbling window algorithm with configurable interval settings.
+
+Example:
+
+```yaml
+buffer:
+  type: memory
+  capacity: 10000  # Maximum number of messages to buffer
+  timeout: 10s  # Maximum time to buffer messages
+```

docs/versioned_docs/version-0.5.x/0-intro.md

@@ -0,0 +1,68 @@
+# ArkFlow Logo Usage Guidelines
+
+## 1. Introduction and Purpose
+
+Welcome to the ArkFlow Logo Usage Guidelines! The ArkFlow Logo (hereinafter referred to as the "Logo") is an important symbol of our project's identity and core values. It represents the innovation and reliability of the ArkFlow project, as well as our commitment to the open-source community.
+
+To help you use the ArkFlow Logo correctly, promote the ArkFlow project, and jointly maintain the integrity and reputation of the ArkFlow brand, we have established the following usage guidelines. We encourage community members, contributors, partners, and the media to use the ArkFlow Logo in compliance with these guidelines.
+
+## 2. Rights Statement
+
+The ArkFlow name and Logo are trademarks of the ArkFlow project team. We reserve all rights associated with this Logo, including but not limited to copyright and trademark rights.
+
+We are happy to grant you permission to use these marks provided you adhere to these guidelines.
+
+## 3. Logo Usage License
+
+We hope the ArkFlow Logo will be used widely and correctly to help promote the project.
+
+**Permitted Uses:**
+
+* On your website, blog, or social media posts to link to the official ArkFlow project address: [https://arkflow-rs.com/](https://arkflow-rs.com/).
+* In introductions, reviews, tutorials, news reports, or technical seminars to refer to or discuss the ArkFlow project, provided it does not create confusion or imply official endorsement or sponsorship.
+* In non-commercial promotional materials for your open-source project or organization built using ArkFlow, to indicate your project's association with ArkFlow (however, please note that you cannot use the ArkFlow Logo as the primary identifier for your project or organization, or imply that your project is an official ArkFlow product).
+* When showcasing your contributions to, support for, or your use of the ArkFlow project.
+* In community-organized non-commercial events or meetups related to ArkFlow.
+
+**Prohibited Uses (unless prior written permission is obtained from ArkFlow team):**
+
+* **Do not modify the Logo:** This includes, but is not limited to, changing the Logo's colors, proportions, fonts, design elements, or adding any other text or graphics to it. Always use the official Logo files provided by us.
+* **Do not use it as your own primary brand identifier:** Do not use the ArkFlow Logo as the primary logo or core brand element for your own products, services, company, domain name, social media account name, or project.
+* **Do not imply official endorsement:** Do not use the Logo in any way that suggests official endorsement, sponsorship, affiliation, or backing of your products, services, events, or commercial activities by ArkFlow or ArkFlow team.
+* **Do not use for commercial products or paid services:** Do not use the Logo directly on commercial products (e.g., printing the Logo on products for sale that require payment) or for the brand promotion of paid services. For commercial cooperation, please contact us.
+* **Do not use in negative or inappropriate contexts:** Do not use the Logo in connection with any defamatory, vulgar, illegal, misleading, controversial content, or in any situation that could harm the reputation of the ArkFlow project or the image of ArkFlow team.
+* **Do not register as a trademark:** You may not attempt to register the ArkFlow Logo or any similar marks as your own trademark.
+
+**Uses Requiring Prior Written Permission:**
+
+* Any commercial use, including but not limited to merchandise sales and promotion of paid services.
+* Any declaration of a formal partnership with a third party.
+* Any use of the Logo not explicitly permitted by these guidelines that you wish to undertake.
+
+## 4. Correct Usage Specifications for the Logo
+
+To ensure the consistency and recognizability of the Logo, please adhere to the following visual specifications:
+
+* **Official Logo Versions:** Always obtain the latest version of the Logo files from official ArkFlow channels. We typically provide SVG (vector) and PNG (bitmap) formats.
+* **Color:** Please use the officially designated colors for the Logo. If you need to use it on a special background, please consult us to see if there are applicable reversed (white) or monochrome versions. Do not change the colors yourself.
+* **Clear Space:** Sufficient empty space (clear space) should be maintained around the Logo to avoid overcrowding with other text, graphics, or borders.
+
+* **Avoid Improper Operations:**
+    * Do not stretch, compress, rotate, or tilt the Logo.
+    * Do not add strokes, shadows, gradients (unless part of the Logo's original design), or other visual effects to the Logo.
+    * Do not use the Logo on overly cluttered backgrounds or backgrounds that make the Logo difficult to discern.
+
+## 5. Disclaimer and Reservation of Rights
+
+These ArkFlow Logo Usage Guidelines may be updated from time to time. The latest version will be published on the official ArkFlow website; please check it regularly. Your continued use of the Logo after an update to the guidelines signifies your acceptance of the revised terms.
+ArkFlow team reserves the right to the final interpretation of the use of the ArkFlow Logo and has the right to demand correction or cessation of any Logo usage that we deem non-compliant with these guidelines or detrimental to the interests of the ArkFlow project.
+The permission granted to you to use the Logo is non-exclusive and may be revoked by ArkFlow team at any time at their sole discretion.
+
+## 6. Contact Information
+
+If you have any questions about the use of the ArkFlow Logo, wish to apply for uses not explicitly permitted by these guidelines, or have any commercial cooperation intentions, please contact Chen Quan via:
+Email: chenquan.dev@gmail.com
+
+We thank you for your support and attention to the ArkFlow project! Correct use of the ArkFlow Logo will help us jointly build a strong and respected brand.
+
+---

docs/versioned_docs/version-0.5.x/999-about-logo.md

@@ -0,0 +1,8 @@
+{
+  "label": "Inputs",
+  "link": {
+    "title": "Inputs",
+    "type": "generated-index",
+    "description": "Input components are responsible for consuming data from various sources such as Kafka, MQTT, HTTP, and Memory. Each input component has its own configuration options that can be customized according to your needs."
+  }
+}

docs/versioned_docs/version-0.5.x/components/0-inputs/_category_.json

@@ -0,0 +1,50 @@
+# Generate
+
+Generate is an input component that generates test data.
+
+## Configuration
+
+### **context**
+
+The context is a JSON object that will be used to generate the data. The JSON object will be serialized to bytes and sent as message content.
+
+type: `string`
+
+optional: `true`
+
+### **count**
+
+The total number of data points to generate. If not specified, the generator will run indefinitely until manually stopped.
+
+type: `integer`
+
+optional: `true`
+
+### **interval**
+
+The interval is the time between each data point.
+
+type: `string`
+
+example: `1ms`, `1s`, `1m`, `1h`, `1d`
+
+optional: `false`
+
+### **batch_size**
+
+The batch size is the number of data points to generate at each interval. If the remaining count is less than batch_size, only the remaining messages will be sent.
+
+type: `integer`
+
+optional: `false`
+
+## Examples
+
+```yaml
+  - input:
+      type: "generate"
+      context: '{ "timestamp": 1625000000000, "value": 10, "sensor": "temp_1" }'
+      interval: 1ms
+      batch_size: 1000
+      count: 10000  # Optional: generate 10000 messages in total
+```