Feature/real time sink patterns

.github/workflows/build.yml

@@ -99,7 +99,7 @@ jobs:
           path: "DataCaterer-1.0.0.exe"
           overwrite: true
 
-  linux:
+  linux-amd64:
     needs: build
     runs-on: [ubuntu-latest]
 
@@ -120,13 +120,65 @@ jobs:
         with:
           name: jars
           path: app/build/libs/
-      - name: Package jar as debian package
+      - name: Package jar as debian package (amd64)
         run: 'jpackage --main-jar data-caterer.jar "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-linux.cfg"'
       - name: List directory
         run: ls -lart
-      - name: Upload deb
+      - name: Upload deb (amd64)
         uses: actions/upload-artifact@v4
         with:
-          name: data-caterer-linux
+          name: data-caterer-linux-amd64
           path: "datacaterer_1.0.0_amd64.deb"
           overwrite: true
+
+  linux-arm64:
+    needs: build
+    runs-on: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Set version
+        run: echo "APP_VERSION=$(grep version gradle.properties | cut -d= -f2)" >> $GITHUB_ENV
+
+      # Set up QEMU for cross-platform builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Download fat jar
+        uses: actions/download-artifact@v4
+        with:
+          name: jars
+          path: app/build/libs/
+
+      # Use Docker container with ARM64 JDK for jpackage
+      - name: Package jar as debian package (arm64)
+        run: |
+          docker run --rm --platform linux/arm64 \
+            -v $(pwd):/workspace \
+            -w /workspace \
+            arm64v8/eclipse-temurin:21-jdk \
+            bash -c "apt-get update && apt-get install -y fakeroot && jpackage --main-jar data-caterer.jar '@misc/jpackage/jpackage.cfg' '@misc/jpackage/jpackage-linux.cfg'"
+
+      - name: List directory
+        run: ls -lart
+
+      - name: Rename output to indicate arm64
+        run: |
+          if [ -f "datacaterer_1.0.0_amd64.deb" ]; then
+            mv datacaterer_1.0.0_amd64.deb datacaterer_1.0.0_arm64.deb
+          fi
+
+      - name: Upload deb (arm64)
+        uses: actions/upload-artifact@v4
+        with:
+          name: data-caterer-linux-arm64
+          path: "datacaterer_1.0.0_arm64.deb"
+          overwrite: true

.github/workflows/test-build-linux-aarch64.yml

@@ -0,0 +1,130 @@
+name: Test Linux ARM64 Build
+
+on:
+  push:
+    branches:
+      - feature/**
+  workflow_dispatch:
+
+jobs:
+  linux-amd64:
+    runs-on: ubuntu-latest
+    name: Build Linux AMD64 (x86_64)
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Set version
+        run: echo "APP_VERSION=$(grep version gradle.properties | cut -d= -f2)" >> $GITHUB_ENV
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          java-package: jdk
+          architecture: x64
+          distribution: oracle
+      - name: Build fat jar
+        run: ./gradlew :app:shadowJar
+      - name: Package jar as debian package (amd64)
+        run: 'jpackage --main-jar data-caterer.jar "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-linux.cfg"'
+      - name: List directory
+        run: ls -lart
+      - name: Upload deb (amd64)
+        uses: actions/upload-artifact@v4
+        with:
+          name: data-caterer-linux-amd64
+          path: "datacaterer_1.0.0_amd64.deb"
+          overwrite: true
+
+  linux-arm64:
+    runs-on: ubuntu-latest
+    name: Build Linux ARM64 (aarch64)
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Set version
+        run: echo "APP_VERSION=$(grep version gradle.properties | cut -d= -f2)" >> $GITHUB_ENV
+
+      # Set up QEMU for cross-platform builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Use x64 JDK to build the fat jar (architecture independent)
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          java-package: jdk
+          architecture: x64
+          distribution: oracle
+
+      - name: Build fat jar
+        run: ./gradlew :app:shadowJar
+
+      # Use Docker container with ARM64 JDK for jpackage
+      - name: Package jar as debian package (arm64)
+        run: |
+          docker run --rm --platform linux/arm64 \
+            -v $(pwd):/workspace \
+            -w /workspace \
+            arm64v8/eclipse-temurin:21-jdk \
+            bash -c "apt-get update && apt-get install -y fakeroot && jpackage --main-jar data-caterer.jar '@misc/jpackage/jpackage.cfg' '@misc/jpackage/jpackage-linux.cfg'"
+
+      - name: List directory
+        run: ls -lart
+
+      - name: Rename output to indicate arm64
+        run: |
+          if [ -f "datacaterer_1.0.0_amd64.deb" ]; then
+            mv datacaterer_1.0.0_amd64.deb datacaterer_1.0.0_arm64.deb
+          fi
+
+      - name: Upload deb (arm64)
+        uses: actions/upload-artifact@v4
+        with:
+          name: data-caterer-linux-arm64
+          path: "datacaterer_1.0.0_arm64.deb"
+          overwrite: true
+
+  # Alternative approach: Use native ARM64 runners (requires GitHub paid plan)
+  linux-arm64-native:
+    runs-on: ubuntu-24.04-arm64
+    name: Build Linux ARM64 (native runner)
+    continue-on-error: true  # In case ARM64 runners aren't available
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Set version
+        run: echo "APP_VERSION=$(grep version gradle.properties | cut -d= -f2)" >> $GITHUB_ENV
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          java-package: jdk
+          architecture: aarch64
+          distribution: oracle
+      - name: Build fat jar
+        run: ./gradlew :app:shadowJar
+      - name: Package jar as debian package (arm64)
+        run: 'jpackage --main-jar data-caterer.jar "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-linux.cfg"'
+      - name: List directory
+        run: ls -lart
+      - name: Rename output to indicate arm64
+        run: |
+          if [ -f "datacaterer_1.0.0_amd64.deb" ]; then
+            mv datacaterer_1.0.0_amd64.deb datacaterer_1.0.0_arm64.deb
+          fi
+      - name: Upload deb (arm64 native)
+        uses: actions/upload-artifact@v4
+        with:
+          name: data-caterer-linux-arm64-native
+          path: "datacaterer_1.0.0_arm64.deb"
+          overwrite: true

README.md

@@ -38,7 +38,7 @@ and deep dive into issues [from the generated report](https://data.catering/late
 
 1. Docker
    ```shell
-   docker run -d -i -p 9898:9898 -e DEPLOY_MODE=standalone --name datacaterer datacatering/data-caterer:0.17.2
+   docker run -d -i -p 9898:9898 -e DEPLOY_MODE=standalone --name datacaterer datacatering/data-caterer:0.18.0
    ```
    [Open localhost:9898](http://localhost:9898).
 1. [Run Scala/Java examples](#run-scalajava-examples)
@@ -117,12 +117,6 @@ Different ways to run Data Caterer based on your use case:
 
 [Can check here for full list of roadmap items.](https://data.catering/latest/use-case/roadmap/)
 
-## Pricing
-
-Data Caterer is set up under a usage pricing model for the latest application version. There are different pricing tiers based on how much you use Data Caterer. This also includes support and requesting features. The current open-source version will be kept for those who want to continue using the open-source version.
-
-[Find out more details here.](https://data.catering/latest/pricing/)
-
 ### Mildly Quick Start
 
 #### Generate and validate data

api/src/main/java/io/github/datacatering/datacaterer/javaapi/api/PlanRun.java

@@ -240,7 +240,7 @@ public ForeignKeyRelation foreignField(String dataSource, String step, String fi
      * @return A ForeignKeyRelation instance.
      */
     public ForeignKeyRelation foreignField(String dataSource, String step, List<String> fields) {
-        return new ForeignKeyRelation(dataSource, step, toScalaList(fields));
+        return new ForeignKeyRelation(dataSource, step, toScalaList(fields), scala.Option.empty(), scala.Option.empty(), scala.Option.empty());
     }
 
     /**
@@ -255,7 +255,8 @@ public ForeignKeyRelation foreignField(ConnectionTaskBuilder<?> connectionTaskBu
         return new ForeignKeyRelation(
                 connectionTaskBuilder.connectionConfigWithTaskBuilder().dataSourceName(),
                 connectionTaskBuilder.getStep().step().name(),
-                toScalaList(List.of(field))
+                toScalaList(List.of(field)),
+                scala.Option.empty(), scala.Option.empty(), scala.Option.empty()
         );
     }
 
@@ -271,7 +272,8 @@ public ForeignKeyRelation foreignField(ConnectionTaskBuilder<?> connectionTaskBu
         return new ForeignKeyRelation(
                 connectionTaskBuilder.connectionConfigWithTaskBuilder().dataSourceName(),
                 connectionTaskBuilder.getStep().step().name(),
-                toScalaList(fields)
+                toScalaList(fields),
+                scala.Option.empty(), scala.Option.empty(), scala.Option.empty()
         );
     }
 
@@ -284,7 +286,7 @@ public ForeignKeyRelation foreignField(ConnectionTaskBuilder<?> connectionTaskBu
      * @return A ForeignKeyRelation instance.
      */
     public ForeignKeyRelation foreignField(ConnectionTaskBuilder<?> connectionTaskBuilder, String step, List<String> fields) {
-        return new ForeignKeyRelation(connectionTaskBuilder.connectionConfigWithTaskBuilder().dataSourceName(), step, toScalaList(fields));
+        return new ForeignKeyRelation(connectionTaskBuilder.connectionConfigWithTaskBuilder().dataSourceName(), step, toScalaList(fields), scala.Option.empty(), scala.Option.empty(), scala.Option.empty());
     }
 
     /**

api/src/main/scala/io/github/datacatering/datacaterer/api/DataCatererConfigurationBuilder.scala

@@ -3,7 +3,7 @@ package io.github.datacatering.datacaterer.api
 import com.softwaremill.quicklens.ModifyPimp
 import io.github.datacatering.datacaterer.api.connection.{BigQueryBuilder, CassandraBuilder, ConnectionTaskBuilder, FileBuilder, HttpBuilder, KafkaBuilder, MySqlBuilder, NoopBuilder, PostgresBuilder, RabbitmqBuilder, SolaceBuilder}
 import io.github.datacatering.datacaterer.api.converter.Converters.toScalaMap
-import io.github.datacatering.datacaterer.api.model.Constants.{BIGQUERY_WRITE_METHOD, _}
+import io.github.datacatering.datacaterer.api.model.Constants._
 import io.github.datacatering.datacaterer.api.model.DataCatererConfiguration
 
 import scala.annotation.varargs