Rework examples based on latest SQRL standards, introduce shared data catalogs (#113)

Author: ferenc-csaky

.github/workflows/build.yml

@@ -23,7 +23,7 @@ concurrency:
 
 env:
   TZ: 'America/Los_Angeles'
-  SQRL_VERSION: '0.10.7'
+  SQRL_VERSION: '0.10.8'
 
 jobs:
   build-and-push:
@@ -44,46 +44,46 @@ jobs:
               compile recommendation_package_kafka.json
 
           - example: credit-card-analytics
-            path: finance-credit-card-chatbot/credit-card-analytics
+            path: finance-credit-card-chatbot
             tag: credit-card-analytics
             test_commands: |
-              compile creditcard_analytics_package_test.json
-              compile creditcard_analytics_package_kafka.json
-              test creditcard_analytics_package_test.json creditcard_analytics_package_jwt.json
+              compile credit-card-analytics/creditcard_analytics_package_test.json
+              compile credit-card-analytics/creditcard_analytics_package_kafka.json
+              test creditcard_analytics_package_test.json creditcard_analytics_package_jwt.json -r credit-card-analytics
           - example: credit-card-rewards
-            path: finance-credit-card-chatbot/credit-card-rewards
+            path: finance-credit-card-chatbot
             tag: credit-card-rewards
             test_commands: |
-              compile creditcard_rewards_package_test.json
-              compile creditcard_rewards_package_kafka.json
-              test creditcard_rewards_package_test.json creditcard_rewards_package_jwt.json
+              compile credit-card-rewards/creditcard_rewards_package_test.json
+              compile credit-card-rewards/creditcard_rewards_package_kafka.json
+              test creditcard_rewards_package_test.json creditcard_rewards_package_jwt.json -r credit-card-rewards
           - example: credit-card-views
-            path: finance-credit-card-chatbot/credit-card-views
+            path: finance-credit-card-chatbot
             tag: credit-card-views
             test_commands: |
-              compile creditcard_views_package_test.json
-              test creditcard_views_package_test.json
+              compile credit-card-views/creditcard_views_package_test.json
+              test credit-card-views/creditcard_views_package_test.json
 
           - example: healthcare-study-analytics
-            path: healthcare-study/healthcare-study-analytics
+            path: healthcare-study
             tag: healthcare-study-analytics
             test_commands: |
-              compile study_analytics_package_test.json
-              test study_analytics_package_test.json
+              compile healthcare-study-analytics/study_analytics_package_test.json
+              test healthcare-study-analytics/study_analytics_package_test.json
           - example: healthcare-study-api
-            path: healthcare-study/healthcare-study-api
+            path: healthcare-study
             tag: healthcare-study-api
             test_commands: |
-              compile study_api_package_api.json
-              compile study_api_package_test.json
-              test study_api_package_test.json
+              compile healthcare-study-api/study_api_package_api.json
+              compile healthcare-study-api/study_api_package_test.json
+              test healthcare-study-api/study_api_package_test.json
           - example: healthcare-study-stream
-            path: healthcare-study/healthcare-study-stream
+            path: healthcare-study
             tag: healthcare-study-stream
             test_commands: |
-              compile study_stream_package_kafka.json
-              compile study_stream_package_test.json
-              test study_stream_package_test.json
+              compile healthcare-study-stream/study_stream_package_kafka.json
+              compile healthcare-study-stream/study_stream_package_test.json
+              test healthcare-study-stream/study_stream_package_test.json
 
           - example: iceberg-dedup
             path: iceberg-data-deduplication
@@ -112,16 +112,16 @@ jobs:
               test logistics_package_test.json
 
           - example: oil-gas-monitoring
-            path: oil-gas-agent-automation/oil-gas-agent-monitoring
+            path: oil-gas-agent-automation
             tag: oil-gas-agent-monitoring
             test_commands: |
-              compile monitoring_agent_package_test.json
+              compile oil-gas-agent-monitoring/monitoring_agent_package_test.json
           - example: oil-gas-operations
-            path: oil-gas-agent-automation/oil-gas-agent-operations
+            path: oil-gas-agent-automation
             tag: oil-gas-agent-operations
             test_commands: |
-              compile operations_agent_package_test.json
-              test operations_agent_package_test.json
+              compile oil-gas-agent-operations/operations_agent_package_test.json
+              test oil-gas-agent-operations/operations_agent_package_test.json
 
           - example: usage-analytics
             path: usage-analytics
@@ -232,7 +232,7 @@ jobs:
             -e AWS_SESSION_TOKEN \
             -e AWS_REGION \
             -e AWS_DEFAULT_REGION \
-            --rm -v "$PWD":/build "$SQRL_IMAGE" $cmd
+            --rm -v "$PWD":/workspace "$SQRL_IMAGE" $cmd
           echo "::endgroup::"
         done
 
@@ -244,8 +244,8 @@ jobs:
         cat <<EOF > ${{ matrix.path }}/Dockerfile
         FROM ${SQRL_IMAGE}
 
-        COPY . /build
-        WORKDIR /build
+        COPY . /workspace
+        WORKDIR /workspace
         EOF
 
     - name: Set up QEMU
@@ -277,7 +277,7 @@ jobs:
     name: Lint GraphQL Schemas
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Lint GraphQL schemas
         run: |

README.md

@@ -21,20 +21,20 @@ To compile and run these examples with DataSQRL you need to have a [recent versi
 
 To run the examples on Linux or MacOS, open a terminal and run the following command:
 ```bash
-docker run -it --rm -p 8888:8888 -p 8081:8081 -p 9092:9092 -v $PWD:/build datasqrl/cmd:latest run [ARGUMENTS GO HERE]
+docker run -it --rm -p 8888:8888 -p 8081:8081 -p 9092:9092 -v $PWD:/workspace datasqrl/cmd:latest run [ARGUMENTS GO HERE]
 ```
 
 If you are on windows using Powershell, you need to reference the local directory with a slightly different syntax:
 ```bash
-docker run -it --rm -p 8888:8888 -p 8081:8081 -p 9092:9092 -v $PWD:/build datasqrl/cmd:latest run [ARGUMENTS GO HERE]
+docker run -it --rm -p 8888:8888 -p 8081:8081 -p 9092:9092 -v $PWD:/workspace datasqrl/cmd:latest run [ARGUMENTS GO HERE]
 ```
 
 Check the `README.md` in the respective directory for more information on the arguments to run each example. 
 We will be using the Unix syntax, so keep in mind that you have to adjust the commands slightly on Windows machines by using `${PWD}` instead.
 
 To compile an example (without running it), use this command:
 ```bash
-docker run -it --rm -v $PWD:/build datasqrl/cmd:latest compile [ARGUMENTS GO HERE]
+docker run -it --rm -v $PWD:/workspace datasqrl/cmd:latest compile [ARGUMENTS GO HERE]
 ```
 
 ## What is DataSQRL?

clickstream-ai-recommendation/README.md

@@ -25,7 +25,7 @@ This version allows you to add content and clickstream data manually through a G
 
 Run the API version with this command:
 ```bash
-docker run -it -p 8081:8081 -p 8888:8888 --rm -v $PWD:/build -e OPENAI_API_KEY=[YOUR_API_KEY_HERE] datasqrl/cmd:latest run -c recommendation_package_api.json
+docker run -it -p 8081:8081 -p 8888:8888 --rm -v $PWD:/workspace -e OPENAI_API_KEY=[YOUR_API_KEY_HERE] datasqrl/cmd:latest run -c recommendation_package_api.json
 ```
 
 Next, open [GraphQL Explorer](http://localhost:8888/v1/graphiql/) and add some content:
@@ -79,7 +79,7 @@ data through a Kafka connector.
 
 To run the pipeline:
 ```bash
-docker run -it -p 8081:8081 -p 8888:8888 -p 9092:9092 --rm -v $PWD:/build -e OPENAI_API_KEY=[YOUR_API_KEY_HERE] datasqrl/cmd:latest run -c recommendation_package_kafka.json
+docker run -it -p 8081:8081 -p 8888:8888 -p 9092:9092 --rm -v $PWD:/workspace -e OPENAI_API_KEY=[YOUR_API_KEY_HERE] datasqrl/cmd:latest run -c recommendation_package_kafka.json
 ```
 
 1. Once everything is started, open another terminal window to add data to Kafka using the

clickstream-ai-recommendation/clickstream-api/clickstream.sqrl

@@ -1,3 +1,10 @@
+/*+ engine(kafka) */
+CREATE TABLE Clickstream (
+    url STRING NOT NULL,
+    userid STRING NOT NULL,
+    event_time TIMESTAMP_LTZ(3) NOT NULL METADATA FROM 'timestamp'
+);
+
 /*+ engine(kafka) */
 CREATE TABLE Content (
     url STRING NOT NULL,

clickstream-ai-recommendation/clickstream-kafka/clickstream.sqrl

@@ -1,3 +1,18 @@
+CREATE TABLE Clickstream (
+    url STRING NOT NULL,
+    userid STRING NOT NULL,
+    event_time TIMESTAMP_LTZ(3) NOT NULL METADATA FROM 'timestamp',
+    WATERMARK FOR event_time AS event_time - INTERVAL '0.001' SECOND
+) WITH (
+    'connector' = 'kafka',
+    'topic' = 'clickstream',
+    'properties.bootstrap.servers' = '${KAFKA_BOOTSTRAP_SERVERS}',
+    'properties.group.id' = 'clickstream-group',
+    'scan.startup.mode' = 'group-offsets',
+    'properties.auto.offset.reset' = 'earliest',
+    'format' = 'flexible-json'
+);
+
 CREATE TABLE Content (
      url STRING NOT NULL,
      title STRING NOT NULL,
@@ -12,4 +27,4 @@ CREATE TABLE Content (
     'scan.startup.mode' = 'group-offsets',
     'properties.auto.offset.reset' = 'earliest',
     'format' = 'flexible-json'
-    );
+);

…-recommendation/content-api/content.sqrl …am-ai-recommendation/connectors/api.sqrlclickstream-ai-recommendation/content-api/content.sqrl renamed to clickstream-ai-recommendation/connectors/api.sqrl clickstream-ai-recommendation/content-api/content.sqrl renamed to clickstream-ai-recommendation/connectors/api.sqrl

@@ -1,8 +1,5 @@
 /*+no_query */
-IMPORT clickstream-{{variant}}.clickstream.*;
-
-/*+no_query */
-IMPORT content-{{variant}}.content.*;
+IMPORT connectors.{{variant}}.*;
 
 /* Find next page visits within 10 minutes */
 _CoVisits := SELECT b.url AS beforeURL, a.url AS afterURL,

…ecommendation/content-kafka/content.sqrl …-ai-recommendation/connectors/kafka.sqrlclickstream-ai-recommendation/content-kafka/content.sqrl renamed to clickstream-ai-recommendation/connectors/kafka.sqrl clickstream-ai-recommendation/content-kafka/content.sqrl renamed to clickstream-ai-recommendation/connectors/kafka.sqrl

@@ -24,7 +24,7 @@ package JSON filename) - the instructions are otherwise identical.
 To run this example, invoke the following command in the example directory on Unix based systems to compile the project:
 ```bash
 cd credit-card-analytics
-docker run -it -p 8081:8081 -p 8888:8888 --rm -v $PWD:/build datasqrl/cmd:latest run -c creditcard_analytics_package_test.json
+docker run -it -p 8081:8081 -p 8888:8888 --rm -v $PWD:/workspace datasqrl/cmd:latest run -c creditcard_analytics_package_test.json
 ```
 
 This command stands up the API using [DataSQRL](https://www.datasqrl.com/), a development tool
@@ -42,7 +42,7 @@ To use Kafka as the data source, follow these steps:
 Invoke the following command in the example directory:
 ```bash
 cd credit-card-analytics
-docker run -it -p 8081:8081 -p 8888:8888 -p 9092:9092 --rm -v $PWD:/build datasqrl/cmd:latest run -c creditcard_analytics_package_kafka.json
+docker run -it -p 8081:8081 -p 8888:8888 -p 9092:9092 --rm -v $PWD:/workspace datasqrl/cmd:latest run -c creditcard_analytics_package_kafka.json
 ```
 
 This command stands up the entire data pipeline and all data services, including Kafka.
@@ -76,7 +76,7 @@ any of the main package configurations to secure the endpoints, regardless of ot
 For example, to secure the API with the File data source, invoke the following command in the specific example directory:
 ```bash
 cd credit-card-analytics
-docker run -it -p 8081:8081 -p 8888:8888 --rm -v $PWD:/build datasqrl/cmd:latest run \
+docker run -it -p 8081:8081 -p 8888:8888 --rm -v $PWD:/workspace datasqrl/cmd:latest run \
   -c creditcard_analytics_package_test.json \
   -c creditcard_analytics_package_jwt.json
 ```