feat!: Ship runner uber JAR in Flink lib/ (#317)

Co-authored-by: Marvin Froeder <marvin@datasqrl.com>

Author: ferenc-csaky

.github/workflows/test-cloud-compilation.yml

@@ -182,37 +182,27 @@ jobs:
       - name: Import GPG key
         id: gpg
         env:
-          GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
+          GPG_PRIVATE_KEY: ${{ secrets.BOT_GPG_PRIVATE_KEY }}
         run: |
           echo "$GPG_PRIVATE_KEY" | gpg --batch --import
           KEY_ID=$(gpg --list-secret-keys --keyid-format=long --with-colons \
             | awk -F: '/^sec/ {print $5; exit}')
-          UID_LINE=$(gpg --list-secret-keys --with-colons \
-            | awk -F: '/^uid/ {print $10; exit}')
-          USER_NAME=$(echo "$UID_LINE" | sed -E 's/ *<[^>]+>$//')
-          USER_EMAIL=$(echo "$UID_LINE" | sed -E 's/.*<([^>]+)>.*/\1/')
-          if [ -z "$KEY_ID" ] || [ -z "$USER_EMAIL" ]; then
-            echo "Failed to determine GPG key id / uid (key_id=${KEY_ID}, uid=${UID_LINE})"
+          if [ -z "$KEY_ID" ]; then
+            echo "Failed to determine GPG key id"
             exit 1
           fi
           mkdir -p ~/.gnupg
           echo "default-key ${KEY_ID}" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
-          {
-            echo "key_id=${KEY_ID}"
-            echo "user_name=${USER_NAME}"
-            echo "user_email=${USER_EMAIL}"
-          } >> "$GITHUB_OUTPUT"
+          echo "key_id=${KEY_ID}" >> "$GITHUB_OUTPUT"
 
       - name: Configure git
         env:
           KEY_ID: ${{ steps.gpg.outputs.key_id }}
-          USER_NAME: ${{ steps.gpg.outputs.user_name }}
-          USER_EMAIL: ${{ steps.gpg.outputs.user_email }}
         run: |
           cd compilation
-          git config user.name "${USER_NAME}"
-          git config user.email "${USER_EMAIL}"
+          git config user.name "Datasqrl - Automation Bot"
+          git config user.email "bot@datasqrl.com"
           git config user.signingkey "${KEY_ID}"
           git config commit.gpgsign true
 
@@ -449,37 +439,27 @@ jobs:
         id: gpg
         if: steps.branch_exists.outputs.exists == 'true'
         env:
-          GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
+          GPG_PRIVATE_KEY: ${{ secrets.BOT_GPG_PRIVATE_KEY }}
         run: |
           echo "$GPG_PRIVATE_KEY" | gpg --batch --import
           KEY_ID=$(gpg --list-secret-keys --keyid-format=long --with-colons \
             | awk -F: '/^sec/ {print $5; exit}')
-          UID_LINE=$(gpg --list-secret-keys --with-colons \
-            | awk -F: '/^uid/ {print $10; exit}')
-          USER_NAME=$(echo "$UID_LINE" | sed -E 's/ *<[^>]+>$//')
-          USER_EMAIL=$(echo "$UID_LINE" | sed -E 's/.*<([^>]+)>.*/\1/')
-          if [ -z "$KEY_ID" ] || [ -z "$USER_EMAIL" ]; then
-            echo "Failed to determine GPG key id / uid (key_id=${KEY_ID}, uid=${UID_LINE})"
+          if [ -z "$KEY_ID" ]; then
+            echo "Failed to determine GPG key id"
             exit 1
           fi
           mkdir -p ~/.gnupg
           echo "default-key ${KEY_ID}" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
-          {
-            echo "key_id=${KEY_ID}"
-            echo "user_name=${USER_NAME}"
-            echo "user_email=${USER_EMAIL}"
-          } >> "$GITHUB_OUTPUT"
+          echo "key_id=${KEY_ID}" >> "$GITHUB_OUTPUT"
 
       - name: Configure git
         if: steps.branch_exists.outputs.exists == 'true'
         env:
           KEY_ID: ${{ steps.gpg.outputs.key_id }}
-          USER_NAME: ${{ steps.gpg.outputs.user_name }}
-          USER_EMAIL: ${{ steps.gpg.outputs.user_email }}
         run: |
-          git config user.name "${USER_NAME}"
-          git config user.email "${USER_EMAIL}"
+          git config user.name "Datasqrl - Automation Bot"
+          git config user.email "bot@datasqrl.com"
           git config user.signingkey "${KEY_ID}"
           git config commit.gpgsign true
 

flink-sql-runner/src/main/docker/Dockerfile

@@ -16,8 +16,6 @@
 
 FROM apache/flink:${flink-base-image}
 
-RUN mkdir -p /opt/flink/plugins/flink-sql-runner
-
 COPY flink-table-planner_2.12-*.jar /opt/flink/lib
 COPY flink-s3-fs-hadoop-*.jar /opt/flink/lib
 COPY hadoop-common-*.jar /opt/flink/lib
@@ -26,11 +24,11 @@ COPY hadoop-mapreduce-client-core-*.jar /opt/flink/lib
 COPY iceberg-flink-runtime-*.jar /opt/flink/lib
 COPY iceberg-aws-bundle-*.jar /opt/flink/lib
 COPY stdlib-utils-*.jar /opt/flink/lib
-COPY flink-sql-runner.uber.jar /opt/flink/plugins/flink-sql-runner
+COPY flink-sql-runner.uber.jar /opt/flink/lib/sql-runner.uber.jar
 COPY --chmod=755 entrypoint.sh /entrypoint.sh
 
 RUN rm -rf /opt/flink/lib/flink-table-planner-loader-*.jar \
-    && ln -s /opt/flink/plugins/flink-sql-runner/flink-sql-runner.uber.jar /opt/flink/flink-sql-runner.jar \
+    && echo 'UEsDBBQACAgIAG2UllwAAAAAAAAAAAAAAAAUAAQATUVUQS1JTkYvTUFOSUZFU1QuTUb+ygAA803My0xLLS7RDUstKs7Mz7NSMNQz4HIuSk0sSU3Rdaq0UkjLyczL1i0uzNEtKs3LSy3i4gIAUEsHCL/cUjU2AAAANAAAAFBLAQIUABQACAgIAG2Ully/3FI1NgAAADQAAAAUAAQAAAAAAAAAAAAAAAAAAABNRVRBLUlORi9NQU5JRkVTVC5NRv7KAABQSwUGAAAAAAEAAQBGAAAAfAAAAAAA' | base64 -d > /opt/flink/noop.jar \
     && mkdir -p /opt/flink/hadoop-conf \
     && echo '<configuration><property><name>fs.s3a.aws.credentials.provider</name><value>com.amazonaws.auth.DefaultAWSCredentialsProviderChain</value></property></configuration>' > /opt/flink/hadoop-conf/core-site.xml
 

flink-sql-runner/src/test/java/com/datasqrl/flinkrunner/AbstractITSupport.java

@@ -165,7 +165,9 @@ protected String flinkRun(List<String> sqlRunnerArgs, @Nullable String savepoint
       execCmd.add(savepointPath);
     }
 
-    execCmd.add("./plugins/flink-sql-runner/flink-sql-runner.uber.jar");
+    execCmd.add("-c");
+    execCmd.add("com.datasqrl.flinkrunner.CliRunner");
+    execCmd.add("noop.jar");
     execCmd.addAll(sqlRunnerArgs);
 
     var execRes = flinkContainer.execInContainer(execCmd.toArray(new String[0]));