Merge remote-tracking branch 'origin/candidate-10.0.x' into candidate-10.2.x

Author: GordonSmith

.github/actions/download-eclwatch-nightly/action.yml

@@ -0,0 +1,62 @@
+name: "Download ECL Watch Nightly"
+description: "Download the latest non-expired ECL Watch nightly artifact into a package build tree."
+
+inputs:
+  destination:
+    description: "Destination directory for the downloaded nightly dist files."
+    required: true
+  artifact-name:
+    description: "Name of the ECL Watch nightly artifact."
+    required: false
+    default: eclwatch-nightly
+
+runs:
+  using: "composite"
+  steps:
+    - name: Locate latest ECL Watch nightly
+      id: nightly
+      uses: actions/github-script@v8
+      env:
+        NIGHTLY_ARTIFACT_NAME: ${{ inputs.artifact-name }}
+        NIGHTLY_DESTINATION: ${{ inputs.destination }}
+      with:
+        github-token: ${{ github.token }}
+        script: |
+          const fs = require("fs");
+
+          const artifactName = process.env.NIGHTLY_ARTIFACT_NAME;
+          const destination = process.env.NIGHTLY_DESTINATION;
+          const [owner, repo] = context.repo.owner && context.repo.repo
+            ? [context.repo.owner, context.repo.repo]
+            : "${{ github.repository }}".split("/");
+
+          fs.rmSync(destination, { recursive: true, force: true });
+          fs.mkdirSync(destination, { recursive: true });
+
+          const artifacts = await github.paginate(github.rest.actions.listArtifactsForRepo, {
+            owner,
+            repo,
+            name: artifactName,
+            per_page: 100
+          });
+
+          const artifact = artifacts
+            .filter(item => !item.expired && item.workflow_run?.id)
+            .sort((left, right) => Date.parse(right.created_at) - Date.parse(left.created_at))[0];
+
+          if (!artifact) {
+            core.setFailed(`Unable to find a non-expired ${artifactName} artifact in ${owner}/${repo}.`);
+            return;
+          }
+
+          core.info(`Using ${artifactName} artifact ${artifact.id} from run ${artifact.workflow_run.id}, created ${artifact.created_at}.`);
+          core.setOutput("run-id", String(artifact.workflow_run.id));
+
+    - name: Download latest ECL Watch nightly
+      uses: actions/download-artifact@v8
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: ${{ inputs.destination }}
+        repository: ${{ github.repository }}
+        run-id: ${{ steps.nightly.outputs.run-id }}
+        github-token: ${{ github.token }}

.github/workflows/build-docker-community.yml

@@ -65,6 +65,7 @@ on:
 
 permissions:
   contents: write
+  actions: read
 
 jobs:
   build:
@@ -140,6 +141,12 @@ jobs:
       - name: Print Vars
         run: echo "${{ toJson(steps.vars.outputs)}}"
 
+      - name: Download ECL Watch Nightly
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        uses: ./HPCC-Platform/.github/actions/download-eclwatch-nightly
+        with:
+          destination: ${{ steps.vars.outputs.folder_build }}/esp/src/build/dist-nightly
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v4

.github/workflows/build-docker-enterprise.yml

@@ -55,6 +55,7 @@ on:
 
 permissions:
   contents: read
+  actions: read
 
 jobs:
   build:
@@ -130,6 +131,12 @@ jobs:
       - name: Print Vars
         run: echo "${{ toJson(steps.vars.outputs)}}"
 
+      - name: Download ECL Watch Nightly
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        uses: ./HPCC-Platform/.github/actions/download-eclwatch-nightly
+        with:
+          destination: ${{ steps.vars.outputs.folder_build }}/esp/src/build/dist-nightly
+
       - name: Checkout LN
         uses: actions/checkout@v6
         with:

.github/workflows/build-docker-internal.yml

@@ -55,6 +55,7 @@ on:
 
 permissions:
   contents: read
+  actions: read
 
 jobs:
   build:
@@ -130,6 +131,12 @@ jobs:
       - name: Print Vars
         run: echo "${{ toJson(steps.vars.outputs)}}"
 
+      - name: Download ECL Watch Nightly
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        uses: ./HPCC-Platform/.github/actions/download-eclwatch-nightly
+        with:
+          destination: ${{ steps.vars.outputs.folder_build }}/esp/src/build/dist-nightly
+
       - name: Checkout LN
         uses: actions/checkout@v6
         with:

.github/workflows/build-docker.yml

@@ -69,6 +69,7 @@ on:
 
 permissions:
   contents: read
+  actions: read
 
 jobs:
 
@@ -129,6 +130,12 @@ jobs:
         run: |
           echo "${{ toJSON(steps.vars.outputs) }}"
 
+      - name: Download ECL Watch Nightly
+        if: ${{ startsWith(github.ref, 'refs/tags/') && inputs.upload-package == true }}
+        uses: ./HPCC-Platform/.github/actions/download-eclwatch-nightly
+        with:
+          destination: ${{ github.workspace }}/build/esp/src/build/dist-nightly
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v4

.github/workflows/build-eclwatch.yml

@@ -14,40 +14,34 @@ jobs:
   build:
     name: "Build eclwatch"
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node: ["24", "22"]
-      fail-fast: false
 
     steps:
-      - name: Free additional disk space (remove Android SDK + Tools)
-        run: |
-          sudo rm -rf /usr/local/lib/android
+
       - uses: actions/checkout@v6
         with:
           submodules: "recursive"
 
       - uses: actions/setup-node@v6
         with:
-          node-version: ${{ matrix.node }}
+          node-version: 24
+          cache: 'npm'
+          cache-dependency-path: ${{ github.workspace }}/esp/src/package-lock.json
 
       - name: Install Dependencies
         working-directory: ${{ github.workspace }}/esp/src
         run: npm ci
 
-      - name: Lint
-        working-directory: ${{ github.workspace }}/esp/src
-        run: npm run lint
-
       - name: Build
         working-directory: ${{ github.workspace }}/esp/src
+        env:
+          ECLWATCH_DIST_URL: /esp/files/dist-nightly/
         run: npm run build
 
       - name: Upload Package
-        if: ${{ matrix.node == '22' && inputs.asset-name}}
+        if: ${{ inputs.asset-name }}
         uses: actions/upload-artifact@v7
         with:
           name: ${{ inputs.asset-name }}
           path: |
-            ${{ github.workspace }}/esp/src/build/**/*
+            ${{ github.workspace }}/esp/src/build/dist/**/*
           if-no-files-found: error

.github/workflows/build-gh_runner.yml

@@ -68,6 +68,7 @@ on:
 
 permissions:
   contents: read
+  actions: read
   packages: read
 
 jobs:
@@ -185,6 +186,12 @@ jobs:
         run: |
           echo "${{ toJSON(steps.vars.outputs) }}"
 
+      - name: Download ECL Watch Nightly
+        if: ${{ startsWith(github.ref, 'refs/tags/') && inputs.upload-package == true }}
+        uses: ./HPCC-Platform/.github/actions/download-eclwatch-nightly
+        with:
+          destination: ${{ github.workspace }}/build/esp/src/build/dist-nightly
+
       - uses: actions/cache@v5
         id: cache
         with:

.github/workflows/build-vcpkg.yml

@@ -356,3 +356,9 @@ jobs:
       cmake-configuration-ex: '-DCMAKE_OSX_ARCHITECTURES=arm64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
     secrets: inherit
 
+  build-build-eclwatch:
+    if: ${{ github.event_name == 'schedule' }}
+    uses: ./.github/workflows/build-eclwatch.yml
+    with:
+      asset-name: 'eclwatch-nightly'
+    secrets: inherit

.github/workflows/test-regression-suite-k8s.yml

@@ -16,6 +16,7 @@ on:
 
 permissions:
   contents: read
+  actions: read
 
 jobs:
   build-docker:

ecl/hthor/hthorkey.cpp

@@ -234,6 +234,7 @@ class CHThorIndexReadActivityBase : public CHThorActivityBase
 private:
     bool firstMultiPart();
     bool nextMultiPart();
+    size32_t getPartBlockedIOSize(IDistributedFilePart &part) const;
     bool setCurrentPart(unsigned whichPart);
     void clearTlk()                                         { tlk.clear(); tlManager.clear(); }
     void openTlk();
@@ -274,6 +275,7 @@ class CHThorIndexReadActivityBase : public CHThorActivityBase
     bool singlePart = false;                // a single part index, not part of a super file - optimize so never reload the part.
     bool localSortKey = false;
     bool initializedFileInfo = false;
+    size32_t foreignBlockedIOSize = (size32_t)-1;
 
 //for layout translation
     Owned<const IDynamicTransform> layoutTrans;
@@ -330,6 +332,9 @@ void CHThorIndexReadActivityBase::resolveIndexFilename()
 {
     // A logical filename for the key should refer to a single physical file - either the TLK or a monolithic key
     OwnedRoxieString lfn(helper.getFileName());
+    CDfsLogicalFileName dlfn;
+    dlfn.set(lfn);
+    foreignBlockedIOSize = dlfn.isForeign() ? getForeignBlockedIOSize(helper.hasSegmentMonitors()) : (size32_t)-1;
     Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(agent, lfn, "IndexRead", 0 != (helper.getFlags() & TIRoptional),true, AccessMode::tbdRead, defaultPrivilegedUser);
     df.set(ldFile ? ldFile->queryDistributedFile() : NULL);
     if (!df)
@@ -452,7 +457,8 @@ bool CHThorIndexReadActivityBase::doPreopenLimitFile(unsigned __int64 & count, u
 IKeyIndex * CHThorIndexReadActivityBase::doPreopenLimitPart(unsigned __int64 & result, unsigned __int64 limit, unsigned part)
 {
     Owned<IKeyIndex> kidx;
-    kidx.setown(openKeyFile(df->queryPart(part)));
+    IDistributedFilePart & filePart = df->queryPart(part);
+    kidx.setown(openKeyFile(filePart, getPartBlockedIOSize(filePart)));
     if(df->numParts() == 1)
         verifyIndex(kidx);
     if (limit != (unsigned) -1)
@@ -573,17 +579,20 @@ void CHThorIndexReadActivityBase::killPart()
         klManager.clear();
 }
 
+size32_t CHThorIndexReadActivityBase::getPartBlockedIOSize(IDistributedFilePart &part) const
+{
+    if (foreignBlockedIOSize != (size32_t)-1)
+        return foreignBlockedIOSize;
+
+    StringBuffer planeName;
+    df->getClusterName(part.copyClusterNum(0), planeName);
+    return getIndexBlockedIOSize(planeName, helper.hasSegmentMonitors());
+}
+
 bool CHThorIndexReadActivityBase::setCurrentPart(unsigned whichPart)
 {
     IDistributedFilePart &part = df->queryPart(whichPart);
-    size32_t blockedSize = 0;
-    if (!helper.hasSegmentMonitors()) // unfiltered
-    {
-        StringBuffer planeName;
-        df->getClusterName(part.copyClusterNum(0), planeName);
-        blockedSize = getBlockedFileIOSize(planeName);
-    }
-    keyIndex.setown(openKeyFile(part, blockedSize));
+    keyIndex.setown(openKeyFile(part, getPartBlockedIOSize(part)));
     if(df->numParts() == 1)
         verifyIndex(keyIndex);
     initPart();