HPCC-35979 feat(eclccserver) Add github app support to eclccserver

common/remote/hooks/git/gitfile.cpp

@@ -250,20 +250,9 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf<IFileIO>
             addPathSepChar(scriptPath).append("bin/hpccaskpass.sh");
             env.emplace_back("GIT_ASKPASS", scriptPath);
 
-            Owned<const IPropertyTree> secret = getSecret("git", gitUser);
-            if (secret)
-            {
-                MemoryBuffer gitKey;
-                if (getSecretKeyValue(gitKey, secret, "password"))
-                {
-                    extractedKey.setown(writeToProtectedTempFile("eclcc", "git", gitKey.length(), gitKey.toByteArray()));
-                    env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename());
-                }
-                else
-                    OWARNLOG("Secret doesn't contain password for git user %s", gitUser);
-            }
-            else
-                OWARNLOG("No secret found for git user %s", gitUser);
+            extractedKey.setown(getFileWithGitAccessToken(gitUser));
+            if (extractedKey)
+                env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename());
         }
         Owned<IPipeProcess> pipe = createPipeProcess();
         for (const auto & cur : env)

devdoc/githubapps.md

@@ -0,0 +1,49 @@
+# Using GitHub apps for authenticating from eclccserver
+
+## Creating and Configuring a GitHub App for eclccserver
+
+Here is a step-by-step guide to creating and configuring a GitHub App to use with the authentication mechanism.
+
+### Step 1: Create the GitHub App
+1. Go to your GitHub account (or Organization) **Settings**.
+2. On the left sidebar, scroll down and click **Developer settings**.
+3. Select **GitHub Apps** in the left sidebar, then click the **New GitHub App** button.
+4. Fill out the **Register new GitHub App** form:
+   * **GitHub App name**: Give it a recognizable name (e.g., `HPCC ECL Fetcher`).
+   * **Homepage URL**: You can use your organization's URL or the HPCC platform repository URL (this is just required by the form).
+   * **Webhook**: Uncheck the **Active** checkbox. (HPCC only needs to pull code; it doesn't need to listen for GitHub events).
+5. Set the **Repository permissions**:
+   * Under **Repository permissions**, find **Contents** and set the access to **Read-only**. (This gives the app permission to perform `git fetch` / `git clone`).
+6. Scroll to the bottom (**Where can this GitHub App be installed?**):
+   * Select **Any account** if you plan to fetch repositories owned by different organizations/users. Select **Only on this account** if you exclusively fetch repositories within your current organization.
+7. Click **Create GitHub App**.
+
+### Step 2: Gather Your Secrets
+Immediately after creating the app, you will land on its settings page. You need to collect three items to populate your HPCC secrets configuration:
+
+1. **The App ID** (`appid`):
+   * Look at the "About" section at the top of the page. Copy the **App ID**.
+2. **The Private Key** (`appkey`):
+   * Scroll down to the **Private keys** section.
+   * Click **Generate a private key**.
+   * This will download a `.pem` file to your computer. The entire contents of this file (including the `-----BEGIN RSA PRIVATE KEY-----` and end tags) will be your `appkey` secret.
+
+### Step 3: Install the App & Get the Installation ID
+The App ID and Private Key alone are not enough; the app must be *installed* on the target account/repository to generate the `installationid`.
+
+1. While still on the App's settings page, click **Install App** on the left sidebar.
+2. Click **Install** next to the user or organization account where the target ECL repositories live.
+3. Choose whether to install it on **All repositories** or **Only select repositories**, then click **Install**.
+4. You will be redirected to the installation settings page. Look at the URL in your browser's address bar. It will look like this:
+   `https://github.com/settings/installations/12345678`
+   or
+   `https://github.com/organizations/YOUR_ORG/settings/installations/12345678`
+5. Copy the numeric value at the end of the URL (e.g., `12345678`). This is your **Installation ID** (`installationid`).
+
+## Configuring the secrets for eclccserver
+
+1. configure the git user name to be x-access-token
+
+2. Create a secret category "git", secret "x-access-token"
+
+3. Place these three values highlighted above (`appid`, `appkey`, and `installationid`) into your HPCC secret.

ecl/hql/hqlrepository.cpp

@@ -1008,33 +1008,12 @@ unsigned EclRepositoryManager::runGitCommand(StringBuffer * output, const char *
             // If gituser is specified never prompt for credentials, otherwise the server can hang.
             env.emplace_back("GIT_TERMINAL_PROMPT", "0");
 
-            if (!options.gitPasswordPath.isEmpty())
+            extractedKey.setown(getFileWithGitAccessToken(options.gitUser.str()));
+            if (extractedKey)
             {
-                //Convert to an absolute path, and check the file exists, because git will be run in a different directory
-                StringBuffer absolutePath;
-                makeAbsolutePath(options.gitPasswordPath.str(), absolutePath, true);
-
-                env.emplace_back("HPCC_GIT_PASSPATH", absolutePath);
+                env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename());
                 useScript = true;
             }
-            else
-            {
-                Owned<const IPropertyTree> secret = getSecret("git", options.gitUser.str());
-                if (secret)
-                {
-                    MemoryBuffer gitKey;
-                    if (!getSecretKeyValue(gitKey, secret, "password"))
-                        DBGLOG("Secret doesn't contain password for git user %s", options.gitUser.str());
-                    else
-                    {
-                        extractedKey.setown(writeToProtectedTempFile("eclcc", "git", gitKey.length(), gitKey.toByteArray()));
-                        env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename());
-                        useScript = true;
-                    }
-                }
-                else
-                    DBGLOG("No secret found for git user %s", options.gitUser.str());
-            }
         }
 
         if (useScript)

system/jlib/CMakeLists.txt

@@ -56,6 +56,9 @@ if (NOT WIN32 AND NOT WIN64 AND NOT APPLE AND NOT EMSCRIPTEN)
     pkg_check_modules(liburing REQUIRED IMPORTED_TARGET liburing>=2.0)
 endif ()
 
+find_package(CURL REQUIRED)
+find_path(JWT_CPP_INCLUDE_DIRS "jwt-cpp/base.h" HINTS "${VCPKG_INSTALLED_DIR}")
+
 SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STRICT_CXX_FLAGS}")
 
 set (    SRCS
@@ -212,6 +215,7 @@ set (    INCLUDES
         ${HPCC_SOURCE_DIR}/system/security/cryptohelper/digisign.cpp
         ${HPCC_SOURCE_DIR}/system/security/cryptohelper/pke.cpp
         ${HPCC_SOURCE_DIR}/system/security/cryptohelper/ske.cpp
+        ${JWT_CPP_INCLUDE_DIRS}/jwt-cpp/jwt.h
         )
 
 set_source_files_properties(jmd5.cpp jsort.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)

system/jlib/jsecrets.cpp

@@ -49,6 +49,20 @@
 #pragma GCC diagnostic pop
 #endif
 
+#ifdef verify
+    #define JWT_HAS_VERIFY_MACRO
+    #define OLD_VERIFY verify
+    #undef verify
+#endif
+
+#include "jwt-cpp/jwt.h"
+#include "nlohmann/json.hpp"
+
+#ifdef JWT_HAS_VERIFY_MACRO
+    //Force an error if verify macro is used in any of the following code
+    #undef verify
+#endif
+
 #ifdef _USE_OPENSSL
 #include <opensslcommon.hpp>
 #include <openssl/x509v3.h>
@@ -2427,3 +2441,70 @@ void maskSecret(StringBuffer & maskedSecret, const char * originalSecret, unsign
             maskedSecret.append(originalSecret[i]);
     }
 }
+
+std::string generateGithubIAT(const char * appId, const char * appKey, const char * installationId)
+{
+    try
+    {
+        auto now = std::chrono::system_clock::now();
+        auto token = jwt::create()
+            .set_issuer(appId)
+            .set_issued_at(now)
+            .set_expires_at(now + std::chrono::minutes(10))
+            .sign(jwt::algorithm::rs256("", appKey));
+
+        httplib::SSLClient cli("api.github.com");
+        cli.set_bearer_token_auth(token.c_str());
+        cli.set_default_headers({
+            {"User-Agent", "ECL-Compiler/1.0"},
+            {"Accept", "application/vnd.github.v3+json"}
+        });
+
+        // Generate Access Token
+        std::string tokenPath = "/app/installations/";
+        tokenPath += installationId;
+        tokenPath += "/access_tokens";
+        auto postRes = cli.Post(tokenPath.c_str());
+        if (!postRes || postRes->status != 201)
+        {
+            DBGLOG("Failed to generate GitHub IAT. HTTP status: %d", postRes ? postRes->status : -1);
+            return "";
+        }
+
+        auto tokenJson = nlohmann::json::parse(postRes->body);
+        return tokenJson["token"].get<std::string>();
+    }
+    catch (const std::exception& e)
+    {
+        DBGLOG("Exception generating GitHub IAT: %s", e.what());
+    }
+
+    return "";
+}
+
+IFile * getFileWithGitAccessToken(const char * gitUser)
+{
+    Owned<const IPropertyTree> secret = getSecret("git", gitUser);
+    if (!secret)
+    {
+        DBGLOG("No secret found for git user %s", gitUser);
+        return nullptr;
+    }
+
+    MemoryBuffer gitKey;
+    if (getSecretKeyValue(gitKey, secret, "password"))
+        return writeToProtectedTempFile("eclcc", "git", gitKey.length(), gitKey.toByteArray());
+
+    StringBuffer appId, appKey, installationId;
+    if (getSecretKeyValue(appId, secret, "appid") && getSecretKeyValue(appKey, secret, "appkey") && getSecretKeyValue(installationId, secret, "installationid"))
+    {
+        // Generate an Installation Access Token (IAT) from the App ID, Private Key, and Installation ID
+        std::string iat = generateGithubIAT(appId.str(), appKey.str(), installationId.str());
+        if (!iat.empty())
+            return writeToProtectedTempFile("eclcc", "git", iat.length(), iat.c_str());
+        return nullptr;
+    }
+
+    DBGLOG("Secret doesn't contain password or github app credentials for git user %s", gitUser);
+    return nullptr;
+}

system/jlib/jsecrets.hpp

@@ -21,6 +21,7 @@
 
 #include "jlib.hpp"
 #include "jstring.hpp"
+#include <string>
 
 interface ISyncedPropertyTree;
 
@@ -71,6 +72,9 @@ constexpr static unsigned defaultMaskingPercentage = 90;
 //creates a masked version of a secret
 extern jlib_decl void maskSecret(StringBuffer & maskedSecret, const char * originalSecret, unsigned maskPercentage = defaultMaskingPercentage, bool maskRightSide = true, char maskChar = defaultMaskChar);
 
+extern jlib_decl std::string generateGithubIAT(const char * appId, const char * appKey, const char * installationId);
+extern jlib_decl IFile * getFileWithGitAccessToken(const char * gitUser);
+
 #ifdef _USE_CPPUNIT
 extern jlib_decl std::string testBuildSecretKey(const char * category, const char * name, const char * optVaultId, const char * optVersion);
 extern jlib_decl void testExpandSecretKey(std::string & category, std::string & name, std::string & optVaultId, std::string & optVersion, const char * key);