From 49fb4b8b8b88577cdaa7e8dbb22159f95929fe3f Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 8 Apr 2026 11:48:32 +0100 Subject: [PATCH] HPCC-35979 feat(eclccserver) Add github app support to eclccserver Signed-off-by: Gavin Halliday --- common/remote/hooks/git/gitfile.cpp | 17 ++---- devdoc/githubapps.md | 49 +++++++++++++++++ ecl/hql/hqlrepository.cpp | 27 ++-------- system/jlib/CMakeLists.txt | 4 ++ system/jlib/jsecrets.cpp | 81 +++++++++++++++++++++++++++++ system/jlib/jsecrets.hpp | 4 ++ 6 files changed, 144 insertions(+), 38 deletions(-) create mode 100644 devdoc/githubapps.md diff --git a/common/remote/hooks/git/gitfile.cpp b/common/remote/hooks/git/gitfile.cpp index 99c35fc1205..32e4ae4bd63 100644 --- a/common/remote/hooks/git/gitfile.cpp +++ b/common/remote/hooks/git/gitfile.cpp @@ -250,20 +250,9 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf addPathSepChar(scriptPath).append("bin/hpccaskpass.sh"); env.emplace_back("GIT_ASKPASS", scriptPath); - Owned secret = getSecret("git", gitUser); - if (secret) - { - MemoryBuffer gitKey; - if (getSecretKeyValue(gitKey, secret, "password")) - { - extractedKey.setown(writeToProtectedTempFile("eclcc", "git", gitKey.length(), gitKey.toByteArray())); - env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename()); - } - else - OWARNLOG("Secret doesn't contain password for git user %s", gitUser); - } - else - OWARNLOG("No secret found for git user %s", gitUser); + extractedKey.setown(getFileWithGitAccessToken(gitUser)); + if (extractedKey) + env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename()); } Owned pipe = createPipeProcess(); for (const auto & cur : env) diff --git a/devdoc/githubapps.md b/devdoc/githubapps.md new file mode 100644 index 00000000000..5d7ae473c7b --- /dev/null +++ b/devdoc/githubapps.md @@ -0,0 +1,49 @@ +# Using GitHub apps for authenticating from eclccserver + +## Creating and Configuring a GitHub App for eclccserver + +Here is a step-by-step guide to creating and configuring a GitHub App to use with the authentication mechanism. + +### Step 1: Create the GitHub App +1. Go to your GitHub account (or Organization) **Settings**. +2. On the left sidebar, scroll down and click **Developer settings**. +3. Select **GitHub Apps** in the left sidebar, then click the **New GitHub App** button. +4. Fill out the **Register new GitHub App** form: + * **GitHub App name**: Give it a recognizable name (e.g., `HPCC ECL Fetcher`). + * **Homepage URL**: You can use your organization's URL or the HPCC platform repository URL (this is just required by the form). + * **Webhook**: Uncheck the **Active** checkbox. (HPCC only needs to pull code; it doesn't need to listen for GitHub events). +5. Set the **Repository permissions**: + * Under **Repository permissions**, find **Contents** and set the access to **Read-only**. (This gives the app permission to perform `git fetch` / `git clone`). +6. Scroll to the bottom (**Where can this GitHub App be installed?**): + * Select **Any account** if you plan to fetch repositories owned by different organizations/users. Select **Only on this account** if you exclusively fetch repositories within your current organization. +7. Click **Create GitHub App**. + +### Step 2: Gather Your Secrets +Immediately after creating the app, you will land on its settings page. You need to collect three items to populate your HPCC secrets configuration: + +1. **The App ID** (`appid`): + * Look at the "About" section at the top of the page. Copy the **App ID**. +2. **The Private Key** (`appkey`): + * Scroll down to the **Private keys** section. + * Click **Generate a private key**. + * This will download a `.pem` file to your computer. The entire contents of this file (including the `-----BEGIN RSA PRIVATE KEY-----` and end tags) will be your `appkey` secret. + +### Step 3: Install the App & Get the Installation ID +The App ID and Private Key alone are not enough; the app must be *installed* on the target account/repository to generate the `installationid`. + +1. While still on the App's settings page, click **Install App** on the left sidebar. +2. Click **Install** next to the user or organization account where the target ECL repositories live. +3. Choose whether to install it on **All repositories** or **Only select repositories**, then click **Install**. +4. You will be redirected to the installation settings page. Look at the URL in your browser's address bar. It will look like this: + `https://github.com/settings/installations/12345678` + or + `https://github.com/organizations/YOUR_ORG/settings/installations/12345678` +5. Copy the numeric value at the end of the URL (e.g., `12345678`). This is your **Installation ID** (`installationid`). + +## Configuring the secrets for eclccserver + +1. configure the git user name to be x-access-token + +2. Create a secret category "git", secret "x-access-token" + +3. Place these three values highlighted above (`appid`, `appkey`, and `installationid`) into your HPCC secret. diff --git a/ecl/hql/hqlrepository.cpp b/ecl/hql/hqlrepository.cpp index 0f959122fa9..27f0a66ae9d 100644 --- a/ecl/hql/hqlrepository.cpp +++ b/ecl/hql/hqlrepository.cpp @@ -1008,33 +1008,12 @@ unsigned EclRepositoryManager::runGitCommand(StringBuffer * output, const char * // If gituser is specified never prompt for credentials, otherwise the server can hang. env.emplace_back("GIT_TERMINAL_PROMPT", "0"); - if (!options.gitPasswordPath.isEmpty()) + extractedKey.setown(getFileWithGitAccessToken(options.gitUser.str())); + if (extractedKey) { - //Convert to an absolute path, and check the file exists, because git will be run in a different directory - StringBuffer absolutePath; - makeAbsolutePath(options.gitPasswordPath.str(), absolutePath, true); - - env.emplace_back("HPCC_GIT_PASSPATH", absolutePath); + env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename()); useScript = true; } - else - { - Owned secret = getSecret("git", options.gitUser.str()); - if (secret) - { - MemoryBuffer gitKey; - if (!getSecretKeyValue(gitKey, secret, "password")) - DBGLOG("Secret doesn't contain password for git user %s", options.gitUser.str()); - else - { - extractedKey.setown(writeToProtectedTempFile("eclcc", "git", gitKey.length(), gitKey.toByteArray())); - env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename()); - useScript = true; - } - } - else - DBGLOG("No secret found for git user %s", options.gitUser.str()); - } } if (useScript) diff --git a/system/jlib/CMakeLists.txt b/system/jlib/CMakeLists.txt index 03dcac7225d..ed98cc72cbe 100644 --- a/system/jlib/CMakeLists.txt +++ b/system/jlib/CMakeLists.txt @@ -56,6 +56,9 @@ if (NOT WIN32 AND NOT WIN64 AND NOT APPLE AND NOT EMSCRIPTEN) pkg_check_modules(liburing REQUIRED IMPORTED_TARGET liburing>=2.0) endif () +find_package(CURL REQUIRED) +find_path(JWT_CPP_INCLUDE_DIRS "jwt-cpp/base.h" HINTS "${VCPKG_INSTALLED_DIR}") + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STRICT_CXX_FLAGS}") set ( SRCS @@ -212,6 +215,7 @@ set ( INCLUDES ${HPCC_SOURCE_DIR}/system/security/cryptohelper/digisign.cpp ${HPCC_SOURCE_DIR}/system/security/cryptohelper/pke.cpp ${HPCC_SOURCE_DIR}/system/security/cryptohelper/ske.cpp + ${JWT_CPP_INCLUDE_DIRS}/jwt-cpp/jwt.h ) set_source_files_properties(jmd5.cpp jsort.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index 4fcba49322c..0da19c4d588 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -49,6 +49,20 @@ #pragma GCC diagnostic pop #endif +#ifdef verify + #define JWT_HAS_VERIFY_MACRO + #define OLD_VERIFY verify + #undef verify +#endif + +#include "jwt-cpp/jwt.h" +#include "nlohmann/json.hpp" + +#ifdef JWT_HAS_VERIFY_MACRO + //Force an error if verify macro is used in any of the following code + #undef verify +#endif + #ifdef _USE_OPENSSL #include #include @@ -2427,3 +2441,70 @@ void maskSecret(StringBuffer & maskedSecret, const char * originalSecret, unsign maskedSecret.append(originalSecret[i]); } } + +std::string generateGithubIAT(const char * appId, const char * appKey, const char * installationId) +{ + try + { + auto now = std::chrono::system_clock::now(); + auto token = jwt::create() + .set_issuer(appId) + .set_issued_at(now) + .set_expires_at(now + std::chrono::minutes(10)) + .sign(jwt::algorithm::rs256("", appKey)); + + httplib::SSLClient cli("api.github.com"); + cli.set_bearer_token_auth(token.c_str()); + cli.set_default_headers({ + {"User-Agent", "ECL-Compiler/1.0"}, + {"Accept", "application/vnd.github.v3+json"} + }); + + // Generate Access Token + std::string tokenPath = "/app/installations/"; + tokenPath += installationId; + tokenPath += "/access_tokens"; + auto postRes = cli.Post(tokenPath.c_str()); + if (!postRes || postRes->status != 201) + { + DBGLOG("Failed to generate GitHub IAT. HTTP status: %d", postRes ? postRes->status : -1); + return ""; + } + + auto tokenJson = nlohmann::json::parse(postRes->body); + return tokenJson["token"].get(); + } + catch (const std::exception& e) + { + DBGLOG("Exception generating GitHub IAT: %s", e.what()); + } + + return ""; +} + +IFile * getFileWithGitAccessToken(const char * gitUser) +{ + Owned secret = getSecret("git", gitUser); + if (!secret) + { + DBGLOG("No secret found for git user %s", gitUser); + return nullptr; + } + + MemoryBuffer gitKey; + if (getSecretKeyValue(gitKey, secret, "password")) + return writeToProtectedTempFile("eclcc", "git", gitKey.length(), gitKey.toByteArray()); + + StringBuffer appId, appKey, installationId; + if (getSecretKeyValue(appId, secret, "appid") && getSecretKeyValue(appKey, secret, "appkey") && getSecretKeyValue(installationId, secret, "installationid")) + { + // Generate an Installation Access Token (IAT) from the App ID, Private Key, and Installation ID + std::string iat = generateGithubIAT(appId.str(), appKey.str(), installationId.str()); + if (!iat.empty()) + return writeToProtectedTempFile("eclcc", "git", iat.length(), iat.c_str()); + return nullptr; + } + + DBGLOG("Secret doesn't contain password or github app credentials for git user %s", gitUser); + return nullptr; +} diff --git a/system/jlib/jsecrets.hpp b/system/jlib/jsecrets.hpp index 286fa5c8291..5c8ce677720 100644 --- a/system/jlib/jsecrets.hpp +++ b/system/jlib/jsecrets.hpp @@ -21,6 +21,7 @@ #include "jlib.hpp" #include "jstring.hpp" +#include interface ISyncedPropertyTree; @@ -71,6 +72,9 @@ constexpr static unsigned defaultMaskingPercentage = 90; //creates a masked version of a secret extern jlib_decl void maskSecret(StringBuffer & maskedSecret, const char * originalSecret, unsigned maskPercentage = defaultMaskingPercentage, bool maskRightSide = true, char maskChar = defaultMaskChar); +extern jlib_decl std::string generateGithubIAT(const char * appId, const char * appKey, const char * installationId); +extern jlib_decl IFile * getFileWithGitAccessToken(const char * gitUser); + #ifdef _USE_CPPUNIT extern jlib_decl std::string testBuildSecretKey(const char * category, const char * name, const char * optVaultId, const char * optVersion); extern jlib_decl void testExpandSecretKey(std::string & category, std::string & name, std::string & optVaultId, std::string & optVersion, const char * key);