feat: private/custom git repos - GitHub, GitLab, Bitbucket, custom

- Add repository URL validation for GitHub, GitLab, Bitbucket, and custom hosts
- Add git provider layer (listDirectory, downloadRawFile) for all providers
- Wire githubJsonService and scriptDownloader to use provider; sync/download from any supported source
- Update GeneralSettingsModal placeholder and help text; .env.example and env schema for GITLAB_TOKEN, BITBUCKET_APP_PASSWORD

Closes #406

Author: michelroegl-brunner

.env.example

@@ -18,7 +18,12 @@ ALLOWED_SCRIPT_PATHS="scripts/"
 WEBSOCKET_PORT="3001"
 
 # User settings
+# Optional tokens for private repos: GITHUB_TOKEN (GitHub), GITLAB_TOKEN (GitLab),
+# BITBUCKET_APP_PASSWORD or BITBUCKET_TOKEN (Bitbucket). REPO_URL and added repos
+# can be GitHub, GitLab, Bitbucket, or custom Git servers.
 GITHUB_TOKEN=
+GITLAB_TOKEN=
+BITBUCKET_APP_PASSWORD=
 SAVE_FILTER=false
 FILTERS=
 AUTH_USERNAME=

src/app/_components/GeneralSettingsModal.tsx

@@ -1617,7 +1617,7 @@ export function GeneralSettingsModal({
                       <Input
                         id="new-repo-url"
                         type="url"
-                        placeholder="https://github.com/owner/repo"
+                        placeholder="https://github.com/owner/repo or https://git.example.com/owner/repo"
                         value={newRepoUrl}
                         onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
                           setNewRepoUrl(e.target.value)
@@ -1626,8 +1626,9 @@ export function GeneralSettingsModal({
                         className="w-full"
                       />
                       <p className="text-muted-foreground mt-1 text-xs">
-                        Enter a GitHub repository URL (e.g.,
-                        https://github.com/owner/repo)
+                        Supported: GitHub, GitLab, Bitbucket, or custom Git
+                        servers (e.g. https://github.com/owner/repo,
+                        https://gitlab.com/owner/repo)
                       </p>
                     </div>
                     <div className="border-border flex items-center justify-between gap-3 rounded-lg border p-3">

src/env.js

@@ -23,8 +23,11 @@ export const env = createEnv({
     ALLOWED_SCRIPT_PATHS: z.string().default("scripts/"),
     // WebSocket Configuration
     WEBSOCKET_PORT: z.string().default("3001"),
-    // GitHub Configuration
+    // Git provider tokens (optional, for private repos)
     GITHUB_TOKEN: z.string().optional(),
+    GITLAB_TOKEN: z.string().optional(),
+    BITBUCKET_APP_PASSWORD: z.string().optional(),
+    BITBUCKET_TOKEN: z.string().optional(),
     // Authentication Configuration
     AUTH_USERNAME: z.string().optional(),
     AUTH_PASSWORD_HASH: z.string().optional(),
@@ -62,8 +65,10 @@ export const env = createEnv({
     ALLOWED_SCRIPT_PATHS: process.env.ALLOWED_SCRIPT_PATHS,
     // WebSocket Configuration
     WEBSOCKET_PORT: process.env.WEBSOCKET_PORT,
-    // GitHub Configuration
     GITHUB_TOKEN: process.env.GITHUB_TOKEN,
+    GITLAB_TOKEN: process.env.GITLAB_TOKEN,
+    BITBUCKET_APP_PASSWORD: process.env.BITBUCKET_APP_PASSWORD,
+    BITBUCKET_TOKEN: process.env.BITBUCKET_TOKEN,
     // Authentication Configuration
     AUTH_USERNAME: process.env.AUTH_USERNAME,
     AUTH_PASSWORD_HASH: process.env.AUTH_PASSWORD_HASH,

src/server/lib/gitProvider/bitbucket.ts

@@ -0,0 +1,55 @@
+import type { DirEntry, GitProvider } from './types';
+import { parseRepoUrl } from '../repositoryUrlValidation';
+
+export class BitbucketProvider implements GitProvider {
+  async listDirectory(repoUrl: string, path: string, branch: string): Promise<DirEntry[]> {
+    const { owner, repo } = parseRepoUrl(repoUrl);
+    const listUrl = `https://api.bitbucket.org/2.0/repositories/${owner}/${repo}/src/${encodeURIComponent(branch)}/${path}`;
+    const headers: Record<string, string> = {
+      'User-Agent': 'PVEScripts-Local/1.0',
+    };
+    const token = process.env.BITBUCKET_APP_PASSWORD ?? process.env.BITBUCKET_TOKEN;
+    if (token) {
+      const auth = Buffer.from(`:${token}`).toString('base64');
+      headers.Authorization = `Basic ${auth}`;
+    }
+
+    const response = await fetch(listUrl, { headers });
+    if (!response.ok) {
+      throw new Error(`Bitbucket API error: ${response.status} ${response.statusText}`);
+    }
+
+    const body = (await response.json()) as { values?: { path: string; type: string }[] };
+    const data = body.values ?? (Array.isArray(body) ? body : []);
+    if (!Array.isArray(data)) {
+      throw new Error('Bitbucket API returned unexpected response');
+    }
+    return data.map((item: { path: string; type: string }) => {
+      const name = item.path.split('/').pop() ?? item.path;
+      return {
+        name,
+        path: item.path,
+        type: item.type === 'commit_directory' ? ('dir' as const) : ('file' as const),
+      };
+    });
+  }
+
+  async downloadRawFile(repoUrl: string, filePath: string, branch: string): Promise<string> {
+    const { owner, repo } = parseRepoUrl(repoUrl);
+    const rawUrl = `https://api.bitbucket.org/2.0/repositories/${owner}/${repo}/src/${encodeURIComponent(branch)}/${filePath}`;
+    const headers: Record<string, string> = {
+      'User-Agent': 'PVEScripts-Local/1.0',
+    };
+    const token = process.env.BITBUCKET_APP_PASSWORD ?? process.env.BITBUCKET_TOKEN;
+    if (token) {
+      const auth = Buffer.from(`:${token}`).toString('base64');
+      headers.Authorization = `Basic ${auth}`;
+    }
+
+    const response = await fetch(rawUrl, { headers });
+    if (!response.ok) {
+      throw new Error(`Failed to download ${filePath}: ${response.status} ${response.statusText}`);
+    }
+    return response.text();
+  }
+}

src/server/lib/gitProvider/custom.ts

@@ -0,0 +1,44 @@
+import type { DirEntry, GitProvider } from "./types";
+import { parseRepoUrl } from "../repositoryUrlValidation";
+
+export class CustomProvider implements GitProvider {
+  async listDirectory(repoUrl: string, path: string, branch: string): Promise<DirEntry[]> {
+    const { origin, owner, repo } = parseRepoUrl(repoUrl);
+    const apiUrl = `${origin}/api/v1/repos/${owner}/${repo}/contents/${path}?ref=${encodeURIComponent(branch)}`;
+    const headers: Record<string, string> = { "User-Agent": "PVEScripts-Local/1.0" };
+    const token = process.env.GITEA_TOKEN ?? process.env.GIT_TOKEN;
+    if (token) headers.Authorization = `token ${token}`;
+
+    const response = await fetch(apiUrl, { headers });
+    if (!response.ok) {
+      throw new Error(`Custom Git server: list directory failed (${response.status}).`);
+    }
+    const data = (await response.json()) as { type: string; name: string; path: string }[];
+    if (!Array.isArray(data)) {
+      const single = data as unknown as { type?: string; name?: string; path?: string };
+      if (single?.name) {
+        return [{ name: single.name, path: single.path ?? path, type: single.type === "dir" ? "dir" : "file" }];
+      }
+      throw new Error("Custom Git server returned unexpected response");
+    }
+    return data.map((item) => ({
+      name: item.name,
+      path: item.path,
+      type: item.type === "dir" ? ("dir" as const) : ("file" as const),
+    }));
+  }
+
+  async downloadRawFile(repoUrl: string, filePath: string, branch: string): Promise<string> {
+    const { origin, owner, repo } = parseRepoUrl(repoUrl);
+    const rawUrl = `${origin}/${owner}/${repo}/raw/${encodeURIComponent(branch)}/${filePath}`;
+    const headers: Record<string, string> = { "User-Agent": "PVEScripts-Local/1.0" };
+    const token = process.env.GITEA_TOKEN ?? process.env.GIT_TOKEN;
+    if (token) headers.Authorization = `token ${token}`;
+
+    const response = await fetch(rawUrl, { headers });
+    if (!response.ok) {
+      throw new Error(`Failed to download ${filePath} from custom Git server (${response.status}).`);
+    }
+    return response.text();
+  }
+}

src/server/lib/gitProvider/github.ts

@@ -0,0 +1,60 @@
+import type { DirEntry, GitProvider } from './types';
+import { parseRepoUrl } from '../repositoryUrlValidation';
+
+export class GitHubProvider implements GitProvider {
+  async listDirectory(repoUrl: string, path: string, branch: string): Promise<DirEntry[]> {
+    const { owner, repo } = parseRepoUrl(repoUrl);
+    const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}?ref=${encodeURIComponent(branch)}`;
+    const headers: Record<string, string> = {
+      Accept: 'application/vnd.github.v3+json',
+      'User-Agent': 'PVEScripts-Local/1.0',
+    };
+    const token = process.env.GITHUB_TOKEN;
+    if (token) headers.Authorization = `token ${token}`;
+
+    const response = await fetch(apiUrl, { headers });
+    if (!response.ok) {
+      if (response.status === 403) {
+        const err = new Error(
+          `GitHub API rate limit exceeded. Consider setting GITHUB_TOKEN. Status: ${response.status} ${response.statusText}`
+        );
+        (err as Error & { name: string }).name = 'RateLimitError';
+        throw err;
+      }
+      throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = (await response.json()) as { type: string; name: string; path: string }[];
+    if (!Array.isArray(data)) {
+      throw new Error('GitHub API returned unexpected response');
+    }
+    return data.map((item) => ({
+      name: item.name,
+      path: item.path,
+      type: item.type === 'dir' ? ('dir' as const) : ('file' as const),
+    }));
+  }
+
+  async downloadRawFile(repoUrl: string, filePath: string, branch: string): Promise<string> {
+    const { owner, repo } = parseRepoUrl(repoUrl);
+    const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${encodeURIComponent(branch)}/${filePath}`;
+    const headers: Record<string, string> = {
+      'User-Agent': 'PVEScripts-Local/1.0',
+    };
+    const token = process.env.GITHUB_TOKEN;
+    if (token) headers.Authorization = `token ${token}`;
+
+    const response = await fetch(rawUrl, { headers });
+    if (!response.ok) {
+      if (response.status === 403) {
+        const err = new Error(
+          `GitHub rate limit exceeded while downloading ${filePath}. Consider setting GITHUB_TOKEN.`
+        );
+        (err as Error & { name: string }).name = 'RateLimitError';
+        throw err;
+      }
+      throw new Error(`Failed to download ${filePath}: ${response.status} ${response.statusText}`);
+    }
+    return response.text();
+  }
+}

src/server/lib/gitProvider/gitlab.ts

@@ -0,0 +1,58 @@
+import type { DirEntry, GitProvider } from './types';
+import { parseRepoUrl } from '../repositoryUrlValidation';
+
+export class GitLabProvider implements GitProvider {
+  private getBaseUrl(repoUrl: string): string {
+    const { origin } = parseRepoUrl(repoUrl);
+    return origin;
+  }
+
+  private getProjectId(repoUrl: string): string {
+    const { owner, repo } = parseRepoUrl(repoUrl);
+    return encodeURIComponent(`${owner}/${repo}`);
+  }
+
+  async listDirectory(repoUrl: string, path: string, branch: string): Promise<DirEntry[]> {
+    const baseUrl = this.getBaseUrl(repoUrl);
+    const projectId = this.getProjectId(repoUrl);
+    const apiUrl = `${baseUrl}/api/v4/projects/${projectId}/repository/tree?path=${encodeURIComponent(path)}&ref=${encodeURIComponent(branch)}&per_page=100`;
+    const headers: Record<string, string> = {
+      'User-Agent': 'PVEScripts-Local/1.0',
+    };
+    const token = process.env.GITLAB_TOKEN;
+    if (token) headers['PRIVATE-TOKEN'] = token;
+
+    const response = await fetch(apiUrl, { headers });
+    if (!response.ok) {
+      throw new Error(`GitLab API error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = (await response.json()) as { type: string; name: string; path: string }[];
+    if (!Array.isArray(data)) {
+      throw new Error('GitLab API returned unexpected response');
+    }
+    return data.map((item) => ({
+      name: item.name,
+      path: item.path,
+      type: item.type === 'tree' ? ('dir' as const) : ('file' as const),
+    }));
+  }
+
+  async downloadRawFile(repoUrl: string, filePath: string, branch: string): Promise<string> {
+    const baseUrl = this.getBaseUrl(repoUrl);
+    const projectId = this.getProjectId(repoUrl);
+    const encodedPath = encodeURIComponent(filePath);
+    const rawUrl = `${baseUrl}/api/v4/projects/${projectId}/repository/files/${encodedPath}/raw?ref=${encodeURIComponent(branch)}`;
+    const headers: Record<string, string> = {
+      'User-Agent': 'PVEScripts-Local/1.0',
+    };
+    const token = process.env.GITLAB_TOKEN;
+    if (token) headers['PRIVATE-TOKEN'] = token;
+
+    const response = await fetch(rawUrl, { headers });
+    if (!response.ok) {
+      throw new Error(`Failed to download ${filePath}: ${response.status} ${response.statusText}`);
+    }
+    return response.text();
+  }
+}

src/server/lib/gitProvider/index.js

@@ -0,0 +1 @@
+export { listDirectory, downloadRawFile, getRepoProvider } from "./index.ts";

src/server/lib/gitProvider/index.ts

@@ -0,0 +1,28 @@
+import type { DirEntry, GitProvider } from "./types";
+import { getRepoProvider } from "../repositoryUrlValidation";
+import { GitHubProvider } from "./github";
+import { GitLabProvider } from "./gitlab";
+import { BitbucketProvider } from "./bitbucket";
+import { CustomProvider } from "./custom";
+
+const providers: Record<string, GitProvider> = {
+  github: new GitHubProvider(),
+  gitlab: new GitLabProvider(),
+  bitbucket: new BitbucketProvider(),
+  custom: new CustomProvider(),
+};
+
+export type { DirEntry, GitProvider };
+export { getRepoProvider };
+
+export function getGitProvider(repoUrl: string): GitProvider {
+  return providers[getRepoProvider(repoUrl)]!;
+}
+
+export async function listDirectory(repoUrl: string, path: string, branch: string): Promise<DirEntry[]> {
+  return getGitProvider(repoUrl).listDirectory(repoUrl, path, branch);
+}
+
+export async function downloadRawFile(repoUrl: string, filePath: string, branch: string): Promise<string> {
+  return getGitProvider(repoUrl).downloadRawFile(repoUrl, filePath, branch);
+}

src/server/lib/gitProvider/types.ts

@@ -0,0 +1,14 @@
+/**
+ * Git provider interface for listing and downloading repository files.
+ */
+
+export type DirEntry = {
+  name: string;
+  path: string;
+  type: 'file' | 'dir';
+};
+
+export interface GitProvider {
+  listDirectory(repoUrl: string, path: string, branch: string): Promise<DirEntry[]>;
+  downloadRawFile(repoUrl: string, filePath: string, branch: string): Promise<string>;
+}