diff --git a/packages/app/app/pages/index.vue b/packages/app/app/pages/index.vue index cee70513..91055957 100644 --- a/packages/app/app/pages/index.vue +++ b/packages/app/app/pages/index.vue @@ -4,6 +4,148 @@ definePageMeta({ mainClass: "", }); +import { ref, onMounted } from "vue"; + +// Define types for the repository and log data +interface RepositoryOwner { + id?: string; + login: string; + avatar_url: string; +} + +interface LatestCommit { + sha: string; + message: string; + date: string; +} + +interface Repository { + id: string; + name: string; + owner: RepositoryOwner; + full_name: string; + description: string | null; + default_branch: string; + html_url: string; + homepage?: string | null; + stargazers_count: number; + watchers_count: number; + forks_count: number; + open_issues_count?: number; + indexed_at: string; + latest_commit: LatestCommit | null; +} + +interface LogEntry { + timestamp: string; + message: string; + type: "info" | "error" | "success"; + data: any; +} + +// State for repositories and logs +const repositories = ref([]); +const loading = ref(true); +const error = ref(null); +const clientLogs = ref([]); +const showLogs = ref(false); + +// Function to add logs that will be visible in the client +const logToClient = ( + message: string, + type: "info" | "error" | "success" = "info", + data: any = null, +): void => { + const timestamp = new Date().toISOString(); + const logEntry: LogEntry = { + timestamp, + message, + type, + data, + }; + clientLogs.value.unshift(logEntry); + console.log(`[CLIENT-LOG][${type}] ${message}`, data || ""); +}; + +// Fetch all repositories from R2 storage +const fetchRepositories = async (): Promise => { + try { + loading.value = true; + logToClient("Fetching repositories from R2 storage...", "info"); + + const response = await fetch("/api/repos"); + const responseData = (await response.json()) as { + repositories: Repository[]; + error?: boolean; + message?: string; + debug_info?: any; + }; + + if (responseData.error) { + logToClient( + `Error fetching repositories: ${responseData.message || "Unknown error"}`, + "error", + responseData.debug_info, + ); + error.value = responseData.message || "Unknown error"; + loading.value = false; + return; + } + + repositories.value = responseData.repositories; + logToClient( + `Successfully fetched ${responseData.repositories.length} repositories from R2 storage`, + "success", + responseData.debug_info, + ); + + // Log available repositories + repositories.value.forEach((repo) => { + logToClient(`Repository available: ${repo.full_name}`, "info", { + default_branch: repo.default_branch, + latest_commit: repo.latest_commit + ? `${repo.latest_commit.sha.substring(0, 7)} - ${repo.latest_commit.message.split("\n")[0]}` + : "No commits found", + }); + }); + } catch (e: any) { + logToClient(`Failed to fetch repositories: ${e.message}`, "error"); + error.value = `Failed to fetch repositories: ${e.message}`; + } finally { + loading.value = false; + } +}; + +// Truncate long texts +const truncate = (text: string, length = 60): string => { + if (!text) return ""; + return text.length > length ? text.substring(0, length) + "..." : text; +}; + +// Format date +const formatDate = (dateString: string): string => { + if (!dateString) return "Unknown"; + const date = new Date(dateString); + return date.toLocaleString(); +}; + +// Toggle logs visibility +const toggleLogs = (): void => { + showLogs.value = !showLogs.value; +}; + +// Clear logs +const clearLogs = (): void => { + clientLogs.value = []; + logToClient("Logs cleared", "info"); +}; + +// Load repositories on mount +onMounted(() => { + fetchRepositories(); +}); + +// Template refs const gettingStartedEl = useTemplateRef("getting-started"); function scrollToGettingStarted() { @@ -16,13 +158,203 @@ function scrollToGettingStarted() { + + diff --git a/packages/app/nuxt.config.ts b/packages/app/nuxt.config.ts index e166f71d..1b607347 100644 --- a/packages/app/nuxt.config.ts +++ b/packages/app/nuxt.config.ts @@ -51,8 +51,8 @@ export default defineNuxtConfig({ webhookSecret: process.env.NITRO_WEBHOOK_SECRET || "", privateKey: process.env.NITRO_PRIVATE_KEY || "", rmStaleKey: process.env.NITRO_RM_STALE_KEY || "", - githubToken: - process.env.GITHUB_TOKEN || process.env.NITRO_GITHUB_TOKEN || "", + // githubToken: + // process.env.GITHUB_TOKEN || process.env.NITRO_GITHUB_TOKEN || "", ghBaseUrl: process.env.NITRO_GH_BASE_URL || "https://api.github.com", test: process.env.NITRO_TEST || "", }, diff --git a/packages/app/server/api/repo/commits.get.ts b/packages/app/server/api/repo/commits.get.ts index 4e32b69b..45769b29 100644 --- a/packages/app/server/api/repo/commits.get.ts +++ b/packages/app/server/api/repo/commits.get.ts @@ -1,148 +1,98 @@ import { z } from "zod"; -import { useGithubREST } from "../../../server/utils/octokit"; +import { useR2GitHubService } from "../../../server/utils/r2-service"; const querySchema = z.object({ owner: z.string(), repo: z.string(), - cursor: z.string().optional(), - page: z.string().optional(), - per_page: z.string().optional().default("10"), + page: z.coerce.number().default(1), + per_page: z.coerce.number().default(10), }); export default defineEventHandler(async (event) => { try { + console.log("[R2API] Commits endpoint called"); const query = await getValidatedQuery(event, (data) => querySchema.parse(data), ); - const octokit = useGithubREST(event); - const { data: repo } = await octokit.request("GET /repos/{owner}/{repo}", { - owner: query.owner, - repo: query.repo, - }); - - const defaultBranch = repo.default_branch; + console.log( + `[R2API] Fetching commits for ${query.owner}/${query.repo} (page=${query.page}, per_page=${query.per_page})`, + ); - const page = query.page - ? Number.parseInt(query.page) - : query.cursor - ? Number.parseInt(query.cursor) - : 1; - const per_page = Number.parseInt(query.per_page); + // Use R2 service exclusively + const r2Service = useR2GitHubService(event); - const { data: commits } = await octokit.request( - "GET /repos/{owner}/{repo}/commits", - { - owner: query.owner, - repo: query.repo, - sha: defaultBranch, - page, - per_page, - }, + // Log storage configuration for debugging + console.log( + `[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`, ); - const commitsWithStatuses = await Promise.all( - commits.map(async (commit) => { - try { - const { data: checkRuns } = await octokit.request( - "GET /repos/{owner}/{repo}/commits/{ref}/check-runs", - { - owner: query.owner, - repo: query.repo, - ref: commit.sha, - }, - ); + // Verify repository exists first + const repository = await r2Service.getRepository(query.owner, query.repo); + if (!repository) { + console.log( + `[R2API] Repository ${query.owner}/${query.repo} not found in R2 storage`, + ); + throw new Error( + `Repository ${query.owner}/${query.repo} not found in R2 storage`, + ); + } - return { - id: commit.node_id || commit.sha, - oid: commit.sha, - abbreviatedOid: commit.sha.substring(0, 7), - message: commit.commit.message, - authoredDate: commit.commit.author?.date || "", - url: commit.html_url, - statusCheckRollup: - checkRuns.check_runs.length > 0 - ? { - id: `status-${commit.sha}`, - state: checkRuns.check_runs.some( - (check) => check.conclusion === "failure", - ) - ? "FAILURE" - : checkRuns.check_runs.some( - (check) => check.conclusion === "success", - ) - ? "SUCCESS" - : "PENDING", - contexts: { - nodes: checkRuns.check_runs.map((check) => ({ - id: check.id.toString(), - status: check.status, - name: check.name, - title: check.name, - summary: check.output?.summary || "", - text: check.output?.text || "", - detailsUrl: check.details_url || "", - url: check.url || check.html_url || "", - })), - }, - } - : null, - }; - } catch (error) { - console.warn( - `Could not fetch check runs for commit ${commit.sha}:`, - error, - ); - return { - id: commit.node_id || commit.sha, - oid: commit.sha, - abbreviatedOid: commit.sha.substring(0, 7), - message: commit.commit.message, - authoredDate: commit.commit.author?.date || "", - url: commit.html_url, - statusCheckRollup: null, - }; - } - }), + console.log( + `[R2API] Found repository in R2: ${query.owner}/${query.repo} (id: ${repository.id})`, + ); + + // Get commits from R2 storage + console.log( + `[R2API] Fetching commits from R2 storage for ${query.owner}/${query.repo}`, + ); + const commits = await r2Service.listCommits( + query.owner, + query.repo, + query.page, + query.per_page, ); - // Check if there are more commits (GitHub API doesn't provide this directly) - // We'll need to check if we got a full page of results - const hasNextPage = commits.length === per_page; - const nextPage = hasNextPage ? (page + 1).toString() : null; + if (commits.length === 0) { + console.log( + `[R2API] No commits found for ${query.owner}/${query.repo} in R2 storage`, + ); + return { + data: [], + message: `No commits found for ${query.owner}/${query.repo} in storage`, + }; + } - return { - id: `branch-${defaultBranch}`, - name: defaultBranch, - target: { - id: `target-${defaultBranch}`, - history: { - nodes: commitsWithStatuses, - pageInfo: { - hasNextPage, - endCursor: nextPage, - }, + console.log(`[R2API] Found ${commits.length} commits in R2 storage`); + + // Format response with detailed logging + const responseData = commits.map((commit) => { + console.log( + `[R2API] Processing commit: ${commit.sha.substring(0, 7)} - "${commit.commit.message.split("\n")[0]}"`, + ); + + // For each commit, check if we have check runs + return { + sha: commit.sha, + commit: { + message: commit.commit.message, + author: commit.commit.author, }, - }, - }; + html_url: commit.html_url, + author: null, // In R2 storage we might not have author details + r2_source: true, // Flag to indicate source of data + indexed_at: commit.indexed_at || null, + }; + }); + + console.log(`[R2API] Returning ${responseData.length} commits`); + return { data: responseData }; } catch (error) { - console.error("Error fetching repository commits:", error); + console.error("[R2API] Error in commits endpoint:", error); - return { - id: "error", - name: "error", - error: true, - message: (error as Error).message, - target: { - id: "error-target", - history: { - nodes: [], - pageInfo: { - hasNextPage: false, - endCursor: null, - }, - }, - }, - }; + throw createError({ + statusCode: 404, + statusMessage: `Commits could not be accessed: ${error instanceof Error ? error.message : "Unknown error"}`, + }); } }); diff --git a/packages/app/server/api/repo/index.get.ts b/packages/app/server/api/repo/index.get.ts index 50269d5f..711ece9d 100644 --- a/packages/app/server/api/repo/index.get.ts +++ b/packages/app/server/api/repo/index.get.ts @@ -1,71 +1,75 @@ -import type { H3Event } from "h3"; +import { useR2GitHubService } from "../../../server/utils/r2-service"; import { z } from "zod"; -import { useGithubREST } from "../../../server/utils/octokit"; const querySchema = z.object({ owner: z.string(), repo: z.string(), }); -const getRepoInfo = defineCachedFunction( - async (owner: string, repo: string, event?: H3Event) => { - try { - const octokit = useGithubREST(event); +export default defineEventHandler(async (event) => { + try { + console.log("[R2API] Repository index endpoint called"); + + const query = await getValidatedQuery(event, (data) => + querySchema.parse(data), + ); + console.log( + `[R2API] Fetching repository data for ${query.owner}/${query.repo}`, + ); + + // Use R2 service exclusively + const r2Service = useR2GitHubService(event); + + // Log storage info for debugging + console.log( + `[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`, + ); - const { data } = await octokit.request("GET /repos/{owner}/{repo}", { - owner, - repo, - }); + // Get repository from R2 + const repository = await r2Service.getRepository(query.owner, query.repo); - return { - id: data.id.toString(), - name: data.name, - owner: { - id: data.owner.id.toString(), - avatarUrl: data.owner.avatar_url, - login: data.owner.login, - }, - url: data.html_url, - homepageUrl: data.homepage || "", - description: data.description || "", - }; - } catch (error) { - console.error( - `Error fetching repository info for ${owner}/${repo}:`, - error, + if (!repository) { + console.log( + `[R2API] Repository ${query.owner}/${query.repo} not found in R2 storage`, + ); + throw new Error( + `Repository ${query.owner}/${query.repo} not found in R2 storage`, ); - throw error; } - }, - { - getKey: (owner: string, repo: string, _event?: H3Event) => - `${owner}/${repo}`, - maxAge: 60 * 30, // 30 minutes - swr: true, - }, -); -export default defineEventHandler(async (event) => { - try { - const query = await getValidatedQuery(event, (data) => - querySchema.parse(data), + console.log( + `[R2API] Successfully retrieved repository from R2: ${query.owner}/${query.repo}`, ); - return getRepoInfo(query.owner, query.repo, event); - } catch (error) { - console.error("Error in repo info endpoint:", error); + console.log( + `[R2API] Repository details: id=${repository.id}, default_branch=${repository.default_branch}, indexed_at=${repository.indexed_at}`, + ); + return { - error: true, - message: (error as Error).message, - id: "error", - name: "error", + id: repository.id, + name: repository.name, + full_name: `${repository.owner.login}/${repository.name}`, owner: { - id: "error", - avatarUrl: "", - login: "error", + id: repository.owner.id, + login: repository.owner.login, + avatar_url: repository.owner.avatar_url, }, - url: "", - homepageUrl: "", - description: "Error fetching repository data", + default_branch: repository.default_branch, + description: repository.description, + html_url: repository.html_url, + homepage: repository.homepage || null, + watchers_count: repository.watchers_count, + stargazers_count: repository.stargazers_count, + forks_count: repository.forks_count, + open_issues_count: repository.open_issues_count, + r2_source: true, // Flag to indicate source of data + indexed_at: repository.indexed_at || null, }; + } catch (error) { + console.error("[R2API] Error in repository index endpoint:", error); + + throw createError({ + statusCode: 404, + statusMessage: `Repository not found or could not be accessed: ${error instanceof Error ? error.message : "Unknown error"}`, + }); } }); diff --git a/packages/app/server/api/repo/search.get.ts b/packages/app/server/api/repo/search.get.ts index 144ed81c..a3d07536 100644 --- a/packages/app/server/api/repo/search.get.ts +++ b/packages/app/server/api/repo/search.get.ts @@ -1,6 +1,5 @@ -import type { RestEndpointMethodTypes } from "@octokit/plugin-rest-endpoint-methods"; import { z } from "zod"; -import { useGithubREST } from "../../../server/utils/octokit"; +import { useR2GitHubService } from "../../../server/utils/r2-service"; const querySchema = z.object({ text: z.string(), @@ -8,44 +7,71 @@ const querySchema = z.object({ export default defineEventHandler(async (event) => { try { + console.log("[R2API] Search endpoint called"); const query = await getValidatedQuery(event, (data) => querySchema.parse(data), ); if (!query.text) { + console.log("[R2API] Empty search query, returning empty results"); return { nodes: [] }; } - const octokit = useGithubREST(event); + // Use R2 service exclusively for searching + console.log(`[R2API] Searching repositories with query: "${query.text}"`); + const r2Service = useR2GitHubService(event); - const { data } = await octokit.request("GET /search/repositories", { - q: query.text, - per_page: 10, - }); + // Log R2 storage configuration for debugging + console.log( + `[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`, + ); - return { - nodes: data.items.map( - ( - repo: RestEndpointMethodTypes["search"]["repos"]["response"]["data"]["items"][0], - ) => ({ - id: repo.id.toString(), + // Get search results + const repositories = await r2Service.searchRepositories(query.text, 10); + + console.log( + `[R2API] Found ${repositories.length} repositories matching the query in R2 storage`, + ); + + if (repositories.length === 0) { + console.log( + "[R2API] No repositories found in R2 storage matching the query", + ); + return { + nodes: [], + message: "No matching repositories found in storage", + }; + } + + // Format the response with detailed logging + const formattedResponse = { + nodes: repositories.map((repo) => { + console.log( + `[R2API] Including repository in results: ${repo.owner.login}/${repo.name} (ID: ${repo.id})`, + ); + return { + id: repo.id, name: repo.name, - owner: repo.owner - ? { - id: repo.owner.id.toString(), - login: repo.owner.login, - avatarUrl: repo.owner.avatar_url, - } - : null, - }), - ), + owner: { + id: repo.owner.id, + login: repo.owner.login, + avatarUrl: repo.owner.avatar_url, + }, + }; + }), }; + + console.log( + `[R2API] Search response prepared with ${formattedResponse.nodes.length} repositories`, + ); + return formattedResponse; } catch (error) { - console.error("Error in repository search:", error); + console.error("[R2API] Error in repository search:", error); + return { nodes: [], error: true, - message: (error as Error).message, + message: "Search failed. Only data in storage can be searched.", }; } }); diff --git a/packages/app/server/api/repos.get.ts b/packages/app/server/api/repos.get.ts new file mode 100644 index 00000000..1b7f97cf --- /dev/null +++ b/packages/app/server/api/repos.get.ts @@ -0,0 +1,163 @@ +import { useR2GitHubService } from "../utils/r2-service"; + +export default defineEventHandler(async (event) => { + try { + console.log("[R2API] List all repositories endpoint called"); + + // Use R2 service + const r2Service = useR2GitHubService(event); + + // First, dump all storage keys for debugging + console.log("[R2API] Dumping R2 storage structure for debugging"); + const storageKeys = await r2Service.dumpStorageKeys(); + console.log(`[R2API] R2 storage has ${storageKeys.total_keys} total keys`); + + // Get sample values to understand the data structure + const sampleValues = await r2Service.getSampleValues(); + console.log( + `[R2API] Got ${sampleValues.sample_count || 0} sample values from R2`, + ); + + // Try to get all repositories directly from storage keys + console.log("[R2API] Attempting to scan all keys for repositories"); + // Use the getStorageAccess method to access storage safely + const storageAccess = r2Service.getStorageAccess(); + const allKeys = await storageAccess.getKeys(); + const repoKeys = allKeys.filter((key) => key.startsWith("repo:")); + console.log(`[R2API] Found ${repoKeys.length} repository keys`); + + // Directly load repositories from keys + const loadedRepositories = []; + for (const key of repoKeys) { + try { + const data = await storageAccess.getItem(key); + if (data) { + const repo = typeof data === "string" ? JSON.parse(data) : data; + console.log( + `[R2API] Successfully loaded repository from key: ${key}`, + ); + loadedRepositories.push(repo); + } + } catch (error) { + console.error( + `[R2API] Error loading repository from key ${key}:`, + error, + ); + } + } + + console.log( + `[R2API] Directly loaded ${loadedRepositories.length} repositories from keys`, + ); + + // Also try the original method + const repositories = await r2Service.listAllRepositories(); + console.log( + `[R2API] Found ${repositories.length} repositories using listAllRepositories()`, + ); + + // Determine which set of repositories to use + const effectiveRepositories = + loadedRepositories.length > 0 ? loadedRepositories : repositories; + console.log( + `[R2API] Using ${effectiveRepositories.length} repositories for response`, + ); + + // Get latest commit for each repository + const reposWithLatestCommit = await Promise.all( + effectiveRepositories.map(async (repo) => { + try { + const commits = await r2Service.listCommits( + repo.owner.login, + repo.name, + 1, + 1, + ); + const latestCommit = commits.length > 0 ? commits[0] : null; + + return { + id: repo.id, + name: repo.name, + owner: { + login: repo.owner.login, + avatar_url: repo.owner.avatar_url, + }, + full_name: `${repo.owner.login}/${repo.name}`, + description: repo.description, + default_branch: repo.default_branch, + html_url: repo.html_url, + stargazers_count: repo.stargazers_count, + watchers_count: repo.watchers_count, + forks_count: repo.forks_count, + indexed_at: repo.indexed_at, + latest_commit: latestCommit + ? { + sha: latestCommit.sha, + message: latestCommit.commit.message, + date: latestCommit.commit.author.date, + } + : null, + }; + } catch (error) { + console.error( + `[R2API] Error fetching commits for ${repo.owner.login}/${repo.name}:`, + error, + ); + return { + id: repo.id, + name: repo.name, + owner: { + login: repo.owner.login, + avatar_url: repo.owner.avatar_url, + }, + full_name: `${repo.owner.login}/${repo.name}`, + description: repo.description, + default_branch: repo.default_branch, + html_url: repo.html_url, + stargazers_count: repo.stargazers_count, + watchers_count: repo.watchers_count, + forks_count: repo.forks_count, + indexed_at: repo.indexed_at, + latest_commit: null, + }; + } + }), + ); + + // Add debug info for client + const clientDebugInfo = { + timestamp: new Date().toISOString(), + storage_info: r2Service.getStorageInfo(), + repository_count: effectiveRepositories.length, + repository_names: effectiveRepositories.map( + (repo) => `${repo.owner.login}/${repo.name}`, + ), + storage_structure: { + total_keys: storageKeys.total_keys, + key_counts: storageKeys.key_counts, + key_samples: storageKeys.key_samples, + }, + sample_values: sampleValues, + }; + + return { + repositories: reposWithLatestCommit, + debug_info: clientDebugInfo, + }; + } catch (error) { + console.error("[R2API] Error listing repositories:", error); + + // Return error with debug info for client + return { + repositories: [], + error: true, + message: `Failed to load repositories: ${error instanceof Error ? error.message : "Unknown error"}`, + debug_info: { + timestamp: new Date().toISOString(), + error_message: error instanceof Error ? error.message : "Unknown error", + error_name: error instanceof Error ? error.name : "Unknown", + r2_connection: "Failed", + }, + }; + } +}); diff --git a/packages/app/server/utils/r2-models.ts b/packages/app/server/utils/r2-models.ts new file mode 100644 index 00000000..f1a2f169 --- /dev/null +++ b/packages/app/server/utils/r2-models.ts @@ -0,0 +1,79 @@ +/** + * Data models for storing GitHub-like data in Cloudflare R2 + */ + +// Repository model that mimics GitHub repo structure +export interface R2Repository { + id: string; + name: string; + owner: { + id: string; + login: string; + avatar_url: string; + }; + description: string | null; + default_branch: string; + created_at: string; + updated_at: string; + pushed_at: string; + html_url: string; + homepage?: string; // Added homepage URL + stargazers_count: number; + watchers_count: number; + forks_count: number; + open_issues_count: number; + // Add any additional fields you need + indexed_at: string; // When this repo was last indexed +} + +// Commit model that mimics GitHub commit structure +export interface R2Commit { + repo_id: string; // Reference to the repository + sha: string; + node_id: string; + commit: { + message: string; + author: { + name: string; + email: string; + date: string; + }; + }; + html_url: string; + // Add additional fields as needed + indexed_at: string; // When this commit was last indexed +} + +// Check run model that mimics GitHub check runs +export interface R2CheckRun { + id: string; + commit_sha: string; // Reference to the commit + name: string; + status: string; + conclusion: string | null; + output?: { + summary: string; + text: string; + }; + details_url: string; + url: string; + html_url: string; + // Add additional fields as needed + indexed_at: string; // When this check run was last indexed +} + +// Index metadata +export interface R2IndexMetadata { + lastFullIndexTime: string; + status: "idle" | "indexing"; +} + +// Search index structure for efficient searching +export interface R2SearchIndex { + repositories: { + [searchTerm: string]: string[]; // Maps search terms to repository IDs + }; + commits: { + [searchTerm: string]: string[]; // Maps search terms to commit SHAs + }; +} diff --git a/packages/app/server/utils/r2-service.ts b/packages/app/server/utils/r2-service.ts new file mode 100644 index 00000000..8bf0c9bd --- /dev/null +++ b/packages/app/server/utils/r2-service.ts @@ -0,0 +1,747 @@ +import { H3Event } from "h3"; +import { + R2Repository, + R2Commit, + R2CheckRun, + R2IndexMetadata, + R2SearchIndex, +} from "./r2-models"; +import { useOctokitInstallation, useGithubREST } from "./octokit"; +import { useStorage } from "#imports"; + +// Prefix keys for different data types +const REPO_PREFIX = "repo:"; +const COMMIT_PREFIX = "commit:"; +const CHECK_RUN_PREFIX = "check-run:"; +const INDEX_METADATA_KEY = "index-metadata"; +const SEARCH_INDEX_KEY = "search-index"; + +/** + * R2 Storage Service for GitHub data + * This service handles storing and retrieving GitHub-like data from R2 + */ +export class R2GitHubService { + private storage: ReturnType; + private event?: H3Event; + private debugEnabled: boolean = true; + + constructor( + storage: ReturnType, + debugEnabled = true, + event?: H3Event, + ) { + this.storage = storage; + this.debugEnabled = debugEnabled; + this.event = event; + this.log("Initialized R2GitHubService"); + } + + private log(message: string, isError: boolean = false): void { + if (isError) { + console.error(`[R2Service] ${message}`); + } else if (this.debugEnabled) { + console.log(`[R2Service] ${message}`); + } + } + + // Repository methods + async getRepository( + owner: string, + repo: string, + ): Promise { + const key = `${REPO_PREFIX}${owner}/${repo}`; + this.log( + `Attempting to fetch repository ${owner}/${repo} from R2 storage with key "${key}"`, + ); + + try { + const data = await this.storage.getItem(key); + if (data) { + this.log(`Found repository ${owner}/${repo} in R2 storage`); + const parsedData = JSON.parse(data as string) as R2Repository; + this.log( + `Repository details: name=${parsedData.name}, default_branch=${parsedData.default_branch}, indexed_at=${parsedData.indexed_at}`, + ); + return parsedData; + } else { + this.log( + `Repository ${owner}/${repo} not found in R2 storage (key: ${key})`, + ); + return null; + } + } catch (error) { + this.log( + `Error fetching repository ${owner}/${repo} from R2 storage: ${error}`, + true, + ); + return null; + } + } + + async storeRepository(repository: R2Repository): Promise { + const key = `${REPO_PREFIX}${repository.owner.login}/${repository.name}`; + this.log( + `Storing repository ${repository.owner.login}/${repository.name} in R2 storage with key: ${key}`, + ); + await this.storage.setItem(key, JSON.stringify(repository)); + + // Update search index + await this.updateSearchIndex( + "repositories", + [repository.name, repository.owner.login, repository.description || ""], + repository.id, + ); + } + + async searchRepositories(query: string, limit = 10): Promise { + this.log(`Searching for repositories matching query: "${query}"`); + + try { + // Get the search index + const indexData = await this.storage.getItem("search-index"); + if (!indexData) { + this.log("Search index not found in R2 storage", true); + return []; + } + + const searchIndex = JSON.parse(indexData as string) as R2SearchIndex; + this.log( + `Retrieved search index with ${Object.keys(searchIndex.repositories).length} terms`, + ); + + // Normalize the query + const terms = query + .toLowerCase() + .split(/\W+/) + .filter((term) => term.length > 1); + + this.log(`Searching for terms: ${terms.join(", ")}`); + + // Find repository IDs matching any of the terms + const repoIds = new Set(); + for (const term of terms) { + const matchingTerms = Object.keys(searchIndex.repositories).filter( + (indexTerm) => indexTerm.includes(term), + ); + + this.log( + `Term "${term}" matched ${matchingTerms.length} indexed terms`, + ); + + for (const matchingTerm of matchingTerms) { + const ids = searchIndex.repositories[matchingTerm] || []; + for (const id of ids) { + repoIds.add(id); + } + } + } + + this.log( + `Found ${repoIds.size} unique repositories matching the search terms`, + ); + + // Get full repository details + const keys = await this.storage.getKeys(); + const repos: R2Repository[] = []; + + for (const key of keys) { + if (key.startsWith("repo:")) { + const data = await this.storage.getItem(key); + if (data) { + const repo = JSON.parse(data as string) as R2Repository; + if (repoIds.has(repo.id)) { + this.log( + `Adding repository to search results: ${repo.owner.login}/${repo.name}`, + ); + repos.push(repo); + + if (repos.length >= limit) { + this.log(`Reached result limit of ${limit}, stopping search`); + break; + } + } + } + } + } + + this.log( + `Returning ${repos.length} repositories matching search query "${query}"`, + ); + return repos; + } catch (error) { + this.log(`Error searching repositories: ${error}`, true); + return []; + } + } + + async listAllRepositories(): Promise { + const keys = await this.storage.getKeys(); + const repoKeys = keys.filter((key) => key.startsWith(REPO_PREFIX)); + + const repositories: R2Repository[] = []; + for (const key of repoKeys) { + const repoData = await this.storage.getItem(key); + if (repoData) { + const repo = JSON.parse(repoData as string) as R2Repository; + repositories.push(repo); + } + } + + return repositories; + } + + // Commit methods + async getCommit( + owner: string, + repo: string, + sha: string, + ): Promise { + const key = `${COMMIT_PREFIX}${owner}/${repo}/${sha}`; + this.log(`Fetching commit ${sha} from R2 storage with key: ${key}`); + const data = await this.storage.getItem(key); + if (data) { + this.log(`Found commit ${sha} in R2 storage`); + return JSON.parse(data as string) as R2Commit; + } else { + this.log(`Commit ${sha} not found in R2 storage`); + return null; + } + } + + async storeCommit( + owner: string, + repo: string, + commit: R2Commit, + ): Promise { + const key = `${COMMIT_PREFIX}${owner}/${repo}/${commit.sha}`; + this.log(`Storing commit ${commit.sha} in R2 storage with key: ${key}`); + await this.storage.setItem(key, JSON.stringify(commit)); + + // Update search index + await this.updateSearchIndex( + "commits", + [ + commit.sha, + commit.commit.message, + commit.commit.author.name, + commit.commit.author.email, + ], + commit.sha, + ); + } + + async listCommits( + owner: string, + repo: string, + page = 1, + per_page = 10, + ): Promise { + this.log( + `Listing commits for repository ${owner}/${repo} (page=${page}, per_page=${per_page}) from R2 storage`, + ); + + try { + // Get repository first to verify it exists + const repository = await this.getRepository(owner, repo); + if (!repository) { + this.log( + `Cannot list commits: Repository ${owner}/${repo} not found in R2 storage`, + true, + ); + return []; + } + + // List all keys matching the commit pattern for this repo + const pattern = `commit:${owner}/${repo}/`; + this.log(`Looking for commit keys matching pattern: ${pattern}`); + + const keys = await this.storage.getKeys(); + const commitKeys = keys.filter((key) => key.startsWith(pattern)); + + this.log( + `Found ${commitKeys.length} total commits for ${owner}/${repo} in R2 storage`, + ); + + // Get commits and sort by date (most recent first) + const commits: R2Commit[] = []; + for (const key of commitKeys) { + const data = await this.storage.getItem(key); + if (data) { + commits.push(JSON.parse(data as string) as R2Commit); + } + } + + commits.sort((a, b) => { + const dateA = new Date(a.commit.author.date).getTime(); + const dateB = new Date(b.commit.author.date).getTime(); + return dateB - dateA; + }); + + // Paginate results + const start = (page - 1) * per_page; + const end = start + per_page; + const paginatedCommits = commits.slice(start, end); + + this.log( + `Returning ${paginatedCommits.length} commits for page ${page} (offset ${start})`, + ); + if (paginatedCommits.length > 0) { + this.log( + `First commit in response: ${paginatedCommits[0].sha.substring(0, 7)} - "${paginatedCommits[0].commit.message.split("\n")[0]}"`, + ); + this.log( + `Last commit in response: ${paginatedCommits[paginatedCommits.length - 1].sha.substring(0, 7)} - "${paginatedCommits[paginatedCommits.length - 1].commit.message.split("\n")[0]}"`, + ); + } + + return paginatedCommits; + } catch (error) { + this.log(`Error listing commits for ${owner}/${repo}: ${error}`, true); + return []; + } + } + + // Check Run methods + async getCheckRuns( + owner: string, + repo: string, + commitSha: string, + ): Promise { + const keys = await this.storage.getKeys(); + const checkRunPrefix = `${CHECK_RUN_PREFIX}${owner}/${repo}/${commitSha}/`; + const checkRunKeys = keys.filter((key) => key.startsWith(checkRunPrefix)); + + const checkRuns: R2CheckRun[] = []; + for (const key of checkRunKeys) { + const checkRunData = await this.storage.getItem(key); + if (checkRunData) { + const checkRun = JSON.parse(checkRunData as string) as R2CheckRun; + checkRuns.push(checkRun); + } + } + + return checkRuns; + } + + async storeCheckRun( + owner: string, + repo: string, + commitSha: string, + checkRun: R2CheckRun, + ): Promise { + const key = `${CHECK_RUN_PREFIX}${owner}/${repo}/${commitSha}/${checkRun.id}`; + this.log(`Storing check run ${checkRun.id} in R2 storage with key: ${key}`); + await this.storage.setItem(key, JSON.stringify(checkRun)); + } + + // Search index methods + private async getSearchIndex(): Promise { + const indexData = await this.storage.getItem(SEARCH_INDEX_KEY); + if (!indexData) { + return { + repositories: {}, + commits: {}, + }; + } + return JSON.parse(indexData as string) as R2SearchIndex; + } + + private async updateSearchIndex( + type: "repositories" | "commits", + searchableTerms: string[], + id: string, + ): Promise { + const index = await this.getSearchIndex(); + + // Extract keywords from searchable terms + const keywords = this.extractKeywords(searchableTerms); + + // Add or update index entries + for (const keyword of keywords) { + if (!index[type][keyword]) { + index[type][keyword] = []; + } + + if (!index[type][keyword].includes(id)) { + index[type][keyword].push(id); + } + } + + await this.storage.setItem(SEARCH_INDEX_KEY, JSON.stringify(index)); + } + + private extractKeywords(terms: string[]): string[] { + // Join all terms, split by non-alphanumeric characters, convert to lowercase + const allTerms = terms.join(" ").toLowerCase(); + const keywords = allTerms.split(/[^a-z0-9]+/).filter(Boolean); + + // Remove common words and ensure minimum length + const stopWords = [ + "the", + "and", + "or", + "a", + "an", + "in", + "on", + "at", + "to", + "for", + "with", + "by", + ]; + return keywords.filter( + (word) => word.length > 2 && !stopWords.includes(word), + ); + } + + private findMatchingIds( + indexSection: Record, + searchTerms: string[], + ): string[] { + if (searchTerms.length === 0) { + return []; + } + + // Find all IDs that match each search term + const matchesByTerm: Set[] = []; + + for (const term of searchTerms) { + const matchingIds = new Set(); + + // Look for any keyword that contains this term + for (const [keyword, ids] of Object.entries(indexSection)) { + if (keyword.includes(term)) { + ids.forEach((id) => matchingIds.add(id)); + } + } + + matchesByTerm.push(matchingIds); + } + + // Find intersection of all matching IDs (IDs that match all search terms) + if (matchesByTerm.length === 0) { + return []; + } + + // Start with all IDs from the first term + const result = new Set(matchesByTerm[0]); + + // Filter to IDs that are in all other term matches + for (let i = 1; i < matchesByTerm.length; i++) { + const currentMatches = matchesByTerm[i]; + for (const id of result) { + if (!currentMatches.has(id)) { + result.delete(id); + } + } + } + + return Array.from(result); + } + + // Utility methods + + // Get information about the storage for debugging + getStorageInfo(): object { + try { + // Return a simple diagnostic representation without accessing potentially non-existent properties + return { + hasKeys: typeof this.storage.getKeys === "function", + hasGetItem: typeof this.storage.getItem === "function", + hasSetItem: typeof this.storage.setItem === "function", + storageType: this.storage.constructor?.name || "Unknown", + debugEnabled: this.debugEnabled, + }; + } catch (error) { + this.log(`Error getting storage info: ${error}`, true); + return { error: "Could not retrieve storage information" }; + } + } + + // Get full details of all keys in the storage for debugging + async dumpStorageKeys(): Promise<{ + total_keys: number; + key_counts: Record; + key_samples: Record; + all_keys: string[]; + }> { + try { + this.log("Dumping all storage keys for debugging"); + const keys = await this.storage.getKeys(); + this.log(`Found ${keys.length} total keys in storage`); + + // Organize keys by prefix for better debugging + const keysByPrefix: Record = {}; + + keys.forEach((key) => { + const prefix = key.split(":")[0] || "unknown"; + if (!keysByPrefix[prefix]) { + keysByPrefix[prefix] = []; + } + keysByPrefix[prefix].push(key); + }); + + // Count each type + const counts: Record = {}; + Object.keys(keysByPrefix).forEach((prefix) => { + counts[prefix] = keysByPrefix[prefix].length; + }); + + this.log(`Key counts by prefix: ${JSON.stringify(counts)}`); + + // Sample a few keys from each prefix (max 5) + const samples: Record = {}; + Object.keys(keysByPrefix).forEach((prefix) => { + samples[prefix] = keysByPrefix[prefix].slice(0, 5); + }); + + return { + total_keys: keys.length, + key_counts: counts, + key_samples: samples, + all_keys: keys, + }; + } catch (error) { + this.log(`Error dumping storage keys: ${error}`, true); + return { + total_keys: 0, + key_counts: {}, + key_samples: {}, + all_keys: [], + }; + } + } + + // Get a few sample values to see the structure + async getSampleValues(): Promise<{ + sample_count: number; + samples: Record; + error?: string; + }> { + try { + this.log("Fetching sample values from storage for debugging"); + const keys = await this.storage.getKeys(); + + if (keys.length === 0) { + return { + sample_count: 0, + samples: {}, + error: "No keys found in storage", + }; + } + + // Sample up to 5 keys of different types + const sampleValues: Record = {}; + const prefixes = ["repo:", "commit:", "check-run:", "search-index"]; + + for (const prefix of prefixes) { + const matchingKeys = keys + .filter((key) => key.startsWith(prefix)) + .slice(0, 2); + if (matchingKeys.length > 0) { + sampleValues[prefix] = {}; + for (const key of matchingKeys) { + try { + const value = await this.storage.getItem(key); + if (value) { + sampleValues[prefix][key] = + typeof value === "string" ? JSON.parse(value) : value; + } + } catch (error) { + sampleValues[prefix][key] = `Error parsing value: ${error}`; + } + } + } + } + + return { + sample_count: Object.keys(sampleValues).length, + samples: sampleValues, + }; + } catch (error) { + this.log(`Error getting sample values: ${error}`, true); + return { + sample_count: 0, + samples: {}, + error: `Could not get sample values: ${error}`, + }; + } + } + + // Expose the storage for debugging purposes + // This is normally not recommended but useful for debugging R2 issues + getStorageAccess() { + return { + getKeys: async () => await this.storage.getKeys(), + getItem: async (key: string) => await this.storage.getItem(key), + }; + } + + // Index methods - fetching data from GitHub and storing in R2 + async indexRepository(owner: string, repo: string): Promise { + try { + // Get GitHub data using Octokit + const octokit = useGithubREST(this.event); + + // Fetch repository data + const { data: repoData } = await octokit.request( + "GET /repos/{owner}/{repo}", + { + owner, + repo, + }, + ); + + // Convert to R2Repository format + const r2Repo: R2Repository = { + id: repoData.id.toString(), + name: repoData.name, + owner: { + id: repoData.owner.id.toString(), + login: repoData.owner.login, + avatar_url: repoData.owner.avatar_url, + }, + description: repoData.description, + default_branch: repoData.default_branch, + created_at: repoData.created_at, + updated_at: repoData.updated_at, + pushed_at: repoData.pushed_at, + html_url: repoData.html_url, + stargazers_count: repoData.stargazers_count, + watchers_count: repoData.watchers_count, + forks_count: repoData.forks_count, + open_issues_count: repoData.open_issues_count, + indexed_at: new Date().toISOString(), + }; + + // Store repository + await this.storeRepository(r2Repo); + + // Index commits (latest 100) + await this.indexCommits(owner, repo); + + console.log(`Indexed repository ${owner}/${repo}`); + } catch (error) { + console.error(`Error indexing repository ${owner}/${repo}:`, error); + throw error; + } + } + + async indexCommits( + owner: string, + repo: string, + maxCommits = 100, + ): Promise { + try { + const octokit = useGithubREST(this.event); + + // Get repository to find default branch + const repository = await this.getRepository(owner, repo); + if (!repository) { + throw new Error(`Repository ${owner}/${repo} not found`); + } + + // Fetch commits + const { data: commits } = await octokit.request( + "GET /repos/{owner}/{repo}/commits", + { + owner, + repo, + sha: repository.default_branch, + per_page: maxCommits, + }, + ); + + // Store each commit + for (const commit of commits) { + const r2Commit: R2Commit = { + repo_id: repository.id, + sha: commit.sha, + node_id: commit.node_id || "", + commit: { + message: commit.commit.message, + author: { + name: commit.commit.author?.name || "", + email: commit.commit.author?.email || "", + date: commit.commit.author?.date || new Date().toISOString(), + }, + }, + html_url: commit.html_url, + indexed_at: new Date().toISOString(), + }; + + await this.storeCommit(owner, repo, r2Commit); + + // Index check runs for this commit + await this.indexCheckRuns(owner, repo, commit.sha); + } + + console.log(`Indexed ${commits.length} commits for ${owner}/${repo}`); + } catch (error) { + console.error(`Error indexing commits for ${owner}/${repo}:`, error); + throw error; + } + } + + async indexCheckRuns( + owner: string, + repo: string, + commitSha: string, + ): Promise { + try { + const octokit = useGithubREST(this.event); + + // Fetch check runs + const { data: checkRunsData } = await octokit.request( + "GET /repos/{owner}/{repo}/commits/{ref}/check-runs", + { + owner, + repo, + ref: commitSha, + }, + ); + + // Store each check run + for (const checkRun of checkRunsData.check_runs) { + const r2CheckRun: R2CheckRun = { + id: checkRun.id.toString(), + commit_sha: commitSha, + name: checkRun.name, + status: checkRun.status, + conclusion: checkRun.conclusion, + output: checkRun.output + ? { + summary: checkRun.output.summary || "", + text: checkRun.output.text || "", + } + : undefined, + details_url: checkRun.details_url || "", + url: checkRun.url || "", + html_url: checkRun.html_url || "", + indexed_at: new Date().toISOString(), + }; + + await this.storeCheckRun(owner, repo, commitSha, r2CheckRun); + } + + console.log( + `Indexed ${checkRunsData.check_runs.length} check runs for commit ${commitSha}`, + ); + } catch (error) { + console.error( + `Error indexing check runs for commit ${commitSha}:`, + error, + ); + // Don't throw the error, just log it - check runs might not exist for all commits + } + } +} + +export function useR2GitHubService(event: H3Event): R2GitHubService { + if (!event) { + throw new Error("H3Event is required for R2GitHubService"); + } + console.log("[R2Service] Creating new R2GitHubService instance"); + return new R2GitHubService(useStorage("r2"), true, event); +}