From ea8d566bc27ebaaeb3228a886548556197e996cc Mon Sep 17 00:00:00 2001 From: AmirSa12 Date: Mon, 5 May 2025 23:04:31 +0330 Subject: [PATCH 1/5] init test --- packages/app/nuxt.config.ts | 4 +- packages/app/server/api/repo/commits.get.ts | 185 +++---- packages/app/server/api/repo/index.get.ts | 102 ++-- packages/app/server/api/repo/search.get.ts | 64 ++- packages/app/server/utils/r2-models.ts | 79 +++ packages/app/server/utils/r2-service.ts | 525 ++++++++++++++++++++ 6 files changed, 749 insertions(+), 210 deletions(-) create mode 100644 packages/app/server/utils/r2-models.ts create mode 100644 packages/app/server/utils/r2-service.ts diff --git a/packages/app/nuxt.config.ts b/packages/app/nuxt.config.ts index e166f71d..1b607347 100644 --- a/packages/app/nuxt.config.ts +++ b/packages/app/nuxt.config.ts @@ -51,8 +51,8 @@ export default defineNuxtConfig({ webhookSecret: process.env.NITRO_WEBHOOK_SECRET || "", privateKey: process.env.NITRO_PRIVATE_KEY || "", rmStaleKey: process.env.NITRO_RM_STALE_KEY || "", - githubToken: - process.env.GITHUB_TOKEN || process.env.NITRO_GITHUB_TOKEN || "", + // githubToken: + // process.env.GITHUB_TOKEN || process.env.NITRO_GITHUB_TOKEN || "", ghBaseUrl: process.env.NITRO_GH_BASE_URL || "https://api.github.com", test: process.env.NITRO_TEST || "", }, diff --git a/packages/app/server/api/repo/commits.get.ts b/packages/app/server/api/repo/commits.get.ts index 4e32b69b..72425150 100644 --- a/packages/app/server/api/repo/commits.get.ts +++ b/packages/app/server/api/repo/commits.get.ts @@ -1,148 +1,77 @@ import { z } from "zod"; -import { useGithubREST } from "../../../server/utils/octokit"; +import { useR2GitHubService } from "../../../server/utils/r2-service"; const querySchema = z.object({ owner: z.string(), repo: z.string(), - cursor: z.string().optional(), - page: z.string().optional(), - per_page: z.string().optional().default("10"), + page: z.coerce.number().default(1), + per_page: z.coerce.number().default(10), }); export default defineEventHandler(async (event) => { try { + console.log("[R2API] Commits endpoint called"); const query = await getValidatedQuery(event, (data) => querySchema.parse(data), ); - const octokit = useGithubREST(event); - const { data: repo } = await octokit.request("GET /repos/{owner}/{repo}", { - owner: query.owner, - repo: query.repo, - }); - - const defaultBranch = repo.default_branch; - - const page = query.page - ? Number.parseInt(query.page) - : query.cursor - ? Number.parseInt(query.cursor) - : 1; - const per_page = Number.parseInt(query.per_page); + console.log(`[R2API] Fetching commits for ${query.owner}/${query.repo} (page=${query.page}, per_page=${query.per_page})`); - const { data: commits } = await octokit.request( - "GET /repos/{owner}/{repo}/commits", - { - owner: query.owner, - repo: query.repo, - sha: defaultBranch, - page, - per_page, - }, - ); - - const commitsWithStatuses = await Promise.all( - commits.map(async (commit) => { - try { - const { data: checkRuns } = await octokit.request( - "GET /repos/{owner}/{repo}/commits/{ref}/check-runs", - { - owner: query.owner, - repo: query.repo, - ref: commit.sha, - }, - ); + // Use R2 service exclusively + const r2Service = useR2GitHubService(event); + + // Log storage configuration for debugging + console.log(`[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`); - return { - id: commit.node_id || commit.sha, - oid: commit.sha, - abbreviatedOid: commit.sha.substring(0, 7), - message: commit.commit.message, - authoredDate: commit.commit.author?.date || "", - url: commit.html_url, - statusCheckRollup: - checkRuns.check_runs.length > 0 - ? { - id: `status-${commit.sha}`, - state: checkRuns.check_runs.some( - (check) => check.conclusion === "failure", - ) - ? "FAILURE" - : checkRuns.check_runs.some( - (check) => check.conclusion === "success", - ) - ? "SUCCESS" - : "PENDING", - contexts: { - nodes: checkRuns.check_runs.map((check) => ({ - id: check.id.toString(), - status: check.status, - name: check.name, - title: check.name, - summary: check.output?.summary || "", - text: check.output?.text || "", - detailsUrl: check.details_url || "", - url: check.url || check.html_url || "", - })), - }, - } - : null, - }; - } catch (error) { - console.warn( - `Could not fetch check runs for commit ${commit.sha}:`, - error, - ); - return { - id: commit.node_id || commit.sha, - oid: commit.sha, - abbreviatedOid: commit.sha.substring(0, 7), - message: commit.commit.message, - authoredDate: commit.commit.author?.date || "", - url: commit.html_url, - statusCheckRollup: null, - }; - } - }), - ); + // Verify repository exists first + const repository = await r2Service.getRepository(query.owner, query.repo); + if (!repository) { + console.log(`[R2API] Repository ${query.owner}/${query.repo} not found in R2 storage`); + throw new Error(`Repository ${query.owner}/${query.repo} not found in R2 storage`); + } + + console.log(`[R2API] Found repository in R2: ${query.owner}/${query.repo} (id: ${repository.id})`); - // Check if there are more commits (GitHub API doesn't provide this directly) - // We'll need to check if we got a full page of results - const hasNextPage = commits.length === per_page; - const nextPage = hasNextPage ? (page + 1).toString() : null; - - return { - id: `branch-${defaultBranch}`, - name: defaultBranch, - target: { - id: `target-${defaultBranch}`, - history: { - nodes: commitsWithStatuses, - pageInfo: { - hasNextPage, - endCursor: nextPage, - }, + // Get commits from R2 storage + console.log(`[R2API] Fetching commits from R2 storage for ${query.owner}/${query.repo}`); + const commits = await r2Service.listCommits(query.owner, query.repo, query.page, query.per_page); + + if (commits.length === 0) { + console.log(`[R2API] No commits found for ${query.owner}/${query.repo} in R2 storage`); + return { + data: [], + message: `No commits found for ${query.owner}/${query.repo} in storage`, + }; + } + + console.log(`[R2API] Found ${commits.length} commits in R2 storage`); + + // Format response with detailed logging + const responseData = commits.map((commit) => { + console.log(`[R2API] Processing commit: ${commit.sha.substring(0, 7)} - "${commit.commit.message.split('\n')[0]}"`); + + // For each commit, check if we have check runs + return { + sha: commit.sha, + commit: { + message: commit.commit.message, + author: commit.commit.author, }, - }, - }; + html_url: commit.html_url, + author: null, // In R2 storage we might not have author details + r2_source: true, // Flag to indicate source of data + indexed_at: commit.indexed_at || null, + }; + }); + + console.log(`[R2API] Returning ${responseData.length} commits`); + return { data: responseData }; } catch (error) { - console.error("Error fetching repository commits:", error); - - return { - id: "error", - name: "error", - error: true, - message: (error as Error).message, - target: { - id: "error-target", - history: { - nodes: [], - pageInfo: { - hasNextPage: false, - endCursor: null, - }, - }, - }, - }; + console.error("[R2API] Error in commits endpoint:", error); + + throw createError({ + statusCode: 404, + statusMessage: `Commits could not be accessed: ${error instanceof Error ? error.message : 'Unknown error'}`, + }); } }); diff --git a/packages/app/server/api/repo/index.get.ts b/packages/app/server/api/repo/index.get.ts index 50269d5f..3698d606 100644 --- a/packages/app/server/api/repo/index.get.ts +++ b/packages/app/server/api/repo/index.get.ts @@ -1,71 +1,61 @@ -import type { H3Event } from "h3"; +import { useR2GitHubService } from "../../../server/utils/r2-service"; import { z } from "zod"; -import { useGithubREST } from "../../../server/utils/octokit"; const querySchema = z.object({ owner: z.string(), repo: z.string(), }); -const getRepoInfo = defineCachedFunction( - async (owner: string, repo: string, event?: H3Event) => { - try { - const octokit = useGithubREST(event); - - const { data } = await octokit.request("GET /repos/{owner}/{repo}", { - owner, - repo, - }); - - return { - id: data.id.toString(), - name: data.name, - owner: { - id: data.owner.id.toString(), - avatarUrl: data.owner.avatar_url, - login: data.owner.login, - }, - url: data.html_url, - homepageUrl: data.homepage || "", - description: data.description || "", - }; - } catch (error) { - console.error( - `Error fetching repository info for ${owner}/${repo}:`, - error, - ); - throw error; - } - }, - { - getKey: (owner: string, repo: string, _event?: H3Event) => - `${owner}/${repo}`, - maxAge: 60 * 30, // 30 minutes - swr: true, - }, -); - export default defineEventHandler(async (event) => { try { - const query = await getValidatedQuery(event, (data) => - querySchema.parse(data), - ); - return getRepoInfo(query.owner, query.repo, event); - } catch (error) { - console.error("Error in repo info endpoint:", error); + console.log("[R2API] Repository index endpoint called"); + + const query = await getValidatedQuery(event, (data) => querySchema.parse(data)); + console.log(`[R2API] Fetching repository data for ${query.owner}/${query.repo}`); + + // Use R2 service exclusively + const r2Service = useR2GitHubService(event); + + // Log storage info for debugging + console.log(`[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`); + + // Get repository from R2 + const repository = await r2Service.getRepository(query.owner, query.repo); + + if (!repository) { + console.log(`[R2API] Repository ${query.owner}/${query.repo} not found in R2 storage`); + throw new Error(`Repository ${query.owner}/${query.repo} not found in R2 storage`); + } + + console.log(`[R2API] Successfully retrieved repository from R2: ${query.owner}/${query.repo}`); + console.log(`[R2API] Repository details: id=${repository.id}, default_branch=${repository.default_branch}, indexed_at=${repository.indexed_at}`); + return { - error: true, - message: (error as Error).message, - id: "error", - name: "error", + id: repository.id, + name: repository.name, + full_name: `${repository.owner.login}/${repository.name}`, owner: { - id: "error", - avatarUrl: "", - login: "error", + id: repository.owner.id, + login: repository.owner.login, + avatar_url: repository.owner.avatar_url, }, - url: "", - homepageUrl: "", - description: "Error fetching repository data", + default_branch: repository.default_branch, + description: repository.description, + html_url: repository.html_url, + homepage: repository.homepage || null, + watchers_count: repository.watchers_count, + stargazers_count: repository.stargazers_count, + forks_count: repository.forks_count, + open_issues_count: repository.open_issues_count, + r2_source: true, // Flag to indicate source of data + indexed_at: repository.indexed_at || null, }; + } catch (error) { + console.error("[R2API] Error in repository index endpoint:", error); + + throw createError({ + statusCode: 404, + statusMessage: `Repository not found or could not be accessed: ${error instanceof Error ? error.message : 'Unknown error'}`, + }); } }); diff --git a/packages/app/server/api/repo/search.get.ts b/packages/app/server/api/repo/search.get.ts index 144ed81c..d7ea15c3 100644 --- a/packages/app/server/api/repo/search.get.ts +++ b/packages/app/server/api/repo/search.get.ts @@ -1,6 +1,5 @@ -import type { RestEndpointMethodTypes } from "@octokit/plugin-rest-endpoint-methods"; import { z } from "zod"; -import { useGithubREST } from "../../../server/utils/octokit"; +import { useR2GitHubService } from "../../../server/utils/r2-service"; const querySchema = z.object({ text: z.string(), @@ -8,44 +7,61 @@ const querySchema = z.object({ export default defineEventHandler(async (event) => { try { + console.log("[R2API] Search endpoint called"); const query = await getValidatedQuery(event, (data) => querySchema.parse(data), ); if (!query.text) { + console.log("[R2API] Empty search query, returning empty results"); return { nodes: [] }; } - const octokit = useGithubREST(event); + // Use R2 service exclusively for searching + console.log(`[R2API] Searching repositories with query: "${query.text}"`); + const r2Service = useR2GitHubService(event); + + // Log R2 storage configuration for debugging + console.log(`[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`); + + // Get search results + const repositories = await r2Service.searchRepositories(query.text, 10); - const { data } = await octokit.request("GET /search/repositories", { - q: query.text, - per_page: 10, - }); + console.log(`[R2API] Found ${repositories.length} repositories matching the query in R2 storage`); + + if (repositories.length === 0) { + console.log("[R2API] No repositories found in R2 storage matching the query"); + return { + nodes: [], + message: "No matching repositories found in storage" + }; + } - return { - nodes: data.items.map( - ( - repo: RestEndpointMethodTypes["search"]["repos"]["response"]["data"]["items"][0], - ) => ({ - id: repo.id.toString(), + // Format the response with detailed logging + const formattedResponse = { + nodes: repositories.map((repo) => { + console.log(`[R2API] Including repository in results: ${repo.owner.login}/${repo.name} (ID: ${repo.id})`); + return { + id: repo.id, name: repo.name, - owner: repo.owner - ? { - id: repo.owner.id.toString(), - login: repo.owner.login, - avatarUrl: repo.owner.avatar_url, - } - : null, - }), - ), + owner: { + id: repo.owner.id, + login: repo.owner.login, + avatarUrl: repo.owner.avatar_url, + }, + }; + }), }; + + console.log(`[R2API] Search response prepared with ${formattedResponse.nodes.length} repositories`); + return formattedResponse; } catch (error) { - console.error("Error in repository search:", error); + console.error("[R2API] Error in repository search:", error); + return { nodes: [], error: true, - message: (error as Error).message, + message: "Search failed. Only data in storage can be searched.", }; } }); diff --git a/packages/app/server/utils/r2-models.ts b/packages/app/server/utils/r2-models.ts new file mode 100644 index 00000000..36475bcd --- /dev/null +++ b/packages/app/server/utils/r2-models.ts @@ -0,0 +1,79 @@ +/** + * Data models for storing GitHub-like data in Cloudflare R2 + */ + +// Repository model that mimics GitHub repo structure +export interface R2Repository { + id: string; + name: string; + owner: { + id: string; + login: string; + avatar_url: string; + }; + description: string | null; + default_branch: string; + created_at: string; + updated_at: string; + pushed_at: string; + html_url: string; + homepage?: string; // Added homepage URL + stargazers_count: number; + watchers_count: number; + forks_count: number; + open_issues_count: number; + // Add any additional fields you need + indexed_at: string; // When this repo was last indexed +} + +// Commit model that mimics GitHub commit structure +export interface R2Commit { + repo_id: string; // Reference to the repository + sha: string; + node_id: string; + commit: { + message: string; + author: { + name: string; + email: string; + date: string; + }; + }; + html_url: string; + // Add additional fields as needed + indexed_at: string; // When this commit was last indexed +} + +// Check run model that mimics GitHub check runs +export interface R2CheckRun { + id: string; + commit_sha: string; // Reference to the commit + name: string; + status: string; + conclusion: string | null; + output?: { + summary: string; + text: string; + }; + details_url: string; + url: string; + html_url: string; + // Add additional fields as needed + indexed_at: string; // When this check run was last indexed +} + +// Index metadata +export interface R2IndexMetadata { + lastFullIndexTime: string; + status: 'idle' | 'indexing'; +} + +// Search index structure for efficient searching +export interface R2SearchIndex { + repositories: { + [searchTerm: string]: string[]; // Maps search terms to repository IDs + }; + commits: { + [searchTerm: string]: string[]; // Maps search terms to commit SHAs + }; +} diff --git a/packages/app/server/utils/r2-service.ts b/packages/app/server/utils/r2-service.ts new file mode 100644 index 00000000..8d7d88bd --- /dev/null +++ b/packages/app/server/utils/r2-service.ts @@ -0,0 +1,525 @@ +import { H3Event } from 'h3'; +import { + R2Repository, + R2Commit, + R2CheckRun, + R2IndexMetadata, + R2SearchIndex +} from './r2-models'; +import { useOctokitInstallation, useGithubREST } from './octokit'; +import { useStorage } from '#imports'; + +// Prefix keys for different data types +const REPO_PREFIX = 'repo:'; +const COMMIT_PREFIX = 'commit:'; +const CHECK_RUN_PREFIX = 'check-run:'; +const INDEX_METADATA_KEY = 'index-metadata'; +const SEARCH_INDEX_KEY = 'search-index'; + +/** + * R2 Storage Service for GitHub data + * This service handles storing and retrieving GitHub-like data from R2 + */ +export class R2GitHubService { + private storage: ReturnType; + private event?: H3Event; + private debugEnabled: boolean = true; + + constructor(storage: ReturnType, debugEnabled = true, event?: H3Event) { + this.storage = storage; + this.debugEnabled = debugEnabled; + this.event = event; + this.log('Initialized R2GitHubService'); + } + + private log(message: string, isError: boolean = false): void { + if (isError) { + console.error(`[R2Service] ${message}`); + } else if (this.debugEnabled) { + console.log(`[R2Service] ${message}`); + } + } + + // Repository methods + async getRepository(owner: string, repo: string): Promise { + const key = `${REPO_PREFIX}${owner}/${repo}`; + this.log(`Attempting to fetch repository ${owner}/${repo} from R2 storage with key "${key}"`); + + try { + const data = await this.storage.getItem(key); + if (data) { + this.log(`Found repository ${owner}/${repo} in R2 storage`); + const parsedData = JSON.parse(data as string) as R2Repository; + this.log(`Repository details: name=${parsedData.name}, default_branch=${parsedData.default_branch}, indexed_at=${parsedData.indexed_at}`); + return parsedData; + } else { + this.log(`Repository ${owner}/${repo} not found in R2 storage (key: ${key})`); + return null; + } + } catch (error) { + this.log(`Error fetching repository ${owner}/${repo} from R2 storage: ${error}`, true); + return null; + } + } + + async storeRepository(repository: R2Repository): Promise { + const key = `${REPO_PREFIX}${repository.owner.login}/${repository.name}`; + this.log(`Storing repository ${repository.owner.login}/${repository.name} in R2 storage with key: ${key}`); + await this.storage.setItem(key, JSON.stringify(repository)); + + // Update search index + await this.updateSearchIndex('repositories', [ + repository.name, + repository.owner.login, + repository.description || '', + ], repository.id); + } + + async searchRepositories(query: string, limit = 10): Promise { + this.log(`Searching for repositories matching query: "${query}"`); + + try { + // Get the search index + const indexData = await this.storage.getItem("search-index"); + if (!indexData) { + this.log("Search index not found in R2 storage", true); + return []; + } + + const searchIndex = JSON.parse(indexData as string) as R2SearchIndex; + this.log(`Retrieved search index with ${Object.keys(searchIndex.repositories).length} terms`); + + // Normalize the query + const terms = query + .toLowerCase() + .split(/\W+/) + .filter((term) => term.length > 1); + + this.log(`Searching for terms: ${terms.join(", ")}`); + + // Find repository IDs matching any of the terms + const repoIds = new Set(); + for (const term of terms) { + const matchingTerms = Object.keys(searchIndex.repositories).filter( + (indexTerm) => indexTerm.includes(term), + ); + + this.log(`Term "${term}" matched ${matchingTerms.length} indexed terms`); + + for (const matchingTerm of matchingTerms) { + const ids = searchIndex.repositories[matchingTerm] || []; + for (const id of ids) { + repoIds.add(id); + } + } + } + + this.log(`Found ${repoIds.size} unique repositories matching the search terms`); + + // Get full repository details + const keys = await this.storage.getKeys(); + const repos: R2Repository[] = []; + + for (const key of keys) { + if (key.startsWith("repo:")) { + const data = await this.storage.getItem(key); + if (data) { + const repo = JSON.parse(data as string) as R2Repository; + if (repoIds.has(repo.id)) { + this.log(`Adding repository to search results: ${repo.owner.login}/${repo.name}`); + repos.push(repo); + + if (repos.length >= limit) { + this.log(`Reached result limit of ${limit}, stopping search`); + break; + } + } + } + } + } + + this.log(`Returning ${repos.length} repositories matching search query "${query}"`); + return repos; + } catch (error) { + this.log(`Error searching repositories: ${error}`, true); + return []; + } + } + + async listAllRepositories(): Promise { + const keys = await this.storage.getKeys(); + const repoKeys = keys.filter(key => key.startsWith(REPO_PREFIX)); + + const repositories: R2Repository[] = []; + for (const key of repoKeys) { + const repoData = await this.storage.getItem(key); + if (repoData) { + const repo = JSON.parse(repoData as string) as R2Repository; + repositories.push(repo); + } + } + + return repositories; + } + + // Commit methods + async getCommit(owner: string, repo: string, sha: string): Promise { + const key = `${COMMIT_PREFIX}${owner}/${repo}/${sha}`; + this.log(`Fetching commit ${sha} from R2 storage with key: ${key}`); + const data = await this.storage.getItem(key); + if (data) { + this.log(`Found commit ${sha} in R2 storage`); + return JSON.parse(data as string) as R2Commit; + } else { + this.log(`Commit ${sha} not found in R2 storage`); + return null; + } + } + + async storeCommit(owner: string, repo: string, commit: R2Commit): Promise { + const key = `${COMMIT_PREFIX}${owner}/${repo}/${commit.sha}`; + this.log(`Storing commit ${commit.sha} in R2 storage with key: ${key}`); + await this.storage.setItem(key, JSON.stringify(commit)); + + // Update search index + await this.updateSearchIndex('commits', [ + commit.sha, + commit.commit.message, + commit.commit.author.name, + commit.commit.author.email, + ], commit.sha); + } + + async listCommits(owner: string, repo: string, page = 1, per_page = 10): Promise { + this.log(`Listing commits for repository ${owner}/${repo} (page=${page}, per_page=${per_page}) from R2 storage`); + + try { + // Get repository first to verify it exists + const repository = await this.getRepository(owner, repo); + if (!repository) { + this.log(`Cannot list commits: Repository ${owner}/${repo} not found in R2 storage`, true); + return []; + } + + // List all keys matching the commit pattern for this repo + const pattern = `commit:${owner}/${repo}/`; + this.log(`Looking for commit keys matching pattern: ${pattern}`); + + const keys = await this.storage.getKeys(); + const commitKeys = keys.filter((key) => key.startsWith(pattern)); + + this.log(`Found ${commitKeys.length} total commits for ${owner}/${repo} in R2 storage`); + + // Get commits and sort by date (most recent first) + const commits: R2Commit[] = []; + for (const key of commitKeys) { + const data = await this.storage.getItem(key); + if (data) { + commits.push(JSON.parse(data as string) as R2Commit); + } + } + + commits.sort((a, b) => { + const dateA = new Date(a.commit.author.date).getTime(); + const dateB = new Date(b.commit.author.date).getTime(); + return dateB - dateA; + }); + + // Paginate results + const start = (page - 1) * per_page; + const end = start + per_page; + const paginatedCommits = commits.slice(start, end); + + this.log(`Returning ${paginatedCommits.length} commits for page ${page} (offset ${start})`); + if (paginatedCommits.length > 0) { + this.log(`First commit in response: ${paginatedCommits[0].sha.substring(0, 7)} - "${paginatedCommits[0].commit.message.split('\n')[0]}"`); + this.log(`Last commit in response: ${paginatedCommits[paginatedCommits.length - 1].sha.substring(0, 7)} - "${paginatedCommits[paginatedCommits.length - 1].commit.message.split('\n')[0]}"`); + } + + return paginatedCommits; + } catch (error) { + this.log(`Error listing commits for ${owner}/${repo}: ${error}`, true); + return []; + } + } + + // Check Run methods + async getCheckRuns(owner: string, repo: string, commitSha: string): Promise { + const keys = await this.storage.getKeys(); + const checkRunPrefix = `${CHECK_RUN_PREFIX}${owner}/${repo}/${commitSha}/`; + const checkRunKeys = keys.filter(key => key.startsWith(checkRunPrefix)); + + const checkRuns: R2CheckRun[] = []; + for (const key of checkRunKeys) { + const checkRunData = await this.storage.getItem(key); + if (checkRunData) { + const checkRun = JSON.parse(checkRunData as string) as R2CheckRun; + checkRuns.push(checkRun); + } + } + + return checkRuns; + } + + async storeCheckRun(owner: string, repo: string, commitSha: string, checkRun: R2CheckRun): Promise { + const key = `${CHECK_RUN_PREFIX}${owner}/${repo}/${commitSha}/${checkRun.id}`; + this.log(`Storing check run ${checkRun.id} in R2 storage with key: ${key}`); + await this.storage.setItem(key, JSON.stringify(checkRun)); + } + + // Search index methods + private async getSearchIndex(): Promise { + const indexData = await this.storage.getItem(SEARCH_INDEX_KEY); + if (!indexData) { + return { + repositories: {}, + commits: {} + }; + } + return JSON.parse(indexData as string) as R2SearchIndex; + } + + private async updateSearchIndex( + type: 'repositories' | 'commits', + searchableTerms: string[], + id: string + ): Promise { + const index = await this.getSearchIndex(); + + // Extract keywords from searchable terms + const keywords = this.extractKeywords(searchableTerms); + + // Add or update index entries + for (const keyword of keywords) { + if (!index[type][keyword]) { + index[type][keyword] = []; + } + + if (!index[type][keyword].includes(id)) { + index[type][keyword].push(id); + } + } + + await this.storage.setItem(SEARCH_INDEX_KEY, JSON.stringify(index)); + } + + private extractKeywords(terms: string[]): string[] { + // Join all terms, split by non-alphanumeric characters, convert to lowercase + const allTerms = terms.join(' ').toLowerCase(); + const keywords = allTerms.split(/[^a-z0-9]+/).filter(Boolean); + + // Remove common words and ensure minimum length + const stopWords = ['the', 'and', 'or', 'a', 'an', 'in', 'on', 'at', 'to', 'for', 'with', 'by']; + return keywords.filter(word => word.length > 2 && !stopWords.includes(word)); + } + + private findMatchingIds( + indexSection: Record, + searchTerms: string[] + ): string[] { + if (searchTerms.length === 0) { + return []; + } + + // Find all IDs that match each search term + const matchesByTerm: Set[] = []; + + for (const term of searchTerms) { + const matchingIds = new Set(); + + // Look for any keyword that contains this term + for (const [keyword, ids] of Object.entries(indexSection)) { + if (keyword.includes(term)) { + ids.forEach(id => matchingIds.add(id)); + } + } + + matchesByTerm.push(matchingIds); + } + + // Find intersection of all matching IDs (IDs that match all search terms) + if (matchesByTerm.length === 0) { + return []; + } + + // Start with all IDs from the first term + const result = new Set(matchesByTerm[0]); + + // Filter to IDs that are in all other term matches + for (let i = 1; i < matchesByTerm.length; i++) { + const currentMatches = matchesByTerm[i]; + for (const id of result) { + if (!currentMatches.has(id)) { + result.delete(id); + } + } + } + + return Array.from(result); + } + + // Utility methods + + // Get information about the storage for debugging + getStorageInfo(): object { + try { + // Return a simple diagnostic representation without accessing potentially non-existent properties + return { + hasKeys: typeof this.storage.getKeys === 'function', + hasGetItem: typeof this.storage.getItem === 'function', + hasSetItem: typeof this.storage.setItem === 'function', + storageType: this.storage.constructor?.name || 'Unknown', + debugEnabled: this.debugEnabled + }; + } catch (error) { + this.log(`Error getting storage info: ${error}`, true); + return { error: 'Could not retrieve storage information' }; + } + } + + // Index methods - fetching data from GitHub and storing in R2 + async indexRepository(owner: string, repo: string): Promise { + try { + // Get GitHub data using Octokit + const octokit = useGithubREST(this.event); + + // Fetch repository data + const { data: repoData } = await octokit.request('GET /repos/{owner}/{repo}', { + owner, + repo, + }); + + // Convert to R2Repository format + const r2Repo: R2Repository = { + id: repoData.id.toString(), + name: repoData.name, + owner: { + id: repoData.owner.id.toString(), + login: repoData.owner.login, + avatar_url: repoData.owner.avatar_url, + }, + description: repoData.description, + default_branch: repoData.default_branch, + created_at: repoData.created_at, + updated_at: repoData.updated_at, + pushed_at: repoData.pushed_at, + html_url: repoData.html_url, + stargazers_count: repoData.stargazers_count, + watchers_count: repoData.watchers_count, + forks_count: repoData.forks_count, + open_issues_count: repoData.open_issues_count, + indexed_at: new Date().toISOString(), + }; + + // Store repository + await this.storeRepository(r2Repo); + + // Index commits (latest 100) + await this.indexCommits(owner, repo); + + console.log(`Indexed repository ${owner}/${repo}`); + } catch (error) { + console.error(`Error indexing repository ${owner}/${repo}:`, error); + throw error; + } + } + + async indexCommits(owner: string, repo: string, maxCommits = 100): Promise { + try { + const octokit = useGithubREST(this.event); + + // Get repository to find default branch + const repository = await this.getRepository(owner, repo); + if (!repository) { + throw new Error(`Repository ${owner}/${repo} not found`); + } + + // Fetch commits + const { data: commits } = await octokit.request('GET /repos/{owner}/{repo}/commits', { + owner, + repo, + sha: repository.default_branch, + per_page: maxCommits, + }); + + // Store each commit + for (const commit of commits) { + const r2Commit: R2Commit = { + repo_id: repository.id, + sha: commit.sha, + node_id: commit.node_id || '', + commit: { + message: commit.commit.message, + author: { + name: commit.commit.author?.name || '', + email: commit.commit.author?.email || '', + date: commit.commit.author?.date || new Date().toISOString(), + }, + }, + html_url: commit.html_url, + indexed_at: new Date().toISOString(), + }; + + await this.storeCommit(owner, repo, r2Commit); + + // Index check runs for this commit + await this.indexCheckRuns(owner, repo, commit.sha); + } + + console.log(`Indexed ${commits.length} commits for ${owner}/${repo}`); + } catch (error) { + console.error(`Error indexing commits for ${owner}/${repo}:`, error); + throw error; + } + } + + async indexCheckRuns(owner: string, repo: string, commitSha: string): Promise { + try { + const octokit = useGithubREST(this.event); + + // Fetch check runs + const { data: checkRunsData } = await octokit.request( + 'GET /repos/{owner}/{repo}/commits/{ref}/check-runs', + { + owner, + repo, + ref: commitSha, + }, + ); + + // Store each check run + for (const checkRun of checkRunsData.check_runs) { + const r2CheckRun: R2CheckRun = { + id: checkRun.id.toString(), + commit_sha: commitSha, + name: checkRun.name, + status: checkRun.status, + conclusion: checkRun.conclusion, + output: checkRun.output ? { + summary: checkRun.output.summary || '', + text: checkRun.output.text || '', + } : undefined, + details_url: checkRun.details_url || '', + url: checkRun.url || '', + html_url: checkRun.html_url || '', + indexed_at: new Date().toISOString(), + }; + + await this.storeCheckRun(owner, repo, commitSha, r2CheckRun); + } + + console.log(`Indexed ${checkRunsData.check_runs.length} check runs for commit ${commitSha}`); + } catch (error) { + console.error(`Error indexing check runs for commit ${commitSha}:`, error); + // Don't throw the error, just log it - check runs might not exist for all commits + } + } +} + +export function useR2GitHubService(event: H3Event): R2GitHubService { + if (!event) { + throw new Error('H3Event is required for R2GitHubService'); + } + console.log('[R2Service] Creating new R2GitHubService instance'); + return new R2GitHubService(useStorage('r2'), true, event); +} From b5a100eee1bd284144a7a061c8e90d98241aa25d Mon Sep 17 00:00:00 2001 From: AmirSa12 Date: Mon, 5 May 2025 23:04:51 +0330 Subject: [PATCH 2/5] prettier --- packages/app/server/api/repo/commits.get.ts | 57 ++- packages/app/server/api/repo/index.get.ts | 46 ++- packages/app/server/api/repo/search.get.ts | 34 +- packages/app/server/utils/r2-models.ts | 2 +- packages/app/server/utils/r2-service.ts | 399 ++++++++++++-------- 5 files changed, 343 insertions(+), 195 deletions(-) diff --git a/packages/app/server/api/repo/commits.get.ts b/packages/app/server/api/repo/commits.get.ts index 72425150..45769b29 100644 --- a/packages/app/server/api/repo/commits.get.ts +++ b/packages/app/server/api/repo/commits.get.ts @@ -15,41 +15,62 @@ export default defineEventHandler(async (event) => { querySchema.parse(data), ); - console.log(`[R2API] Fetching commits for ${query.owner}/${query.repo} (page=${query.page}, per_page=${query.per_page})`); + console.log( + `[R2API] Fetching commits for ${query.owner}/${query.repo} (page=${query.page}, per_page=${query.per_page})`, + ); // Use R2 service exclusively const r2Service = useR2GitHubService(event); - + // Log storage configuration for debugging - console.log(`[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`); + console.log( + `[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`, + ); // Verify repository exists first const repository = await r2Service.getRepository(query.owner, query.repo); if (!repository) { - console.log(`[R2API] Repository ${query.owner}/${query.repo} not found in R2 storage`); - throw new Error(`Repository ${query.owner}/${query.repo} not found in R2 storage`); + console.log( + `[R2API] Repository ${query.owner}/${query.repo} not found in R2 storage`, + ); + throw new Error( + `Repository ${query.owner}/${query.repo} not found in R2 storage`, + ); } - - console.log(`[R2API] Found repository in R2: ${query.owner}/${query.repo} (id: ${repository.id})`); + + console.log( + `[R2API] Found repository in R2: ${query.owner}/${query.repo} (id: ${repository.id})`, + ); // Get commits from R2 storage - console.log(`[R2API] Fetching commits from R2 storage for ${query.owner}/${query.repo}`); - const commits = await r2Service.listCommits(query.owner, query.repo, query.page, query.per_page); - + console.log( + `[R2API] Fetching commits from R2 storage for ${query.owner}/${query.repo}`, + ); + const commits = await r2Service.listCommits( + query.owner, + query.repo, + query.page, + query.per_page, + ); + if (commits.length === 0) { - console.log(`[R2API] No commits found for ${query.owner}/${query.repo} in R2 storage`); + console.log( + `[R2API] No commits found for ${query.owner}/${query.repo} in R2 storage`, + ); return { data: [], message: `No commits found for ${query.owner}/${query.repo} in storage`, }; } - + console.log(`[R2API] Found ${commits.length} commits in R2 storage`); - + // Format response with detailed logging const responseData = commits.map((commit) => { - console.log(`[R2API] Processing commit: ${commit.sha.substring(0, 7)} - "${commit.commit.message.split('\n')[0]}"`); - + console.log( + `[R2API] Processing commit: ${commit.sha.substring(0, 7)} - "${commit.commit.message.split("\n")[0]}"`, + ); + // For each commit, check if we have check runs return { sha: commit.sha, @@ -63,15 +84,15 @@ export default defineEventHandler(async (event) => { indexed_at: commit.indexed_at || null, }; }); - + console.log(`[R2API] Returning ${responseData.length} commits`); return { data: responseData }; } catch (error) { console.error("[R2API] Error in commits endpoint:", error); - + throw createError({ statusCode: 404, - statusMessage: `Commits could not be accessed: ${error instanceof Error ? error.message : 'Unknown error'}`, + statusMessage: `Commits could not be accessed: ${error instanceof Error ? error.message : "Unknown error"}`, }); } }); diff --git a/packages/app/server/api/repo/index.get.ts b/packages/app/server/api/repo/index.get.ts index 3698d606..711ece9d 100644 --- a/packages/app/server/api/repo/index.get.ts +++ b/packages/app/server/api/repo/index.get.ts @@ -9,27 +9,41 @@ const querySchema = z.object({ export default defineEventHandler(async (event) => { try { console.log("[R2API] Repository index endpoint called"); - - const query = await getValidatedQuery(event, (data) => querySchema.parse(data)); - console.log(`[R2API] Fetching repository data for ${query.owner}/${query.repo}`); - + + const query = await getValidatedQuery(event, (data) => + querySchema.parse(data), + ); + console.log( + `[R2API] Fetching repository data for ${query.owner}/${query.repo}`, + ); + // Use R2 service exclusively const r2Service = useR2GitHubService(event); - + // Log storage info for debugging - console.log(`[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`); - + console.log( + `[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`, + ); + // Get repository from R2 const repository = await r2Service.getRepository(query.owner, query.repo); - + if (!repository) { - console.log(`[R2API] Repository ${query.owner}/${query.repo} not found in R2 storage`); - throw new Error(`Repository ${query.owner}/${query.repo} not found in R2 storage`); + console.log( + `[R2API] Repository ${query.owner}/${query.repo} not found in R2 storage`, + ); + throw new Error( + `Repository ${query.owner}/${query.repo} not found in R2 storage`, + ); } - - console.log(`[R2API] Successfully retrieved repository from R2: ${query.owner}/${query.repo}`); - console.log(`[R2API] Repository details: id=${repository.id}, default_branch=${repository.default_branch}, indexed_at=${repository.indexed_at}`); - + + console.log( + `[R2API] Successfully retrieved repository from R2: ${query.owner}/${query.repo}`, + ); + console.log( + `[R2API] Repository details: id=${repository.id}, default_branch=${repository.default_branch}, indexed_at=${repository.indexed_at}`, + ); + return { id: repository.id, name: repository.name, @@ -52,10 +66,10 @@ export default defineEventHandler(async (event) => { }; } catch (error) { console.error("[R2API] Error in repository index endpoint:", error); - + throw createError({ statusCode: 404, - statusMessage: `Repository not found or could not be accessed: ${error instanceof Error ? error.message : 'Unknown error'}`, + statusMessage: `Repository not found or could not be accessed: ${error instanceof Error ? error.message : "Unknown error"}`, }); } }); diff --git a/packages/app/server/api/repo/search.get.ts b/packages/app/server/api/repo/search.get.ts index d7ea15c3..a3d07536 100644 --- a/packages/app/server/api/repo/search.get.ts +++ b/packages/app/server/api/repo/search.get.ts @@ -20,27 +20,35 @@ export default defineEventHandler(async (event) => { // Use R2 service exclusively for searching console.log(`[R2API] Searching repositories with query: "${query.text}"`); const r2Service = useR2GitHubService(event); - + // Log R2 storage configuration for debugging - console.log(`[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`); - + console.log( + `[R2API] R2 storage configuration: ${JSON.stringify(r2Service.getStorageInfo())}`, + ); + // Get search results const repositories = await r2Service.searchRepositories(query.text, 10); - console.log(`[R2API] Found ${repositories.length} repositories matching the query in R2 storage`); - + console.log( + `[R2API] Found ${repositories.length} repositories matching the query in R2 storage`, + ); + if (repositories.length === 0) { - console.log("[R2API] No repositories found in R2 storage matching the query"); - return { + console.log( + "[R2API] No repositories found in R2 storage matching the query", + ); + return { nodes: [], - message: "No matching repositories found in storage" + message: "No matching repositories found in storage", }; } // Format the response with detailed logging const formattedResponse = { nodes: repositories.map((repo) => { - console.log(`[R2API] Including repository in results: ${repo.owner.login}/${repo.name} (ID: ${repo.id})`); + console.log( + `[R2API] Including repository in results: ${repo.owner.login}/${repo.name} (ID: ${repo.id})`, + ); return { id: repo.id, name: repo.name, @@ -52,12 +60,14 @@ export default defineEventHandler(async (event) => { }; }), }; - - console.log(`[R2API] Search response prepared with ${formattedResponse.nodes.length} repositories`); + + console.log( + `[R2API] Search response prepared with ${formattedResponse.nodes.length} repositories`, + ); return formattedResponse; } catch (error) { console.error("[R2API] Error in repository search:", error); - + return { nodes: [], error: true, diff --git a/packages/app/server/utils/r2-models.ts b/packages/app/server/utils/r2-models.ts index 36475bcd..f1a2f169 100644 --- a/packages/app/server/utils/r2-models.ts +++ b/packages/app/server/utils/r2-models.ts @@ -65,7 +65,7 @@ export interface R2CheckRun { // Index metadata export interface R2IndexMetadata { lastFullIndexTime: string; - status: 'idle' | 'indexing'; + status: "idle" | "indexing"; } // Search index structure for efficient searching diff --git a/packages/app/server/utils/r2-service.ts b/packages/app/server/utils/r2-service.ts index 8d7d88bd..0dd501ed 100644 --- a/packages/app/server/utils/r2-service.ts +++ b/packages/app/server/utils/r2-service.ts @@ -1,20 +1,20 @@ -import { H3Event } from 'h3'; -import { - R2Repository, - R2Commit, +import { H3Event } from "h3"; +import { + R2Repository, + R2Commit, R2CheckRun, R2IndexMetadata, - R2SearchIndex -} from './r2-models'; -import { useOctokitInstallation, useGithubREST } from './octokit'; -import { useStorage } from '#imports'; + R2SearchIndex, +} from "./r2-models"; +import { useOctokitInstallation, useGithubREST } from "./octokit"; +import { useStorage } from "#imports"; // Prefix keys for different data types -const REPO_PREFIX = 'repo:'; -const COMMIT_PREFIX = 'commit:'; -const CHECK_RUN_PREFIX = 'check-run:'; -const INDEX_METADATA_KEY = 'index-metadata'; -const SEARCH_INDEX_KEY = 'search-index'; +const REPO_PREFIX = "repo:"; +const COMMIT_PREFIX = "commit:"; +const CHECK_RUN_PREFIX = "check-run:"; +const INDEX_METADATA_KEY = "index-metadata"; +const SEARCH_INDEX_KEY = "search-index"; /** * R2 Storage Service for GitHub data @@ -25,13 +25,17 @@ export class R2GitHubService { private event?: H3Event; private debugEnabled: boolean = true; - constructor(storage: ReturnType, debugEnabled = true, event?: H3Event) { + constructor( + storage: ReturnType, + debugEnabled = true, + event?: H3Event, + ) { this.storage = storage; this.debugEnabled = debugEnabled; this.event = event; - this.log('Initialized R2GitHubService'); + this.log("Initialized R2GitHubService"); } - + private log(message: string, isError: boolean = false): void { if (isError) { console.error(`[R2Service] ${message}`); @@ -41,43 +45,57 @@ export class R2GitHubService { } // Repository methods - async getRepository(owner: string, repo: string): Promise { + async getRepository( + owner: string, + repo: string, + ): Promise { const key = `${REPO_PREFIX}${owner}/${repo}`; - this.log(`Attempting to fetch repository ${owner}/${repo} from R2 storage with key "${key}"`); - + this.log( + `Attempting to fetch repository ${owner}/${repo} from R2 storage with key "${key}"`, + ); + try { const data = await this.storage.getItem(key); if (data) { this.log(`Found repository ${owner}/${repo} in R2 storage`); const parsedData = JSON.parse(data as string) as R2Repository; - this.log(`Repository details: name=${parsedData.name}, default_branch=${parsedData.default_branch}, indexed_at=${parsedData.indexed_at}`); + this.log( + `Repository details: name=${parsedData.name}, default_branch=${parsedData.default_branch}, indexed_at=${parsedData.indexed_at}`, + ); return parsedData; } else { - this.log(`Repository ${owner}/${repo} not found in R2 storage (key: ${key})`); + this.log( + `Repository ${owner}/${repo} not found in R2 storage (key: ${key})`, + ); return null; } } catch (error) { - this.log(`Error fetching repository ${owner}/${repo} from R2 storage: ${error}`, true); + this.log( + `Error fetching repository ${owner}/${repo} from R2 storage: ${error}`, + true, + ); return null; } } async storeRepository(repository: R2Repository): Promise { const key = `${REPO_PREFIX}${repository.owner.login}/${repository.name}`; - this.log(`Storing repository ${repository.owner.login}/${repository.name} in R2 storage with key: ${key}`); + this.log( + `Storing repository ${repository.owner.login}/${repository.name} in R2 storage with key: ${key}`, + ); await this.storage.setItem(key, JSON.stringify(repository)); - + // Update search index - await this.updateSearchIndex('repositories', [ - repository.name, - repository.owner.login, - repository.description || '', - ], repository.id); + await this.updateSearchIndex( + "repositories", + [repository.name, repository.owner.login, repository.description || ""], + repository.id, + ); } async searchRepositories(query: string, limit = 10): Promise { this.log(`Searching for repositories matching query: "${query}"`); - + try { // Get the search index const indexData = await this.storage.getItem("search-index"); @@ -85,27 +103,31 @@ export class R2GitHubService { this.log("Search index not found in R2 storage", true); return []; } - + const searchIndex = JSON.parse(indexData as string) as R2SearchIndex; - this.log(`Retrieved search index with ${Object.keys(searchIndex.repositories).length} terms`); - + this.log( + `Retrieved search index with ${Object.keys(searchIndex.repositories).length} terms`, + ); + // Normalize the query const terms = query .toLowerCase() .split(/\W+/) .filter((term) => term.length > 1); - + this.log(`Searching for terms: ${terms.join(", ")}`); - + // Find repository IDs matching any of the terms const repoIds = new Set(); for (const term of terms) { const matchingTerms = Object.keys(searchIndex.repositories).filter( (indexTerm) => indexTerm.includes(term), ); - - this.log(`Term "${term}" matched ${matchingTerms.length} indexed terms`); - + + this.log( + `Term "${term}" matched ${matchingTerms.length} indexed terms`, + ); + for (const matchingTerm of matchingTerms) { const ids = searchIndex.repositories[matchingTerm] || []; for (const id of ids) { @@ -113,22 +135,26 @@ export class R2GitHubService { } } } - - this.log(`Found ${repoIds.size} unique repositories matching the search terms`); - + + this.log( + `Found ${repoIds.size} unique repositories matching the search terms`, + ); + // Get full repository details const keys = await this.storage.getKeys(); const repos: R2Repository[] = []; - + for (const key of keys) { if (key.startsWith("repo:")) { const data = await this.storage.getItem(key); if (data) { const repo = JSON.parse(data as string) as R2Repository; if (repoIds.has(repo.id)) { - this.log(`Adding repository to search results: ${repo.owner.login}/${repo.name}`); + this.log( + `Adding repository to search results: ${repo.owner.login}/${repo.name}`, + ); repos.push(repo); - + if (repos.length >= limit) { this.log(`Reached result limit of ${limit}, stopping search`); break; @@ -137,8 +163,10 @@ export class R2GitHubService { } } } - - this.log(`Returning ${repos.length} repositories matching search query "${query}"`); + + this.log( + `Returning ${repos.length} repositories matching search query "${query}"`, + ); return repos; } catch (error) { this.log(`Error searching repositories: ${error}`, true); @@ -148,8 +176,8 @@ export class R2GitHubService { async listAllRepositories(): Promise { const keys = await this.storage.getKeys(); - const repoKeys = keys.filter(key => key.startsWith(REPO_PREFIX)); - + const repoKeys = keys.filter((key) => key.startsWith(REPO_PREFIX)); + const repositories: R2Repository[] = []; for (const key of repoKeys) { const repoData = await this.storage.getItem(key); @@ -158,12 +186,16 @@ export class R2GitHubService { repositories.push(repo); } } - + return repositories; } // Commit methods - async getCommit(owner: string, repo: string, sha: string): Promise { + async getCommit( + owner: string, + repo: string, + sha: string, + ): Promise { const key = `${COMMIT_PREFIX}${owner}/${repo}/${sha}`; this.log(`Fetching commit ${sha} from R2 storage with key: ${key}`); const data = await this.storage.getItem(key); @@ -176,40 +208,60 @@ export class R2GitHubService { } } - async storeCommit(owner: string, repo: string, commit: R2Commit): Promise { + async storeCommit( + owner: string, + repo: string, + commit: R2Commit, + ): Promise { const key = `${COMMIT_PREFIX}${owner}/${repo}/${commit.sha}`; this.log(`Storing commit ${commit.sha} in R2 storage with key: ${key}`); await this.storage.setItem(key, JSON.stringify(commit)); - + // Update search index - await this.updateSearchIndex('commits', [ + await this.updateSearchIndex( + "commits", + [ + commit.sha, + commit.commit.message, + commit.commit.author.name, + commit.commit.author.email, + ], commit.sha, - commit.commit.message, - commit.commit.author.name, - commit.commit.author.email, - ], commit.sha); + ); } - async listCommits(owner: string, repo: string, page = 1, per_page = 10): Promise { - this.log(`Listing commits for repository ${owner}/${repo} (page=${page}, per_page=${per_page}) from R2 storage`); - + async listCommits( + owner: string, + repo: string, + page = 1, + per_page = 10, + ): Promise { + this.log( + `Listing commits for repository ${owner}/${repo} (page=${page}, per_page=${per_page}) from R2 storage`, + ); + try { // Get repository first to verify it exists const repository = await this.getRepository(owner, repo); if (!repository) { - this.log(`Cannot list commits: Repository ${owner}/${repo} not found in R2 storage`, true); + this.log( + `Cannot list commits: Repository ${owner}/${repo} not found in R2 storage`, + true, + ); return []; } - + // List all keys matching the commit pattern for this repo const pattern = `commit:${owner}/${repo}/`; this.log(`Looking for commit keys matching pattern: ${pattern}`); - + const keys = await this.storage.getKeys(); const commitKeys = keys.filter((key) => key.startsWith(pattern)); - - this.log(`Found ${commitKeys.length} total commits for ${owner}/${repo} in R2 storage`); - + + this.log( + `Found ${commitKeys.length} total commits for ${owner}/${repo} in R2 storage`, + ); + // Get commits and sort by date (most recent first) const commits: R2Commit[] = []; for (const key of commitKeys) { @@ -218,24 +270,30 @@ export class R2GitHubService { commits.push(JSON.parse(data as string) as R2Commit); } } - + commits.sort((a, b) => { const dateA = new Date(a.commit.author.date).getTime(); const dateB = new Date(b.commit.author.date).getTime(); return dateB - dateA; }); - + // Paginate results const start = (page - 1) * per_page; const end = start + per_page; const paginatedCommits = commits.slice(start, end); - - this.log(`Returning ${paginatedCommits.length} commits for page ${page} (offset ${start})`); + + this.log( + `Returning ${paginatedCommits.length} commits for page ${page} (offset ${start})`, + ); if (paginatedCommits.length > 0) { - this.log(`First commit in response: ${paginatedCommits[0].sha.substring(0, 7)} - "${paginatedCommits[0].commit.message.split('\n')[0]}"`); - this.log(`Last commit in response: ${paginatedCommits[paginatedCommits.length - 1].sha.substring(0, 7)} - "${paginatedCommits[paginatedCommits.length - 1].commit.message.split('\n')[0]}"`); + this.log( + `First commit in response: ${paginatedCommits[0].sha.substring(0, 7)} - "${paginatedCommits[0].commit.message.split("\n")[0]}"`, + ); + this.log( + `Last commit in response: ${paginatedCommits[paginatedCommits.length - 1].sha.substring(0, 7)} - "${paginatedCommits[paginatedCommits.length - 1].commit.message.split("\n")[0]}"`, + ); } - + return paginatedCommits; } catch (error) { this.log(`Error listing commits for ${owner}/${repo}: ${error}`, true); @@ -244,11 +302,15 @@ export class R2GitHubService { } // Check Run methods - async getCheckRuns(owner: string, repo: string, commitSha: string): Promise { + async getCheckRuns( + owner: string, + repo: string, + commitSha: string, + ): Promise { const keys = await this.storage.getKeys(); const checkRunPrefix = `${CHECK_RUN_PREFIX}${owner}/${repo}/${commitSha}/`; - const checkRunKeys = keys.filter(key => key.startsWith(checkRunPrefix)); - + const checkRunKeys = keys.filter((key) => key.startsWith(checkRunPrefix)); + const checkRuns: R2CheckRun[] = []; for (const key of checkRunKeys) { const checkRunData = await this.storage.getItem(key); @@ -257,11 +319,16 @@ export class R2GitHubService { checkRuns.push(checkRun); } } - + return checkRuns; } - async storeCheckRun(owner: string, repo: string, commitSha: string, checkRun: R2CheckRun): Promise { + async storeCheckRun( + owner: string, + repo: string, + commitSha: string, + checkRun: R2CheckRun, + ): Promise { const key = `${CHECK_RUN_PREFIX}${owner}/${repo}/${commitSha}/${checkRun.id}`; this.log(`Storing check run ${checkRun.id} in R2 storage with key: ${key}`); await this.storage.setItem(key, JSON.stringify(checkRun)); @@ -273,78 +340,93 @@ export class R2GitHubService { if (!indexData) { return { repositories: {}, - commits: {} + commits: {}, }; } return JSON.parse(indexData as string) as R2SearchIndex; } private async updateSearchIndex( - type: 'repositories' | 'commits', + type: "repositories" | "commits", searchableTerms: string[], - id: string + id: string, ): Promise { const index = await this.getSearchIndex(); - + // Extract keywords from searchable terms const keywords = this.extractKeywords(searchableTerms); - + // Add or update index entries for (const keyword of keywords) { if (!index[type][keyword]) { index[type][keyword] = []; } - + if (!index[type][keyword].includes(id)) { index[type][keyword].push(id); } } - + await this.storage.setItem(SEARCH_INDEX_KEY, JSON.stringify(index)); } private extractKeywords(terms: string[]): string[] { // Join all terms, split by non-alphanumeric characters, convert to lowercase - const allTerms = terms.join(' ').toLowerCase(); + const allTerms = terms.join(" ").toLowerCase(); const keywords = allTerms.split(/[^a-z0-9]+/).filter(Boolean); - + // Remove common words and ensure minimum length - const stopWords = ['the', 'and', 'or', 'a', 'an', 'in', 'on', 'at', 'to', 'for', 'with', 'by']; - return keywords.filter(word => word.length > 2 && !stopWords.includes(word)); + const stopWords = [ + "the", + "and", + "or", + "a", + "an", + "in", + "on", + "at", + "to", + "for", + "with", + "by", + ]; + return keywords.filter( + (word) => word.length > 2 && !stopWords.includes(word), + ); } private findMatchingIds( indexSection: Record, - searchTerms: string[] + searchTerms: string[], ): string[] { if (searchTerms.length === 0) { return []; } - + // Find all IDs that match each search term const matchesByTerm: Set[] = []; - + for (const term of searchTerms) { const matchingIds = new Set(); - + // Look for any keyword that contains this term for (const [keyword, ids] of Object.entries(indexSection)) { if (keyword.includes(term)) { - ids.forEach(id => matchingIds.add(id)); + ids.forEach((id) => matchingIds.add(id)); } } - + matchesByTerm.push(matchingIds); } - + // Find intersection of all matching IDs (IDs that match all search terms) if (matchesByTerm.length === 0) { return []; } - + // Start with all IDs from the first term const result = new Set(matchesByTerm[0]); - + // Filter to IDs that are in all other term matches for (let i = 1; i < matchesByTerm.length; i++) { const currentMatches = matchesByTerm[i]; @@ -354,7 +436,7 @@ export class R2GitHubService { } } } - + return Array.from(result); } @@ -365,15 +447,15 @@ export class R2GitHubService { try { // Return a simple diagnostic representation without accessing potentially non-existent properties return { - hasKeys: typeof this.storage.getKeys === 'function', - hasGetItem: typeof this.storage.getItem === 'function', - hasSetItem: typeof this.storage.setItem === 'function', - storageType: this.storage.constructor?.name || 'Unknown', - debugEnabled: this.debugEnabled + hasKeys: typeof this.storage.getKeys === "function", + hasGetItem: typeof this.storage.getItem === "function", + hasSetItem: typeof this.storage.setItem === "function", + storageType: this.storage.constructor?.name || "Unknown", + debugEnabled: this.debugEnabled, }; } catch (error) { this.log(`Error getting storage info: ${error}`, true); - return { error: 'Could not retrieve storage information' }; + return { error: "Could not retrieve storage information" }; } } @@ -382,13 +464,16 @@ export class R2GitHubService { try { // Get GitHub data using Octokit const octokit = useGithubREST(this.event); - + // Fetch repository data - const { data: repoData } = await octokit.request('GET /repos/{owner}/{repo}', { - owner, - repo, - }); - + const { data: repoData } = await octokit.request( + "GET /repos/{owner}/{repo}", + { + owner, + repo, + }, + ); + // Convert to R2Repository format const r2Repo: R2Repository = { id: repoData.id.toString(), @@ -410,13 +495,13 @@ export class R2GitHubService { open_issues_count: repoData.open_issues_count, indexed_at: new Date().toISOString(), }; - + // Store repository await this.storeRepository(r2Repo); - + // Index commits (latest 100) await this.indexCommits(owner, repo); - + console.log(`Indexed repository ${owner}/${repo}`); } catch (error) { console.error(`Error indexing repository ${owner}/${repo}:`, error); @@ -424,48 +509,55 @@ export class R2GitHubService { } } - async indexCommits(owner: string, repo: string, maxCommits = 100): Promise { + async indexCommits( + owner: string, + repo: string, + maxCommits = 100, + ): Promise { try { const octokit = useGithubREST(this.event); - + // Get repository to find default branch const repository = await this.getRepository(owner, repo); if (!repository) { throw new Error(`Repository ${owner}/${repo} not found`); } - + // Fetch commits - const { data: commits } = await octokit.request('GET /repos/{owner}/{repo}/commits', { - owner, - repo, - sha: repository.default_branch, - per_page: maxCommits, - }); - + const { data: commits } = await octokit.request( + "GET /repos/{owner}/{repo}/commits", + { + owner, + repo, + sha: repository.default_branch, + per_page: maxCommits, + }, + ); + // Store each commit for (const commit of commits) { const r2Commit: R2Commit = { repo_id: repository.id, sha: commit.sha, - node_id: commit.node_id || '', + node_id: commit.node_id || "", commit: { message: commit.commit.message, author: { - name: commit.commit.author?.name || '', - email: commit.commit.author?.email || '', + name: commit.commit.author?.name || "", + email: commit.commit.author?.email || "", date: commit.commit.author?.date || new Date().toISOString(), }, }, html_url: commit.html_url, indexed_at: new Date().toISOString(), }; - + await this.storeCommit(owner, repo, r2Commit); - + // Index check runs for this commit await this.indexCheckRuns(owner, repo, commit.sha); } - + console.log(`Indexed ${commits.length} commits for ${owner}/${repo}`); } catch (error) { console.error(`Error indexing commits for ${owner}/${repo}:`, error); @@ -473,20 +565,24 @@ export class R2GitHubService { } } - async indexCheckRuns(owner: string, repo: string, commitSha: string): Promise { + async indexCheckRuns( + owner: string, + repo: string, + commitSha: string, + ): Promise { try { const octokit = useGithubREST(this.event); - + // Fetch check runs const { data: checkRunsData } = await octokit.request( - 'GET /repos/{owner}/{repo}/commits/{ref}/check-runs', + "GET /repos/{owner}/{repo}/commits/{ref}/check-runs", { owner, repo, ref: commitSha, }, ); - + // Store each check run for (const checkRun of checkRunsData.check_runs) { const r2CheckRun: R2CheckRun = { @@ -495,22 +591,29 @@ export class R2GitHubService { name: checkRun.name, status: checkRun.status, conclusion: checkRun.conclusion, - output: checkRun.output ? { - summary: checkRun.output.summary || '', - text: checkRun.output.text || '', - } : undefined, - details_url: checkRun.details_url || '', - url: checkRun.url || '', - html_url: checkRun.html_url || '', + output: checkRun.output + ? { + summary: checkRun.output.summary || "", + text: checkRun.output.text || "", + } + : undefined, + details_url: checkRun.details_url || "", + url: checkRun.url || "", + html_url: checkRun.html_url || "", indexed_at: new Date().toISOString(), }; - + await this.storeCheckRun(owner, repo, commitSha, r2CheckRun); } - - console.log(`Indexed ${checkRunsData.check_runs.length} check runs for commit ${commitSha}`); + + console.log( + `Indexed ${checkRunsData.check_runs.length} check runs for commit ${commitSha}`, + ); } catch (error) { - console.error(`Error indexing check runs for commit ${commitSha}:`, error); + console.error( + `Error indexing check runs for commit ${commitSha}:`, + error, + ); // Don't throw the error, just log it - check runs might not exist for all commits } } @@ -518,8 +621,8 @@ export class R2GitHubService { export function useR2GitHubService(event: H3Event): R2GitHubService { if (!event) { - throw new Error('H3Event is required for R2GitHubService'); + throw new Error("H3Event is required for R2GitHubService"); } - console.log('[R2Service] Creating new R2GitHubService instance'); - return new R2GitHubService(useStorage('r2'), true, event); + console.log("[R2Service] Creating new R2GitHubService instance"); + return new R2GitHubService(useStorage("r2"), true, event); } From baadc8c9843040f169ebc10e54a42ff729a6b203 Mon Sep 17 00:00:00 2001 From: AmirSa12 Date: Mon, 5 May 2025 23:20:58 +0330 Subject: [PATCH 3/5] update --- packages/app/app/pages/index.vue | 274 ++++++++++++++++++++++++++- packages/app/server/api/repos.get.ts | 94 +++++++++ 2 files changed, 364 insertions(+), 4 deletions(-) create mode 100644 packages/app/server/api/repos.get.ts diff --git a/packages/app/app/pages/index.vue b/packages/app/app/pages/index.vue index cee70513..1d8676bf 100644 --- a/packages/app/app/pages/index.vue +++ b/packages/app/app/pages/index.vue @@ -4,6 +4,140 @@ definePageMeta({ mainClass: "", }); +import { ref, onMounted } from 'vue'; + +// Define types for the repository and log data +interface RepositoryOwner { + id?: string; + login: string; + avatar_url: string; +} + +interface LatestCommit { + sha: string; + message: string; + date: string; +} + +interface Repository { + id: string; + name: string; + owner: RepositoryOwner; + full_name: string; + description: string | null; + default_branch: string; + html_url: string; + homepage?: string | null; + stargazers_count: number; + watchers_count: number; + forks_count: number; + open_issues_count?: number; + indexed_at: string; + latest_commit: LatestCommit | null; +} + +interface LogEntry { + timestamp: string; + message: string; + type: 'info' | 'error' | 'success'; + data: any; +} + +// State for repositories and logs +const repositories = ref([]); +const loading = ref(true); +const error = ref(null); +const clientLogs = ref([]); +const showLogs = ref(false); + +// Function to add logs that will be visible in the client +const logToClient = (message: string, type: 'info' | 'error' | 'success' = 'info', data: any = null): void => { + const timestamp = new Date().toISOString(); + const logEntry: LogEntry = { + timestamp, + message, + type, + data + }; + clientLogs.value.unshift(logEntry); + console.log(`[CLIENT-LOG][${type}] ${message}`, data || ''); +}; + +// Fetch all repositories from R2 storage +const fetchRepositories = async (): Promise => { + try { + loading.value = true; + logToClient('Fetching repositories from R2 storage...', 'info'); + + const response = await fetch('/api/repos'); + const responseData = await response.json() as { + repositories: Repository[]; + error?: boolean; + message?: string; + debug_info?: any; + }; + + if (responseData.error) { + logToClient(`Error fetching repositories: ${responseData.message || 'Unknown error'}`, 'error', responseData.debug_info); + error.value = responseData.message || 'Unknown error'; + loading.value = false; + return; + } + + repositories.value = responseData.repositories; + logToClient( + `Successfully fetched ${responseData.repositories.length} repositories from R2 storage`, + 'success', + responseData.debug_info + ); + + // Log available repositories + repositories.value.forEach(repo => { + logToClient(`Repository available: ${repo.full_name}`, 'info', { + default_branch: repo.default_branch, + latest_commit: repo.latest_commit ? + `${repo.latest_commit.sha.substring(0, 7)} - ${repo.latest_commit.message.split('\n')[0]}` : + 'No commits found' + }); + }); + } catch (e: any) { + logToClient(`Failed to fetch repositories: ${e.message}`, 'error'); + error.value = `Failed to fetch repositories: ${e.message}`; + } finally { + loading.value = false; + } +}; + +// Truncate long texts +const truncate = (text: string, length = 60): string => { + if (!text) return ''; + return text.length > length ? text.substring(0, length) + '...' : text; +}; + +// Format date +const formatDate = (dateString: string): string => { + if (!dateString) return 'Unknown'; + const date = new Date(dateString); + return date.toLocaleString(); +}; + +// Toggle logs visibility +const toggleLogs = (): void => { + showLogs.value = !showLogs.value; +}; + +// Clear logs +const clearLogs = (): void => { + clientLogs.value = []; + logToClient('Logs cleared', 'info'); +}; + +// Load repositories on mount +onMounted(() => { + fetchRepositories(); +}); + +// Template refs const gettingStartedEl = useTemplateRef("getting-started"); function scrollToGettingStarted() { @@ -15,14 +149,131 @@ function scrollToGettingStarted() { + + diff --git a/packages/app/server/api/repos.get.ts b/packages/app/server/api/repos.get.ts new file mode 100644 index 00000000..7404feec --- /dev/null +++ b/packages/app/server/api/repos.get.ts @@ -0,0 +1,94 @@ +import { useR2GitHubService } from "../utils/r2-service"; + +export default defineEventHandler(async (event) => { + try { + console.log("[R2API] List all repositories endpoint called"); + + // Use R2 service + const r2Service = useR2GitHubService(event); + + // Get all repositories from R2 + const repositories = await r2Service.listAllRepositories(); + + console.log(`[R2API] Found ${repositories.length} repositories in R2 storage`); + + // Get latest commit for each repository + const reposWithLatestCommit = await Promise.all( + repositories.map(async (repo) => { + try { + const commits = await r2Service.listCommits(repo.owner.login, repo.name, 1, 1); + const latestCommit = commits.length > 0 ? commits[0] : null; + + return { + id: repo.id, + name: repo.name, + owner: { + login: repo.owner.login, + avatar_url: repo.owner.avatar_url, + }, + full_name: `${repo.owner.login}/${repo.name}`, + description: repo.description, + default_branch: repo.default_branch, + html_url: repo.html_url, + stargazers_count: repo.stargazers_count, + watchers_count: repo.watchers_count, + forks_count: repo.forks_count, + indexed_at: repo.indexed_at, + latest_commit: latestCommit ? { + sha: latestCommit.sha, + message: latestCommit.commit.message, + date: latestCommit.commit.author.date, + } : null, + }; + } catch (error) { + console.error(`[R2API] Error fetching commits for ${repo.owner.login}/${repo.name}:`, error); + return { + id: repo.id, + name: repo.name, + owner: { + login: repo.owner.login, + avatar_url: repo.owner.avatar_url, + }, + full_name: `${repo.owner.login}/${repo.name}`, + description: repo.description, + default_branch: repo.default_branch, + html_url: repo.html_url, + stargazers_count: repo.stargazers_count, + watchers_count: repo.watchers_count, + forks_count: repo.forks_count, + indexed_at: repo.indexed_at, + latest_commit: null, + }; + } + }) + ); + + // Add debug info for client + const clientDebugInfo = { + timestamp: new Date().toISOString(), + storage_info: r2Service.getStorageInfo(), + repository_count: repositories.length, + repository_names: repositories.map(repo => `${repo.owner.login}/${repo.name}`), + }; + + return { + repositories: reposWithLatestCommit, + debug_info: clientDebugInfo + }; + } catch (error) { + console.error("[R2API] Error listing repositories:", error); + + // Return error with debug info for client + return { + repositories: [], + error: true, + message: `Failed to load repositories: ${error instanceof Error ? error.message : 'Unknown error'}`, + debug_info: { + timestamp: new Date().toISOString(), + error_message: error instanceof Error ? error.message : 'Unknown error', + error_name: error instanceof Error ? error.name : 'Unknown', + r2_connection: 'Failed' + } + }; + } +}); From d484739db6fe073f60f00825d2b9da4c5fe3fff0 Mon Sep 17 00:00:00 2001 From: AmirSa12 Date: Mon, 5 May 2025 23:21:29 +0330 Subject: [PATCH 4/5] prettier --- packages/app/app/pages/index.vue | 241 ++++++++++++++++++--------- packages/app/server/api/repos.get.ts | 64 ++++--- 2 files changed, 200 insertions(+), 105 deletions(-) diff --git a/packages/app/app/pages/index.vue b/packages/app/app/pages/index.vue index 1d8676bf..91055957 100644 --- a/packages/app/app/pages/index.vue +++ b/packages/app/app/pages/index.vue @@ -4,7 +4,7 @@ definePageMeta({ mainClass: "", }); -import { ref, onMounted } from 'vue'; +import { ref, onMounted } from "vue"; // Define types for the repository and log data interface RepositoryOwner { @@ -39,7 +39,7 @@ interface Repository { interface LogEntry { timestamp: string; message: string; - type: 'info' | 'error' | 'success'; + type: "info" | "error" | "success"; data: any; } @@ -51,57 +51,65 @@ const clientLogs = ref([]); const showLogs = ref(false); // Function to add logs that will be visible in the client -const logToClient = (message: string, type: 'info' | 'error' | 'success' = 'info', data: any = null): void => { +const logToClient = ( + message: string, + type: "info" | "error" | "success" = "info", + data: any = null, +): void => { const timestamp = new Date().toISOString(); const logEntry: LogEntry = { timestamp, message, type, - data + data, }; clientLogs.value.unshift(logEntry); - console.log(`[CLIENT-LOG][${type}] ${message}`, data || ''); + console.log(`[CLIENT-LOG][${type}] ${message}`, data || ""); }; // Fetch all repositories from R2 storage const fetchRepositories = async (): Promise => { try { loading.value = true; - logToClient('Fetching repositories from R2 storage...', 'info'); - - const response = await fetch('/api/repos'); - const responseData = await response.json() as { + logToClient("Fetching repositories from R2 storage...", "info"); + + const response = await fetch("/api/repos"); + const responseData = (await response.json()) as { repositories: Repository[]; error?: boolean; message?: string; debug_info?: any; }; - + if (responseData.error) { - logToClient(`Error fetching repositories: ${responseData.message || 'Unknown error'}`, 'error', responseData.debug_info); - error.value = responseData.message || 'Unknown error'; + logToClient( + `Error fetching repositories: ${responseData.message || "Unknown error"}`, + "error", + responseData.debug_info, + ); + error.value = responseData.message || "Unknown error"; loading.value = false; return; } - + repositories.value = responseData.repositories; logToClient( - `Successfully fetched ${responseData.repositories.length} repositories from R2 storage`, - 'success', - responseData.debug_info + `Successfully fetched ${responseData.repositories.length} repositories from R2 storage`, + "success", + responseData.debug_info, ); - + // Log available repositories - repositories.value.forEach(repo => { - logToClient(`Repository available: ${repo.full_name}`, 'info', { + repositories.value.forEach((repo) => { + logToClient(`Repository available: ${repo.full_name}`, "info", { default_branch: repo.default_branch, - latest_commit: repo.latest_commit ? - `${repo.latest_commit.sha.substring(0, 7)} - ${repo.latest_commit.message.split('\n')[0]}` : - 'No commits found' + latest_commit: repo.latest_commit + ? `${repo.latest_commit.sha.substring(0, 7)} - ${repo.latest_commit.message.split("\n")[0]}` + : "No commits found", }); }); } catch (e: any) { - logToClient(`Failed to fetch repositories: ${e.message}`, 'error'); + logToClient(`Failed to fetch repositories: ${e.message}`, "error"); error.value = `Failed to fetch repositories: ${e.message}`; } finally { loading.value = false; @@ -110,13 +118,13 @@ const fetchRepositories = async (): Promise => { // Truncate long texts const truncate = (text: string, length = 60): string => { - if (!text) return ''; - return text.length > length ? text.substring(0, length) + '...' : text; + if (!text) return ""; + return text.length > length ? text.substring(0, length) + "..." : text; }; // Format date const formatDate = (dateString: string): string => { - if (!dateString) return 'Unknown'; + if (!dateString) return "Unknown"; const date = new Date(dateString); return date.toLocaleString(); }; @@ -129,7 +137,7 @@ const toggleLogs = (): void => { // Clear logs const clearLogs = (): void => { clientLogs.value = []; - logToClient('Logs cleared', 'info'); + logToClient("Logs cleared", "info"); }; // Load repositories on mount @@ -149,7 +157,9 @@ function scrollToGettingStarted() {