feat: add github extraction feature

Author: andre-dietrich

dist/index.js

@@ -5,6 +5,7 @@ import * as path from 'path'
 import * as fs from 'fs-extra'
 import { tmpdir } from 'os'
 import * as YAML from 'yaml'
+import { removeDirectory } from '../utils/zipExtractor'
 
 export interface ExportJob {
   id: string
@@ -183,8 +184,13 @@ export class JobQueue extends EventEmitter {
             inputFile = readmeFile ? readmeFile.path : job.source.files[0].path
           }
         } else if (job.source.type === 'git' && job.source.gitUrl) {
-          // For git repos, we'd need to clone first - not implemented yet
-          throw new Error('Git repository export not yet implemented')
+          // Use main file from cloned git repository
+          if ((job.source as any).mainFile) {
+            inputFile = (job.source as any).mainFile
+            console.log(`Using main markdown from Git: ${inputFile}`)
+          } else {
+            throw new Error('No main file found in Git repository')
+          }
         } else {
           throw new Error('No valid input source')
         }
@@ -445,13 +451,55 @@ export class JobQueue extends EventEmitter {
             }
 
             console.log(`Export completed: ${job.id} -> ${outputPath}`)
+
+            // Cleanup temporary directories
+            try {
+              // Clean up git clone directory if it exists
+              if (job.source.type === 'git' && (job.source as any).cloneDir) {
+                console.log(
+                  `Cleaning up git clone: ${(job.source as any).cloneDir}`,
+                )
+                await removeDirectory((job.source as any).cloneDir)
+              }
+
+              // Clean up upload directory if it exists
+              if (
+                job.source.type === 'upload' &&
+                (job.source as any).uploadDir
+              ) {
+                console.log(
+                  `Cleaning up upload directory: ${(job.source as any).uploadDir}`,
+                )
+                await removeDirectory((job.source as any).uploadDir)
+              }
+            } catch (cleanupError) {
+              console.warn(`Cleanup warning for job ${job.id}:`, cleanupError)
+              // Don't fail the job if cleanup fails
+            }
+
             resolve()
           } catch (error) {
             reject(error)
           }
         })
       } catch (error) {
         console.error(`Export failed for job ${job.id}:`, error)
+
+        // Cleanup on error as well
+        try {
+          if (job.source.type === 'git' && (job.source as any).cloneDir) {
+            await removeDirectory((job.source as any).cloneDir)
+          }
+          if (job.source.type === 'upload' && (job.source as any).uploadDir) {
+            await removeDirectory((job.source as any).uploadDir)
+          }
+        } catch (cleanupError) {
+          console.warn(
+            `Cleanup error after failed job ${job.id}:`,
+            cleanupError,
+          )
+        }
+
         reject(error)
       }
     })

src/server/queue/jobQueue.ts

@@ -6,7 +6,12 @@ import { randomUUID } from 'crypto'
 import { tmpdir } from 'os'
 import { fileURLToPath } from 'url'
 import * as YAML from 'yaml'
-import { extractZip, findMainMarkdown, isZipFile } from '../utils/zipExtractor'
+import {
+  extractZip,
+  findMainMarkdown,
+  isZipFile,
+  cloneGitRepo,
+} from '../utils/zipExtractor'
 import { dirname } from '../../export/helper'
 
 export const exportRouter: FastifyPluginAsync = async (fastify) => {
@@ -54,7 +59,7 @@ export const exportRouter: FastifyPluginAsync = async (fastify) => {
             // Save file temporarily
             const filepath = join(uploadDir, part.filename)
             const buffer = await part.toBuffer()
-            await writeFile(filepath, buffer)
+            await writeFile(filepath, new Uint8Array(buffer))
 
             files.push({
               filename: part.filename,
@@ -117,7 +122,50 @@ export const exportRouter: FastifyPluginAsync = async (fastify) => {
             jobData.source.extractedFrom = zipFile.filename
           }
         } else if (jobData.source.gitUrl) {
+          // Handle git repository cloning
           jobData.source.type = 'git'
+
+          fastify.log.info(`Cloning git repository: ${jobData.source.gitUrl}`)
+
+          // Create clone directory
+          const cloneId = randomUUID()
+          const cloneDir = join(tmpdir(), 'liaex-git', cloneId)
+          await mkdir(cloneDir, { recursive: true })
+
+          try {
+            // Clone the repository
+            const repoPath = await cloneGitRepo(
+              jobData.source.gitUrl,
+              cloneDir,
+              jobData.source.gitBranch,
+              jobData.source.gitSubdir,
+            )
+
+            fastify.log.info(`Git repository cloned to: ${repoPath}`)
+
+            // Find main markdown file
+            const mainMarkdown = await findMainMarkdown(repoPath)
+            if (!mainMarkdown) {
+              return reply.code(400).send({
+                error:
+                  'No markdown file found in Git repository. Please include a README.md or any .md file.',
+              })
+            }
+
+            fastify.log.info(`Found main markdown: ${mainMarkdown}`)
+
+            // Update job data with cloned repo information
+            jobData.source.mainFile = mainMarkdown
+            jobData.source.cloneDir = cloneDir
+            jobData.source.repoPath = repoPath
+          } catch (error: any) {
+            fastify.log.error(
+              `Failed to clone git repository: ${error.message}`,
+            )
+            return reply.code(400).send({
+              error: `Failed to clone git repository: ${error.message}`,
+            })
+          }
         } else {
           return reply.code(400).send({
             error: 'No files uploaded and no git URL provided',

src/server/routes/export.ts

@@ -1,7 +1,11 @@
 import { createReadStream } from 'fs'
-import { readdir, stat } from 'fs/promises'
+import { readdir, stat, rm } from 'fs/promises'
 import { join, extname } from 'path'
 import unzipper from 'unzipper'
+import { exec } from 'child_process'
+import { promisify } from 'util'
+
+const execAsync = promisify(exec)
 
 /**
  * Extracts a ZIP file to a target directory
@@ -91,3 +95,61 @@ export async function findMainMarkdown(dir: string): Promise<string | null> {
 export function isZipFile(filename: string): boolean {
   return extname(filename).toLowerCase() === '.zip'
 }
+
+/**
+ * Clones a Git repository to a target directory
+ * @param gitUrl Git repository URL
+ * @param cloneDir Directory where to clone the repository
+ * @param branch Optional branch or tag to checkout (defaults to 'main')
+ * @param subdir Optional subdirectory within the repo to use as root
+ * @returns Path to the cloned directory (including subdir if specified)
+ */
+export async function cloneGitRepo(
+  gitUrl: string,
+  cloneDir: string,
+  branch?: string,
+  subdir?: string,
+): Promise<string> {
+  try {
+    // Build git clone command
+    const branchArg = branch ? `--branch ${branch}` : ''
+    const cmd = `git clone --depth 1 ${branchArg} "${gitUrl}" "${cloneDir}"`
+
+    console.log(`Cloning git repository: ${cmd}`)
+    const { stdout, stderr } = await execAsync(cmd)
+
+    if (stderr && !stderr.includes('Cloning into')) {
+      console.warn('Git clone warnings:', stderr)
+    }
+    if (stdout) {
+      console.log('Git clone output:', stdout)
+    }
+
+    // If subdirectory specified, return the full path to it
+    const finalPath = subdir ? join(cloneDir, subdir) : cloneDir
+
+    // Verify the path exists
+    try {
+      await stat(finalPath)
+    } catch (error) {
+      throw new Error(`Subdirectory '${subdir}' not found in cloned repository`)
+    }
+
+    return finalPath
+  } catch (error: any) {
+    console.error('Git clone failed:', error)
+    throw new Error(`Failed to clone git repository: ${error.message}`)
+  }
+}
+
+/**
+ * Removes a directory and all its contents
+ * @param dir Directory to remove
+ */
+export async function removeDirectory(dir: string): Promise<void> {
+  try {
+    await rm(dir, { recursive: true, force: true })
+  } catch (error) {
+    console.warn(`Failed to remove directory ${dir}:`, error)
+  }
+}