backup commit

Author: nikgreg99

analyticq-backend/analyticq/api/analyze.py

@@ -0,0 +1,170 @@
+import json
+import logging
+import os
+from tempfile import NamedTemporaryFile
+from typing import Dict, List, Optional
+from uuid import uuid4
+
+from analyticq.manager.tool_manager import AnalyticQSASTManager
+from analyticq.schemas.analyse_dto import AnalysisResponse, GitRepoRequest
+from fastapi import (APIRouter, BackgroundTasks, File, Form, HTTPException,
+                     UploadFile)
+
+logger = logging.getLogger(__name__)
+analyze_router = APIRouter(prefix="/analyze", tags=["Analysis"])
+sast_manager = AnalyticQSASTManager()
+
+# In-memory analysis tracking (consider replacing with persistent storage in production)
+analysis_results = {}
+
+
+async def _run_git_analysis(
+    analysis_id: str,
+    git_url: str,
+    branch: Optional[str] = None,
+    config_paths: Optional[Dict[str, str]] = None,
+    timeout: Optional[int] = None
+):
+    """Asynchronous task to analyze a Git repository."""
+    try:
+        logger.info(f"[{analysis_id}] Starting Git analysis for {git_url}")
+        analysis_results[analysis_id]["status"] = "running"
+
+        result = await sast_manager.scan_codebase(
+            codebase_path=git_url,
+            config_paths=config_paths,
+            timeout=timeout,
+            branch=branch or "main"
+        )
+
+        analysis_results[analysis_id] = {
+            "analysis_id": analysis_id,
+            "status": "completed",
+            "results": result
+        }
+
+    except Exception as e:
+        logger.error(f"[{analysis_id}] Git analysis failed: {str(e)}")
+        analysis_results[analysis_id] = {
+            "status": "failed",
+            "error": str(e)
+        }
+
+
+async def _run_file_analysis(
+    analysis_id: str,
+    file_paths: List[str],
+    config_paths: Optional[Dict[str, str]] = None,
+    timeout: Optional[int] = None,
+    original_filenames: Optional[List[str]] = None
+):
+    """Asynchronous task to analyze uploaded files."""
+    try:
+        logger.info(f"[{analysis_id}] Starting file analysis")
+        analysis_results[analysis_id]["status"] = "running"
+        logger.info(f"[{analysis_id}] File paths: {file_paths}")
+        result = await sast_manager.scan_codebase(
+            codebase_path=file_paths,
+            config_paths=config_paths,
+            timeout=timeout,
+            original_path=original_filenames
+        )
+
+        analysis_results[analysis_id] = {
+            "analysis_id": analysis_id,
+            "status": "completed",
+            "results": result
+        }
+
+    except Exception as e:
+        logger.error(f"[{analysis_id}] File analysis failed: {str(e)}")
+        analysis_results[analysis_id] = {
+            "status": "failed",
+            "error": str(e)
+        }
+
+
+@analyze_router.post("/git", response_model=AnalysisResponse)
+async def analyze_git_repo(
+    background_tasks: BackgroundTasks,
+    repo_request: GitRepoRequest,
+):
+    """Initiate analysis on a Git repository."""
+    try:
+        analysis_id = str(uuid4())
+        analysis_results[analysis_id] = {"status": "pending"}
+
+        background_tasks.add_task(
+            _run_git_analysis,
+            analysis_id,
+            repo_request.git_url,
+            repo_request.branch,
+            repo_request.config_paths,
+            repo_request.timeout
+        )
+
+        return AnalysisResponse(analysis_id=analysis_id, status="pending")
+
+    except Exception as e:
+        logger.error(f"Error initiating Git analysis: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@analyze_router.post("/files", response_model=AnalysisResponse)
+async def analyze_files(
+    background_tasks: BackgroundTasks,
+    files: List[UploadFile] = File(...),
+    config_paths: Optional[str] = Form(None),
+    timeout: Optional[int] = Form(None),
+):
+    """Initiate analysis on uploaded file archives."""
+    try:
+        logger.info(f"Received {len(files)} files for analysis")
+        analysis_id = str(uuid4())
+        analysis_results[analysis_id] = {"status": "pending"}
+
+        config_dict = {}
+        if config_paths:
+            try:
+                config_dict = json.loads(config_paths)
+            except json.JSONDecodeError:
+                raise HTTPException(status_code=400, detail="Invalid config_paths JSON format")
+
+        temp_file_paths = []
+        original_filenames = []
+
+        for upload_file in files:
+            if not any(upload_file.filename.endswith(ext) for ext in ['.zip', '.tar', '.gz']):
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Unsupported file format: {upload_file.filename}"
+                )
+
+            with NamedTemporaryFile(delete=False, suffix=os.path.splitext(upload_file.filename)[1]) as temp_file:
+                temp_file.write(await upload_file.read())
+                temp_file_paths.append(temp_file.name)
+                original_filenames.append(upload_file.filename)
+                logger.info(upload_file.filename)
+
+                background_tasks.add_task(
+                    _run_file_analysis,
+                    analysis_id,
+                    temp_file_paths[0],
+                    config_dict,
+                    timeout,
+                    original_filenames[0]
+                )
+
+        return AnalysisResponse(analysis_id=analysis_id, status="pending")
+
+    except Exception as e:
+        logger.error(f"Error initiating file analysis: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@analyze_router.get("/status/{analysis_id}", response_model=AnalysisResponse)
+async def get_analysis_status(analysis_id: str):
+    if analysis_id not in analysis_results:
+        raise HTTPException(status_code=404, detail="Analysis ID not found")
+
+    return AnalysisResponse(analysis_id=analysis_id, status=analysis_results[analysis_id]["status"])

analyticq-backend/analyticq/api/report.py

@@ -0,0 +1,52 @@
+import io
+import json
+from datetime import datetime
+from typing import Literal
+
+from analyticq.service.scan_service import (AnalyticQScanResultRepository,
+                                            AnalyticQScanService)
+from fastapi import APIRouter, Depends, Query, Response
+from fastapi.responses import HTMLResponse, StreamingResponse
+
+report_router = APIRouter(prefix="/reports", tags=["reports"])
+
+
+def get_scan_service(
+    repo: AnalyticQScanResultRepository = Depends(AnalyticQScanResultRepository)
+) -> AnalyticQScanService:
+    """Dependency provider for scan service."""
+    return AnalyticQScanService(repo)
+
+
+@report_router.get("/download/{scan_id}")
+async def download_report(
+    scan_id: str,
+    format: Literal["pdf", "json", "html", "csv"] = Query(..., description="Report format: pdf, json, html, or csv"),
+    scan_service: AnalyticQScanService = Depends(get_scan_service)
+):
+    content = await scan_service.get_scan_report(scan_id, format)
+    report_filename = f"analyticq_report_{scan_id}_{datetime.now().strftime('%Y%m%d')}".lower()
+
+    if format == "json":
+        return Response(
+            content=json.dumps(content, indent=2),
+            media_type="application/json",
+            headers={"Content-Disposition": f"attachment; filename={report_filename}.json"}
+        )
+    elif format == "html":
+        return HTMLResponse(
+            content=content,
+            headers={"Content-Disposition": f"attachment; filename={report_filename}.html"}
+        )
+    elif format == "pdf":
+        return StreamingResponse(
+            io.BytesIO(content),
+            media_type="application/pdf",
+            headers={"Content-Disposition": f"attachment; filename={report_filename}.pdf"}
+        )
+    elif format == "csv":
+        return Response(
+            content=content,
+            media_type="text/csv",
+            headers={"Content-Disposition": f"attachment; filename={report_filename}.csv"}
+        )

analyticq-backend/analyticq/app.py

@@ -7,14 +7,15 @@
 from contextlib import asynccontextmanager
 from typing import List
 
+from .jinja import setup_jinja
+
 if platform.system() == "Windows":
     asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
 
 from analyticq.celery_app import get_celery_app
 from analyticq.config.di import AnalyticQContainer
 from analyticq.engine.core import AnalyticQToolDiscoverer
 from analyticq.manager import AnalyticQDatabaseManager, DockerImageManager
-from analyticq.manager.tool_manager import AnalyticQSASTManager
 from analyticq.util import AnalyticQConst, PathUtil
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
@@ -165,14 +166,15 @@ async def backend_context(app: FastAPI):
         celery_app = get_celery_app()
         app.state.celery = celery_app
 
+        logger.info("Init Jinja for templating engine...")
+        setup_jinja(app)
+
         logger.info("Checking root exsistence for scanning codebase")
         await create_AnalyticQ_root_structure()
 
         logger.info("Initializing AnalyticQ SAST Tool Registry...")
         AnalyticQToolDiscoverer.discover_and_register_tools()
 
-        analyticq_mangaer = AnalyticQSASTManager()
-        await analyticq_mangaer.scan_codebase("https://github.com/paulc4/microservices-demo")
         logger.info(f"Analyticq backend started in {time.time() - start_time:2f} seconds")
 
         yield
@@ -234,13 +236,16 @@ def set_app_routes(app: FastAPI) -> None :
         - tool_router: Handles tool-related endpoints
         - issue_router: Handles issue-related endpoints
     """
-    from analyticq.api import contexts, issues, scans, stats, tools
+    from analyticq.api import (analyze, contexts, issues, report, scans, stats,
+                               tools)
 
     app.include_router(stats.stats_router)
     app.include_router(contexts.context_router)
     app.include_router(scans.scan_router)
     app.include_router(tools.tool_router)
     app.include_router(issues.issue_router)
+    app.include_router(report.report_router)
+    app.include_router(analyze.analyze_router)
 
 
 def create_app(config_file: str = AnalyticQConst.ANALYTICQ_DEFAULT_CONFIG_FILE,

analyticq-backend/analyticq/engine/core/models.py

@@ -34,6 +34,7 @@ class AnalyticQSeverity(Enum):
     Enumeration representing severity levels for AnalyticQ issues.
 
     Attributes:
+        INFO (str): INFO severity level
         HIGH (str): Critical severity level
         MEDIUM (str): Moderate severity level
         LOW (str): Minor severity level
@@ -44,6 +45,7 @@ class AnalyticQSeverity(Enum):
     MEDIUM = "MEDIUM"
     LOW = "LOW"
     CRITICAL = "CRITICAL"
+    WARNING = "WARNING"  # Not all SAST tool support this severity lev
     UNKNOWN = "UNKNOWN"
 
     @classmethod

analyticq-backend/analyticq/engine/core/result_parser.py

@@ -172,6 +172,28 @@ def map_endline(self, end_line: Union[List, str]):
             return end_line if end_line else 0
 
     def _process_metadata(self, raw_issue: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Processes and extracts metadata from a raw issue dictionary based on configured field mapping.
+
+        This method traverses the raw issue dictionary using dot notation specified in the field mapping
+        to locate and extract metadata. It handles different data types (dict, list, simple values)
+        and processes them accordingly.
+
+        Args:
+            raw_issue (Dict[str, Any]): The raw issue dictionary containing metadata information.
+
+        Returns:
+            Dict[str, Any]: Processed metadata dictionary. Returns:
+                - Copy of all fields if metadata is a dictionary
+                - Dictionary with "items" key if metadata is a list
+                - Dictionary with "value" key if metadata is a simple value
+                - Empty dictionary if metadata field is not found or on error
+
+        Example:
+            If field_mapping["issue_metadata"] = "metadata.fields"
+            raw_issue = {"metadata": {"fields": {"key1": "value1", "key2": "value2"}}}
+            Returns: {"key1": "value1", "key2": "value2"}
+        """
 
         metadata_field = self.field_mapping.get("issue_metadata")
         if not metadata_field:
@@ -224,6 +246,8 @@ def parse_scan_result(self, raw_result: Dict[str, Any]) -> AnalyticQSASTScanResu
             summary = self._generate_summary(issues)
             metadata = self._generate_metadata()
 
+            print(issues)
+
             return AnalyticQSASTScanResultModel(
                 scan_id=new_scan_id,
                 issues=issues,

analyticq-backend/analyticq/engine/parser/js/eslint_parser.py

@@ -19,9 +19,8 @@ def __init__(self):
             "column": "column",
             "end_column": "endColumn",
             "severity": "severity",
-            "node_type": "nodeType",
             "code": "source",
-            "issue_metadata": "metadata"
+            "issue_metadata": "issue_metadata"
         }
         super().__init__(tool_name="eslint", field_mapping=field_mapping)
 
@@ -51,23 +50,26 @@ def transform_output(self, raw_result: List[Dict[str, Any]]) -> List[Dict[str, A
             source_code = file_data.get("source", "")
 
             for message in file_data.get("messages", []):
+                rule_id = message.get("ruleId")
+                if not isinstance(rule_id, str):
+                    rule_id = str(rule_id) if rule_id is not None else "unknown"
+
                 transformed_issue = {
-                    "ruleId": message.get("ruleId", "unknown"),
+                    "ruleId": rule_id,
                     "filePath": file_path,
                     "message": message.get("message", "unknown"),
                     "severity": message.get("severity", 0),
                     "line": message.get("line", 0),
                     "endLine": message.get("endLine", message.get("line", 0)),
                     "column": message.get("column", 0),
                     "endColumn": message.get("endColumn", message.get("column", 0)),
-                    "nodeType": message.get("nodeType", "unknown"),
                     "source": source_code
                 }
 
-                # Add any suggestions as metadata
-                if "suggestions" in message:
-                    transformed_issue["metadata"] = {
-                        "suggestions": message.get("suggestions", [])
+                suggestions = message.get("suggestions")
+                if isinstance(suggestions, list) and suggestions:
+                    transformed_issue["issue_metadata"] = {
+                        "suggestions": suggestions
                     }
 
                 transformed_issues.append(transformed_issue)

analyticq-backend/analyticq/engine/parser/js/njsscan_parser.py

@@ -11,7 +11,7 @@ class NjsScanParser(AnalyticQResultParser):
 
     def __init__(self):
         field_mapping = {
-            "ruele_id": "rule_id",
+            "rule_id": "rule_id",
             "message": "description",
             "path": "file_path",
             "start_line": "start_line",

analyticq-backend/analyticq/engine/parser/python/pylint_parser.py

@@ -22,6 +22,7 @@ def __init__(self):
         }
 
         self.severity_mapping = {
+            "info": "INFO",
             "convention": "LOW",
             "refactor": "LOW",
             "warning": "MEDIUM",