Merge pull request #4 from kpj2006/patch-1

feat: Implement semantic clustering and analysis for PRs

Author: kpj2006

.gitignore

@@ -324,3 +324,9 @@ TSWLatexianTemp*
 # option is specified. Footnotes are the stored in a file with suffix Notes.bib.
 # Uncomment the next line to have this generated file ignored.
 #*Notes.bib
+
+__pycache__/
+venv/
+*.pyc
+conflicts_tree.html
+isolated_prs.html

context.md

@@ -0,0 +1,20 @@
+# MiniChain Context
+
+MiniChain is a minimal, educational implementation of a Proof-of-Work (PoW) blockchain written in Python. It includes a P2P network layer, a mempool for transaction management, cryptographic transaction signing, state management with accounts, and a minimal sandboxed smart contract execution environment.
+
+## Architecture & Components
+
+The `minichain` package is composed of the following files, each responsible for a specific part of the blockchain:
+
+- **`__init__.py`**: Exports the primary classes and functions for the package, acting as the public API.
+- **`block.py`**: Defines the `Block` class, which handles block structure, deterministic timestamps, Merkle root calculation for transactions, and serialization for mining and hashing.
+- **`chain.py`**: Contains the `Blockchain` class which manages the chain of blocks, handles adding new blocks after validating their hashes, and executes transactions against a temporary state to ensure atomicity.
+- **`contract.py`**: Implements `ContractMachine`, a minimal, sandboxed Python-based smart contract execution environment. It uses `multiprocessing`, resource limits, and AST validation to restrict available built-ins and prevent malicious code execution.
+- **`mempool.py`**: Defines the `Mempool` class, which holds pending valid transactions before they are mined into a block. It handles duplicate prevention, max size limits, and deterministic sorting for block inclusion.
+- **`p2p.py`**: Implements a lightweight TCP-based peer-to-peer network (`P2PNetwork`) using `asyncio`. It handles connecting to peers, listening for incoming connections, broadcasting blocks and transactions, and validating incoming JSON messages.
+- **`persistence.py`**: Provides utility functions to `save` and `load` the blockchain and state to a local JSON file (`data.json`) atomically. It uses fsync and temp files to prevent corruption and verifies chain integrity on load.
+- **`pow.py`**: Contains the Proof-of-Work mining logic (`mine_block` and `calculate_hash`), enforcing difficulty targets, nonces, limits, and optional timeout conditions.
+- **`serialization.py`**: Provides helper functions (`canonical_json_dumps`, `canonical_json_bytes`, `canonical_json_hash`) for deterministic JSON serialization, crucial for consistent signature verification and hashing across environments.
+- **`state.py`**: Defines the `State` class, which manages account balances, nonces, and contract storage. It evaluates and applies transactions via distinct branches: regular transfers, contract deployments, and contract calls.
+- **`transaction.py`**: Defines the `Transaction` class, handling transaction structure, digital signatures using `nacl` (Ed25519), verification, deterministic transaction IDs, and serialization.
+- **`validators.py`**: Provides simple validation utilities, such as `is_valid_receiver`, ensuring addresses correspond to standard hex lengths (40 or 64 characters).

dashboard.py

@@ -0,0 +1,128 @@
+"""
+github.py — all GitHub CLI interactions
+"""
+
+import subprocess, json, re
+
+REPO = "StabilityNexus/MiniChain"
+
+def gh(endpoint):
+    result = subprocess.run(
+        ["gh", "api", f"https://api.github.com/{endpoint}"],
+        capture_output=True, encoding="utf-8", errors="replace"
+    )
+    if result.returncode != 0:
+        return []
+    try:
+        return json.loads(result.stdout)
+    except Exception:
+        return []
+
+def gh_paginate(endpoint):
+    result = subprocess.run(
+        ["gh", "api", "--paginate", f"https://api.github.com/{endpoint}"],
+        capture_output=True, encoding="utf-8", errors="replace"
+    )
+    if result.returncode != 0:
+        return []
+    text = result.stdout.strip()
+    if not text:
+        return []
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        try:
+            text = "[" + text.replace("][", ",") + "]"
+            flat = json.loads(text)
+            return [i for sub in flat for i in sub] if flat and isinstance(flat[0], list) else flat
+        except Exception:
+            return []
+
+def fetch_prs():
+    # TEST: last 10 closed. For production use gh_paginate below.
+    prs = gh(f"repos/{REPO}/pulls?state=closed&per_page=10&sort=updated&direction=desc")
+    return prs if isinstance(prs, list) else []
+    # PRODUCTION:
+    # return gh_paginate(f"repos/{REPO}/pulls?state=open&per_page=100")
+
+def fetch_pr_files(n):
+    files = gh(f"repos/{REPO}/pulls/{n}/files?per_page=100")
+    return [f["filename"] for f in files] if isinstance(files, list) else []
+
+def fetch_coderabbit_sections(n):
+    """
+    Fetch ONLY the Walkthrough and Changes sections from CodeRabbit.
+    Everything else (sequence diagrams, poems, tips) is ignored.
+    """
+    raw = None
+
+    # Check issue comments first
+    comments = gh(f"repos/{REPO}/issues/{n}/comments?per_page=50")
+    if isinstance(comments, list):
+        for c in comments:
+            if "coderabbit" in c.get("user", {}).get("login", "").lower() and c.get("body"):
+                raw = c["body"]
+                break
+
+    # Fallback: PR reviews
+    if not raw:
+        reviews = gh(f"repos/{REPO}/pulls/{n}/reviews?per_page=50")
+        if isinstance(reviews, list):
+            for r in reviews:
+                if "coderabbit" in r.get("user", {}).get("login", "").lower() and r.get("body"):
+                    raw = r["body"]
+                    break
+
+    if not raw:
+        return None
+
+    return extract_walkthrough_and_changes(raw)
+
+def extract_walkthrough_and_changes(text):
+    """
+    Pull only the Walkthrough and Changes table from a CodeRabbit comment.
+    CodeRabbit structure:
+      ## Walkthrough
+      <text>
+      ## Changes
+      | File | Summary |
+      ...
+      ## Sequence Diagram(s)   ← stop here
+    """
+    result = {}
+
+    # Extract Walkthrough section
+    wt_match = re.search(
+        r"##\s*Walkthrough\s*\n(.*?)(?=\n##\s|\Z)",
+        text, re.DOTALL | re.IGNORECASE
+    )
+    if wt_match:
+        result["walkthrough"] = wt_match.group(1).strip()
+
+    # Extract Changes section (table)
+    ch_match = re.search(
+        r"##\s*Changes\s*\n(.*?)(?=\n##\s|\Z)",
+        text, re.DOTALL | re.IGNORECASE
+    )
+    if ch_match:
+        result["changes"] = ch_match.group(1).strip()
+
+    if not result:
+        # Fallback: return first 800 chars if structure not found
+        result["walkthrough"] = text[:800]
+        result["changes"] = ""
+
+    return result
+
+def extract_linked_issue(body):
+    if not body:
+        return None
+    m = re.search(r"(?:fixes|closes|resolves)\s+#(\d+)", body, re.IGNORECASE)
+    return int(m.group(1)) if m else None
+
+def check_gh_auth():
+    result = subprocess.run(
+        ["gh", "auth", "status"],
+        capture_output=True, encoding="utf-8", errors="replace"
+    )
+    return result.returncode == 0

github.py

@@ -0,0 +1,103 @@
+"""
+grouping.py — semantic clustering via sentence-transformers
+No LLM for grouping. Embeddings + community_detection instead.
+Ollama only called AFTER groups are found, to explain WHY they conflict.
+"""
+
+from sentence_transformers import SentenceTransformer, util
+from ollama import analyse_group, analyse_single_pr
+
+MODEL_NAME = "all-MiniLM-L6-v2"   # ~90MB, downloads once, runs on CPU
+THRESHOLD  = 0.55                  # similarity threshold — tune if too many/few matches
+MIN_SIZE   = 2                     # min PRs to form a conflict group
+
+def resolve_groups(pr_data, repo_context=""):
+    """
+    1. Embed each PR walkthrough using sentence-transformers
+    2. Cluster by semantic similarity (community_detection)
+    3. Groups with 2+ PRs = conflict → deep Ollama analysis
+    4. Singletons = isolated → single Ollama analysis
+    """
+    print("\n  Loading embedding model (first run downloads ~90MB)...")
+    model = SentenceTransformer(MODEL_NAME)
+
+    # Build text to embed: title + walkthrough for each PR
+    texts = []
+    for pr in pr_data:
+        cr  = pr.get("coderabbit") or {}
+        wt  = cr.get("walkthrough", "")[:600]
+        ch  = cr.get("changes", "")[:300]
+        texts.append(f"{pr['title']}\n{wt}\n{ch}")
+
+    print(f"  Embedding {len(pr_data)} PRs...")
+    embeddings = model.encode(texts, convert_to_tensor=True, show_progress_bar=False)
+
+    print(f"  Clustering (threshold={THRESHOLD})...")
+    clusters = util.community_detection(
+        embeddings,
+        min_community_size=MIN_SIZE,
+        threshold=THRESHOLD
+    )
+
+    # clusters = list of lists of indices into pr_data
+    # PRs not in any cluster are isolated
+    clustered_indices = set(idx for cluster in clusters for idx in cluster)
+    isolated_indices  = [i for i in range(len(pr_data)) if i not in clustered_indices]
+
+    print(f"\n  Found {len(clusters)} conflict group(s), {len(isolated_indices)} isolated PR(s)")
+
+    # Show similarity scores for transparency
+    similarity_matrix = util.cos_sim(embeddings, embeddings)
+    for ci, cluster in enumerate(clusters):
+        pr_nums = [pr_data[i]["number"] for i in cluster]
+        scores  = []
+        for x in range(len(cluster)):
+            for y in range(x+1, len(cluster)):
+                s = float(similarity_matrix[cluster[x]][cluster[y]])
+                scores.append(f"#{pr_nums[x]}&#{pr_nums[y]}={s:.2f}")
+        print(f"  Group {ci+1}: PRs {pr_nums} — similarity: {', '.join(scores)}")
+
+    conflict_groups = []
+    isolated_prs    = []
+
+    # ── Conflict groups ───────────────────────────────────────────────────────
+    for cluster in clusters:
+        prs = [pr_data[i] for i in cluster]
+        pr_nums = [p["number"] for p in prs]
+
+        # Build a pseudo group_meta for analyse_group
+        group_meta = {
+            "problem": f"PRs {pr_nums} grouped by semantic similarity",
+            "problem_category": "unknown",
+        }
+
+        print(f"\n  Conflict group: PRs {pr_nums}")
+        print(f"  Running deep group analysis with Ollama...")
+        group_analysis = analyse_group(group_meta, prs, repo_context)
+
+        if group_analysis and "pr_analyses" in group_analysis:
+            pr_by_num = {p["number"]: p for p in prs}
+            for pa in group_analysis["pr_analyses"]:
+                n = pa.get("pr_number")
+                if n in pr_by_num:
+                    pr_by_num[n]["analysis"] = pa
+
+        # Use avg similarity to guess duplicate vs conflict (NLI will improve this later)
+        avg_sim = float(similarity_matrix[cluster[0]][cluster[1]]) if len(cluster) > 1 else 0
+        gtype = "duplicate" if avg_sim > 0.75 else "conflict"
+        conflict_groups.append({
+            "category":   group_analysis.get("problem_category", "unknown") if group_analysis else "unknown",
+            "problem":    group_analysis.get("shared_problem", f"PRs {pr_nums}") if group_analysis else f"PRs {pr_nums}",
+            "prs":        prs,
+            "analysis":   group_analysis or {},
+            "group_type": gtype,
+        })
+
+    # ── Isolated PRs ──────────────────────────────────────────────────────────
+    for i in isolated_indices:
+        pr = pr_data[i]
+        print(f"\n  Isolated PR #{pr['number']} — analysing...")
+        pr["analysis"] = analyse_single_pr(pr, repo_context)
+        isolated_prs.append(pr)
+
+    return conflict_groups, isolated_prs

grouping.py

@@ -0,0 +1,94 @@
+"""
+main.py — entry point
+
+Flow:
+  1. Load context.md (repo context — what MiniChain is, what's already built)
+  2. Fetch all PRs + extract CodeRabbit walkthrough + changes only
+  3. One combined Ollama call → groups PRs by problem
+  4. Deep Ollama analysis per conflict group (all PRs in group together)
+  5. Single Ollama analysis per isolated PR
+  6. Render two HTML files and open in browser
+
+Run: python main.py
+Requires: gh (authenticated), ollama running on localhost:11434
+Optional: context.md in same folder (drop it in when ready)
+"""
+
+import os, time, webbrowser
+from github   import fetch_prs, fetch_pr_files, fetch_coderabbit_sections, check_gh_auth, REPO
+from ollama   import check_ollama
+from grouping import resolve_groups
+from render   import build_conflict_html, build_isolated_html
+
+OUT_DIR      = os.path.dirname(os.path.abspath(__file__))
+CONTEXT_FILE = os.path.join(OUT_DIR, "context.md")
+
+def load_context():
+    if os.path.exists(CONTEXT_FILE):
+        with open(CONTEXT_FILE, "r", encoding="utf-8") as f:
+            content = f.read().strip()
+        print(f"  Loaded context.md ({len(content)} chars)")
+        return content
+    print("  No context.md found — running without repo context")
+    print("  (Drop context.md in the same folder to enable it)")
+    return ""
+
+def main():
+    if not check_gh_auth():
+        print("ERROR: gh not authenticated. Run: gh auth login")
+        return
+
+    if not check_ollama():
+        print("ERROR: Ollama not reachable at localhost:11434")
+        return
+
+    print(f"\nLoading repo context...")
+    repo_context = load_context()
+
+    print(f"\nFetching PRs for {REPO}...")
+    raw_prs = fetch_prs()
+    if not raw_prs:
+        print("No PRs found.")
+        return
+
+    print(f"Found {len(raw_prs)} PRs. Fetching walkthroughs...\n")
+    pr_data = []
+    for raw in raw_prs:
+        num    = raw["number"]
+        author = raw["user"]["login"]
+        print(f"  PR #{num} — {raw['title'][:55]}")
+        pr_data.append({
+            "number":     num,
+            "title":      raw["title"],
+            "author":     author,
+            "created_at": raw["created_at"],
+            "body":       raw.get("body", "") or "",
+            "files":      fetch_pr_files(num),
+            "coderabbit": fetch_coderabbit_sections(num),
+        })
+
+    # ── Step 2: Semantic clustering + deep analysis ──────────────────────────
+    print(f"\nClustering {len(pr_data)} PRs by semantic similarity...")
+    conflict_groups, isolated = resolve_groups(pr_data, repo_context)
+
+    print(f"\n  Conflict groups : {len(conflict_groups)}")
+    print(f"  Isolated PRs    : {len(isolated)}")
+
+    # ── Render ────────────────────────────────────────────────────────────────
+    tree_path = os.path.join(OUT_DIR, "conflicts_tree.html")
+    iso_path  = os.path.join(OUT_DIR, "isolated_prs.html")
+
+    with open(tree_path, "w", encoding="utf-8") as f:
+        f.write(build_conflict_html(conflict_groups))
+    with open(iso_path, "w", encoding="utf-8") as f:
+        f.write(build_isolated_html(isolated))
+
+    print(f"\nOpening conflict tree -> {tree_path}")
+    webbrowser.open(f"file:///{tree_path}")
+    time.sleep(1)
+    print(f"Opening isolated PRs  -> {iso_path}")
+    webbrowser.open(f"file:///{iso_path}")
+    print("\nDone.")
+
+if __name__ == "__main__":
+    main()