Add email-first knowledge graph with IMAP ingestion

- Add IMAP polling service for email ingestion
- Add MIME parser for extracting email content and thread refs
- Add email lens with primitives (person, org, action_item, decision, date, topic)
- Add auto-extraction pipeline with webhook subscription
- Extend extraction.py with zone-aware email extraction (subject/body)
- Add inbox web UI with email list, search, pagination
- Add email detail view with inline anchor highlights
- Add D3 graph panel showing anchor connections
- Add sync state persistence for resumable polling
- Add Flask routes for /inbox and email APIs

Author: systemshift

scripts/create_email_subscription.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""
+Create the email auto-extraction subscription in Memex.
+
+This subscription triggers a webhook when Email nodes are created,
+allowing automatic anchor extraction.
+"""
+
+import sys
+import os
+import requests
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'workspace'))
+
+from config.email import email_config
+
+MEMEX_URL = os.getenv("MEMEX_URL", "http://localhost:8080")
+
+SUBSCRIPTION = {
+    "id": "sub:email-auto-extract",
+    "name": "Email Auto-Extraction",
+    "description": "Automatically extract anchors when emails are ingested",
+    "pattern": {
+        "event_types": ["node.created"],
+        "node_types": ["Email"]
+    },
+    "webhook": {
+        "url": email_config.extraction_webhook,
+        "method": "POST",
+        "headers": {
+            "Content-Type": "application/json"
+        },
+        "retry": {
+            "max_attempts": 3,
+            "backoff_ms": 1000
+        }
+    },
+    "enabled": True
+}
+
+
+def create_subscription():
+    """Create the email extraction subscription"""
+    print("Creating email auto-extraction subscription...")
+    print(f"  Memex URL: {MEMEX_URL}")
+    print(f"  Webhook URL: {SUBSCRIPTION['webhook']['url']}")
+
+    try:
+        # Check if subscription exists
+        resp = requests.get(f"{MEMEX_URL}/api/subscriptions/{SUBSCRIPTION['id']}")
+        if resp.status_code == 200:
+            print(f"  Subscription {SUBSCRIPTION['id']} already exists, updating...")
+            resp = requests.put(
+                f"{MEMEX_URL}/api/subscriptions/{SUBSCRIPTION['id']}",
+                json=SUBSCRIPTION
+            )
+        else:
+            print(f"  Creating new subscription...")
+            resp = requests.post(
+                f"{MEMEX_URL}/api/subscriptions",
+                json=SUBSCRIPTION
+            )
+
+        if resp.status_code in [200, 201]:
+            print(f"  Successfully created/updated subscription: {SUBSCRIPTION['id']}")
+            print("\nSubscription details:")
+            print(f"  - Triggers on: {SUBSCRIPTION['pattern']['event_types']}")
+            print(f"  - For node types: {SUBSCRIPTION['pattern']['node_types']}")
+            print(f"  - Webhook: {SUBSCRIPTION['webhook']['url']}")
+            return True
+        else:
+            print(f"  Failed to create subscription: {resp.status_code}")
+            print(f"  Response: {resp.text}")
+            return False
+
+    except requests.exceptions.ConnectionError:
+        print(f"  Error: Could not connect to Memex at {MEMEX_URL}")
+        print("  Is memex-server running?")
+        return False
+    except Exception as e:
+        print(f"  Error: {e}")
+        return False
+
+
+def list_subscriptions():
+    """List all subscriptions"""
+    try:
+        resp = requests.get(f"{MEMEX_URL}/api/subscriptions")
+        if resp.status_code == 200:
+            data = resp.json()
+            subs = data.get("subscriptions", [])
+            print(f"\nExisting subscriptions ({len(subs)}):")
+            for sub in subs:
+                status = "enabled" if sub.get("enabled") else "disabled"
+                print(f"  - {sub.get('id')}: {sub.get('name')} [{status}]")
+        else:
+            print("Could not list subscriptions")
+    except Exception as e:
+        print(f"Error listing subscriptions: {e}")
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Manage email extraction subscription")
+    parser.add_argument("--list", action="store_true", help="List all subscriptions")
+    parser.add_argument("--delete", action="store_true", help="Delete the subscription")
+
+    args = parser.parse_args()
+
+    if args.list:
+        list_subscriptions()
+    elif args.delete:
+        try:
+            resp = requests.delete(f"{MEMEX_URL}/api/subscriptions/{SUBSCRIPTION['id']}")
+            if resp.status_code in [200, 204]:
+                print(f"Deleted subscription: {SUBSCRIPTION['id']}")
+            else:
+                print(f"Failed to delete: {resp.status_code}")
+        except Exception as e:
+            print(f"Error: {e}")
+    else:
+        success = create_subscription()
+        if success:
+            list_subscriptions()
+        sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/run_email_poller.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""
+Run the email IMAP poller.
+
+This script starts the email polling service that:
+1. Connects to the configured IMAP server
+2. Polls for new emails at regular intervals
+3. Ingests emails into Memex as Email nodes
+4. Persists sync state for resumable operation
+
+Configuration via environment variables:
+- IMAP_HOST: IMAP server hostname (default: imap.gmail.com)
+- IMAP_PORT: IMAP server port (default: 993)
+- EMAIL_USERNAME: Email address/username
+- EMAIL_PASSWORD: Email password (use app-specific password for Gmail)
+- EMAIL_MAILBOX: Mailbox to poll (default: INBOX)
+- EMAIL_POLL_INTERVAL: Seconds between polls (default: 60)
+- EMAIL_BATCH_SIZE: Max emails per poll (default: 50)
+"""
+
+import sys
+import os
+import signal
+import threading
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'workspace'))
+
+from config.email import email_config
+from services.email_ingestion import IMAPPoller
+from services.email_sync import get_sync_state
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run the email IMAP poller",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run with environment variables
+  EMAIL_USERNAME=user@gmail.com EMAIL_PASSWORD=xxxx python run_email_poller.py
+
+  # Run once (no polling loop)
+  python run_email_poller.py --once
+
+  # Reset sync state and start fresh
+  python run_email_poller.py --reset
+
+  # Show current sync status
+  python run_email_poller.py --status
+"""
+    )
+    parser.add_argument("--once", action="store_true", help="Poll once and exit")
+    parser.add_argument("--reset", action="store_true", help="Reset sync state")
+    parser.add_argument("--status", action="store_true", help="Show sync status")
+    parser.add_argument("--test", action="store_true", help="Test connection only")
+    parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
+
+    args = parser.parse_args()
+
+    # Validate configuration
+    errors = email_config.validate()
+    if errors and not args.status:
+        print("Configuration errors:")
+        for error in errors:
+            print(f"  - {error}")
+        print("\nSet environment variables or create a .env file")
+        sys.exit(1)
+
+    sync_state = get_sync_state()
+
+    # Handle --status
+    if args.status:
+        stats = sync_state.get_stats()
+        print("Email Sync Status")
+        print("=" * 40)
+        print(f"Account: {email_config.username or '(not configured)'}")
+        print(f"IMAP Host: {email_config.imap_host}")
+        print(f"Mailbox: {email_config.mailbox}")
+        print(f"Last UID: {stats['last_uid']}")
+        print(f"Last Sync: {stats['last_sync'] or 'never'}")
+        print(f"Total Emails Ingested: {stats['total_emails_ingested']}")
+        print(f"Total Errors: {stats['total_errors']}")
+        return
+
+    # Handle --reset
+    if args.reset:
+        print("Resetting sync state...")
+        sync_state.reset()
+        print("Sync state reset. Next poll will start fresh.")
+        return
+
+    # Create poller
+    poller = IMAPPoller(email_config)
+
+    # Set callbacks
+    def on_email_ingested(node_id, email_msg):
+        if args.verbose:
+            print(f"  Ingested: {email_msg.subject[:60]}")
+
+    def on_error(error):
+        print(f"  Error: {error}")
+        sync_state.record_sync(0, errors=1)
+
+    poller.on_email_ingested = on_email_ingested
+    poller.on_error = on_error
+
+    # Restore last UID from sync state
+    poller.last_uid = sync_state.get_last_uid()
+
+    print("=" * 50)
+    print("MEMEX EMAIL POLLER")
+    print("=" * 50)
+    print(f"Host: {email_config.imap_host}")
+    print(f"User: {email_config.username}")
+    print(f"Mailbox: {email_config.mailbox}")
+    print(f"Poll Interval: {email_config.poll_interval}s")
+    print(f"Starting from UID: {poller.last_uid}")
+    print("=" * 50)
+
+    # Handle --test
+    if args.test:
+        print("\nTesting IMAP connection...")
+        if poller.connect():
+            print("Connection successful!")
+            poller.disconnect()
+        else:
+            print("Connection failed!")
+            sys.exit(1)
+        return
+
+    # Handle --once
+    if args.once:
+        print("\nPolling once...")
+        count = poller.poll_once()
+        print(f"Ingested {count} emails")
+
+        # Save state
+        sync_state.set_last_uid(poller.last_uid)
+        sync_state.record_sync(count)
+
+        poller.disconnect()
+        return
+
+    # Setup signal handlers for graceful shutdown
+    stop_event = threading.Event()
+
+    def signal_handler(signum, frame):
+        print("\nShutting down...")
+        stop_event.set()
+        poller.stop()
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    # Run polling loop
+    print("\nStarting poll loop (Ctrl+C to stop)...")
+    print()
+
+    try:
+        while not stop_event.is_set():
+            count = poller.poll_once()
+
+            # Save state after each poll
+            if count > 0 or poller.last_uid > sync_state.get_last_uid():
+                sync_state.set_last_uid(poller.last_uid)
+                sync_state.record_sync(count)
+
+            # Wait for next poll
+            stop_event.wait(timeout=email_config.poll_interval)
+
+    except KeyboardInterrupt:
+        pass
+    finally:
+        poller.disconnect()
+        print("Poller stopped.")
+        stats = sync_state.get_stats()
+        print(f"Final UID: {stats['last_uid']}")
+        print(f"Total emails ingested this session: {stats['total_emails_ingested']}")
+
+
+if __name__ == "__main__":
+    main()