diff --git a/.gitignore b/.gitignore index 2eea525..43166c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ -.env \ No newline at end of file +.env +target/ +*.lock +.venv/ +__pycache__/ diff --git a/scrapper/START.md b/scrapper/START.md new file mode 100644 index 0000000..7a831d0 --- /dev/null +++ b/scrapper/START.md @@ -0,0 +1,231 @@ +# Scrapper Startup Guide + +This guide explains how to launch the technical watch scrapper system. + +## Prerequisites + +- **Python 3.9+** +- **PostgreSQL** with **pgvector** extension +- **OpenAI API Key** (for embeddings and entity extraction) +- **(Optional)** GitHub Token for higher rate limits + +## Installation + +### 1. Create a Python Virtual Environment + +```bash +cd scrapper +python3 -m venv .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate +``` + +### 2. Install Dependencies + +```bash +pip install -r requirements.txt +``` + +### 3. Setup PostgreSQL with pgvector + +Install PostgreSQL and the pgvector extension: + +```bash +# On Ubuntu/Debian +sudo apt install postgresql postgresql-contrib +sudo -u postgres psql -c "CREATE EXTENSION vector;" + +# Or using Docker +docker run -d \ + --name postgres-pgvector \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=veille_technique \ + -p 5432:5432 \ + pgvector/pgvector:pg16 +``` + +### 4. Configure Environment Variables + +Copy the example environment file and configure it: + +```bash +cp .env.example .env +``` + +Edit `.env` and set your credentials: + +```env +OPENAI_API_KEY=your_openai_api_key_here +DATABASE_URL=postgresql://postgres:postgres@localhost:5432/veille_technique +EMBEDDING_MODEL=text-embedding-3-small +GITHUB_TOKEN=your_github_token_here # Optional +``` + +### 5. Initialize the Database + +The database schema will be created automatically on first run. + +## Running the Scrapper + +The scrapper has **3 modes**: + +### 1. Backfill Mode (Historical Data) + +Scrape entire available history from all sources: + +```bash +python main.py backfill +``` + +Options: +- `--limit N` - Maximum articles per source (default: 100) +- `--db-url URL` - Override database URL +- `--embedding-model MODEL` - Override embedding model +- `--llm-model MODEL` - Override LLM model for entities + +Example with custom limit: +```bash +python main.py backfill --limit 200 +``` + +### 2. Watch Mode (Continuous Monitoring) + +Scrape new articles continuously at regular intervals: + +```bash +python main.py watch +``` + +Options: +- `--interval SECONDS` - Scraping interval (default: 300s = 5 minutes) +- `--db-url URL` - Override database URL +- `--embedding-model MODEL` - Override embedding model +- `--llm-model MODEL` - Override LLM model for entities + +Example with 10-minute interval: +```bash +python main.py watch --interval 600 +``` + +Press `Ctrl+C` to stop the watch mode. + +### 3. Stats Mode (View Statistics) + +Display database statistics: + +```bash +python main.py stats +``` + +## Available Scrapers + +The system includes scrapers for: + +- **ArXiv** - Scientific papers (cs.LG category by default) +- **GitHub** - Trending repositories +- **Medium** - Technical articles +- **Le Monde** - News articles +- **Hugging Face** - ML models and papers + +## Features + +Each scraped article is automatically: +1. **Deduplicated** - By ID and content hash +2. **Embedded** - Using OpenAI embeddings (for similarity search) +3. **Analyzed** - Entities extracted via LLM (technologies, companies, people, etc.) + +## Configuration + +### Database Connection + +Set via environment variable or command-line: +- Environment: `DATABASE_URL=postgresql://user:pass@host:port/dbname` +- CLI: `--db-url postgresql://user:pass@host:port/dbname` + +### Embedding Model + +Configure the OpenAI embedding model: +- Environment: `EMBEDDING_MODEL=text-embedding-3-small` +- CLI: `--embedding-model text-embedding-3-small` + +Available models: +- `text-embedding-3-small` (1536 dimensions, faster) +- `text-embedding-3-large` (3072 dimensions, more accurate) + +### LLM Model + +Configure the LLM for entity extraction: +- Environment: `LLM_MODEL=gpt-4o-mini` +- CLI: `--llm-model gpt-4o-mini` + +## Troubleshooting + +### Missing OpenAI API Key + +``` +Error: OpenAI API key not found +``` + +**Solution**: Set `OPENAI_API_KEY` in your `.env` file. + +### PostgreSQL Connection Error + +``` +Error: could not connect to server +``` + +**Solution**: +1. Check PostgreSQL is running: `sudo systemctl status postgresql` +2. Verify DATABASE_URL in `.env` +3. Ensure pgvector extension is installed + +### Scraper Initialization Failed + +If a scraper fails to initialize, it will be skipped automatically. Check the logs for details. + +### Port Already in Use (PostgreSQL) + +If port 5432 is already used, either: +1. Stop the conflicting service +2. Use a different port in `DATABASE_URL` + +## Development Tips + +### Check Architecture + +See [ARCHITECTURE.md](ARCHITECTURE.md) for system design details. + +### Database Management + +View articles directly in PostgreSQL: +```sql +-- Connect to database +psql $DATABASE_URL + +-- Count articles +SELECT COUNT(*) FROM articles; + +-- View recent articles +SELECT title, source, published_at FROM articles +ORDER BY published_at DESC LIMIT 10; + +-- Check embeddings +SELECT COUNT(*) FROM embeddings; +``` + +## Recommended Workflow + +1. **Initial setup**: Run `backfill` mode once to populate historical data +2. **Continuous monitoring**: Run `watch` mode to keep data up-to-date +3. **Check progress**: Use `stats` mode to monitor collection + +Example: +```bash +# One-time: populate history +python main.py backfill --limit 50 + +# Continuous: monitor new content +python main.py watch --interval 600 + +# Anytime: check statistics +python main.py stats +``` diff --git a/scrapper/examples.py b/scrapper/examples.py deleted file mode 100644 index 606606e..0000000 --- a/scrapper/examples.py +++ /dev/null @@ -1,147 +0,0 @@ -""" -Usage examples for the watch server. -""" - -import asyncio -import os -from main import WatchServer - - -def example_backfill(): - """Example 1: Backfill mode (history).""" - print("\n" + "="*60) - print("EXAMPLE 1 : BACKFILL Mode") - print("="*60) - - server = WatchServer( - db_url=os.getenv("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/veille_technique"), - check_interval=300 - ) - - print("\n✓ Server created, launching backfill...") - server.run_backfill_mode(limit_per_scraper=10) - - server.print_stats() - - -def example_watch_limited(): - """Example 2: Watch mode with iteration limit (for testing).""" - print("\n" + "="*60) - print("EXAMPLE 2 : WATCH Mode (limited to 3 iterations)") - print("="*60) - - server = WatchServer( - db_url=os.getenv("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/veille_technique"), - check_interval=10 - ) - - print("\n✓ Server created, launching watch (3 iterations)...") - print(" Each iteration scrapes all sources\n") - - try: - import threading - - def stop_after_30s(srv): - import time - time.sleep(30) - srv.running = False - print("\n⏹️ Auto-stopped after 30 seconds") - - thread = threading.Thread(target=stop_after_30s, args=(server,), daemon=True) - thread.start() - - asyncio.run(server.run_watch_mode()) - except KeyboardInterrupt: - print("\n⏹️ Stopped by user") - - server.print_stats() - - -def example_multi_source_stats(): - """Example 3: View stats with multiple sources scraped.""" - print("\n" + "="*60) - print("EXAMPLE 3 : Backfill + Stats") - print("="*60) - - server = WatchServer(db_url=os.getenv("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/veille_technique")) - - print("\n📥 Scraping each source...") - server.run_backfill_mode(limit_per_scraper=5) - - print("\n📊 Checking stats...") - stats = server.get_stats() - - print(f"\nSummary :") - print(f" Total articles : {stats['total_articles']}") - print(f" Embeddings : {stats['total_embeddings']}") - print(f" Missing : {stats['articles_without_embeddings']}") - - print(f"\nPer source :") - for source, count in sorted(stats['articles_by_source'].items()): - pct = 100 * count / max(1, stats['total_articles']) - print(f" {source:15} : {count:3} ({pct:.1f}%)") - - -def example_custom_config(): - """Example 4: Use custom configuration.""" - print("\n" + "="*60) - print("EXAMPLE 4 : Custom Configuration") - print("="*60) - - from config import ServerConfig, ScraperConfig - - custom_config = ServerConfig( - db_url=os.getenv("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/veille_technique"), - watch_interval_seconds=120, - scrapers={ - "arxiv": ScraperConfig(enabled=True, limit_latest=10, limit_all=30), - "github": ScraperConfig(enabled=True, limit_latest=15, limit_all=50), - "medium": ScraperConfig(enabled=False), - "lemonde": ScraperConfig(enabled=False), - "huggingface": ScraperConfig(enabled=True, limit_latest=10, limit_all=30), - } - ) - - print(f"\n✓ Custom config created") - print(f" DB : {custom_config.db_url}") - print(f" Interval : {custom_config.watch_interval_seconds}s") - - print(f"\n✓ Enabled scrapers :") - for name, cfg in custom_config.scrapers.items(): - if cfg.enabled: - print(f" - {name} (latest:{cfg.limit_latest}, all:{cfg.limit_all})") - - -if __name__ == "__main__": - import sys - - print("\n" + "="*60) - print("USAGE EXAMPLES") - print("Watch Server") - print("="*60) - - examples = { - "1": ("Backfill (history)", example_backfill), - "2": ("Watch limited (test)", example_watch_limited), - "3": ("Multi-source stats", example_multi_source_stats), - "4": ("Custom config", example_custom_config), - } - - print("\nChoose an example :") - for key, (desc, _) in examples.items(): - print(f" {key}. {desc}") - - if len(sys.argv) > 1: - choice = sys.argv[1] - else: - choice = input("\nEnter your choice (1-4) : ").strip() - - if choice in examples: - try: - examples[choice][1]() - except Exception as e: - print(f"\n❌ Error : {e}") - import traceback - traceback.print_exc() - else: - print(f"❌ Invalid choice : {choice}") diff --git a/scrapper/examples_new_features.py b/scrapper/examples_new_features.py deleted file mode 100644 index c55e1a4..0000000 --- a/scrapper/examples_new_features.py +++ /dev/null @@ -1,238 +0,0 @@ -""" -Examples demonstrating the new primary/secondary entity extraction and cross-encoder usage. -""" - -import os -import json -from typing import Dict, List - -from cross_encoder import CrossEncoderManager -from entity_llm_processor import EntityLLMProcessor -from database import DatabaseManager - - -def example_cross_encoder_relevance(): - """ - Example: Using Cross Encoder to compute semantic relevance between articles. - - The cross encoder is more efficient than computing embeddings for many pairs - and provides direct relevance scores optimized for ranking/clustering tasks. - """ - print("\n" + "="*60) - print("EXAMPLE: Cross-Encoder Relevance Scoring") - print("="*60) - - # Initialize cross encoder - ce_manager = CrossEncoderManager() - - # Sample articles - article1 = { - "id": "arxiv_001", - "title": "Efficient Transformer Architectures with Flash Attention", - "description": "New optimization techniques for transformer models", - "full_content": "This paper presents Flash Attention, an I/O-aware attention mechanism..." - } - - article2 = { - "id": "github_001", - "title": "OpenAI Releases GPT-4 Turbo Model", - "description": "Announcement of new capabilities in GPT-4 Turbo", - "full_content": "OpenAI has released GPT-4 Turbo with improved performance..." - } - - article3 = { - "id": "arxiv_002", - "title": "Attention Mechanisms and Transformer Optimization", - "description": "Survey of efficiency improvements in attention", - "full_content": "This comprehensive survey covers attention optimization techniques..." - } - - # Compute relevance scores - print("\nComputing relevance scores:") - print(f"\n1. Article1 vs Article2 (different topics):") - score_1_2 = ce_manager.compute_relevance_score(article1, article2) - print(f" Relevance: {score_1_2:.3f}") - - print(f"\n2. Article1 vs Article3 (similar topics - attention/transformers):") - score_1_3 = ce_manager.compute_relevance_score(article1, article3) - print(f" Relevance: {score_1_3:.3f}") - - print("\n✓ Article1 and Article3 should have higher relevance (same topic)") - print(f" Difference: {score_1_3 - score_1_2:.3f}") - - -def example_primary_secondary_extraction(): - """ - Example: Using enhanced LLM processor with primary/secondary entity extraction. - - The new system distinguishes between: - - PRIMARY entities: Central focus of the article - - SECONDARY entities: Supporting or contextual information - - This allows more nuanced clustering and relevance matching. - """ - print("\n" + "="*60) - print("EXAMPLE: Primary/Secondary Entity Extraction") - print("="*60) - - processor = EntityLLMProcessor(model="gpt-4o-mini") - - # Example article - article = { - "id": "arxiv_123", - "title": "OpenAI Releases GPT-4 with Multimodal Capabilities", - "description": "OpenAI announces GPT-4, a new multimodal AI model with vision understanding", - "full_content": ( - "OpenAI has released GPT-4, their most advanced model to date. " - "GPT-4 can process both text and images, setting a new standard for AI capabilities. " - "The model was trained using RLHF with input from domain experts. " - "Anthropic's Claude model also supports multimodal inputs, showing industry convergence." - ) - } - - print(f"\nProcessing article: {article['title']}") - print("\nExpected extraction:") - print(" PRIMARY subject: 'Large Language Models' or 'GPT-4'") - print(" SECONDARY subject: 'Multimodal AI' or 'Vision Understanding'") - print(" PRIMARY organizations: ['OpenAI']") - print(" SECONDARY organizations: ['Anthropic']") - print(" PRIMARY event: 'Model Release'") - print(" SECONDARY event: 'Industry Convergence'") - - print("\nNote: In production, this would be called on actual article processing") - print(" and would integrate with the database clustering system.") - - -def example_clustering_with_primary_secondary(): - """ - Example: How primary/secondary entities improve clustering. - - The clustering algorithm now considers: - 1. Embedding similarity (from vector database) - 30% weight - 2. Cross-encoder relevance score - 30% weight - 3. Entity matching with primary/secondary distinction - 40% weight - - This creates more meaningful clusters with better article grouping. - """ - print("\n" + "="*60) - print("EXAMPLE: Clustering with Primary/Secondary Entities") - print("="*60) - - print("\nClustering Flow:") - print("1. Article enters the system") - print("2. LLM extracts PRIMARY and SECONDARY entities") - print("3. Embedding is computed via text-embedding-3-small") - print("4. Candidate articles are fetched (vector similarity)") - print("5. For each candidate:") - print(" a) Embedding similarity: cosine distance (30%)") - print(" b) Cross-encoder score: semantic relevance (30%)") - print(" c) Entity matching: primary > secondary (40%)") - print("6. Best matching cluster is selected") - print("7. If no good match: create new cluster") - - print("\nEntity Matching Weights:") - print(" PRIMARY subject match : +1.0") - print(" SECONDARY subject match : +0.3") - print(" PRIMARY event match : +0.5") - print(" SECONDARY event match : +0.2") - print(" PRIMARY organizations match : +0.3 per org") - print(" SECONDARY organizations match: +0.1 per org") - - print("\n✓ Primary entities weighted higher = more focused clustering") - - -def example_database_schema(): - """ - Show the updated database schema with primary/secondary support. - """ - print("\n" + "="*60) - print("DATABASE SCHEMA - Articles Table") - print("="*60) - - schema = { - "articles": { - "columns": { - "id": "TEXT PRIMARY KEY", - "source_site": "TEXT", - "title": "TEXT", - "description": "TEXT", - "full_content": "TEXT", - "primary_subject": "TEXT [NEW]", - "secondary_subject": "TEXT [NEW]", - "primary_organizations": "JSONB [NEW] (array of strings)", - "secondary_organizations": "JSONB [NEW] (array of strings)", - "primary_event_type": "TEXT [NEW]", - "secondary_event_type": "TEXT [NEW]", - "cluster_id": "INTEGER", - "created_at": "TIMESTAMPTZ", - "updated_at": "TIMESTAMPTZ", - } - } - } - - print("\nNew columns for entity extraction:") - for col in ["primary_subject", "secondary_subject", "primary_organizations", - "secondary_organizations", "primary_event_type", "secondary_event_type"]: - print(f" ✓ {col}") - - print("\n✓ Old columns removed: subject, organization_list, event_type") - - -def example_llm_prompt(): - """ - Show the new LLM system prompt for entity extraction. - """ - print("\n" + "="*60) - print("LLM SYSTEM PROMPT - Enhanced with Primary/Secondary") - print("="*60) - - prompt = """You are an expert technical analysis system. Your task is to extract -key entities from a given technical article text and return them in JSON format. - -Extract and categorize entities as follows: -1. PRIMARY entities: Main subjects/topics that are the central focus of the article -2. SECONDARY entities: Supporting topics, related subjects, or contextual information - -For organizations and event types, also use primary/secondary classification: -- PRIMARY organizations: Directly involved or central to the article -- SECONDARY organizations: Mentioned but peripheral to the main narrative -- PRIMARY event type: The main event/announcement being discussed -- SECONDARY event types: Related or background events - -Return results in JSON format with clear separation between primary and secondary entities.""" - - print(prompt) - - print("\n\nExpected JSON output structure:") - expected_output = { - "primary_subject": "Main topic of the article", - "secondary_subject": "Supporting or related topic", - "primary_organizations": ["Org1", "Org2"], - "secondary_organizations": ["Org3"], - "primary_event_type": "Main event (e.g., 'Model Release', 'Acquisition')", - "secondary_event_type": "Related event or context" - } - print(json.dumps(expected_output, indent=2)) - - -def main(): - """Run all examples.""" - print("\n" + "="*60) - print("TECHNICAL WATCH SERVER - NEW FEATURES EXAMPLES") - print("="*60) - - # Note: Uncomment to run examples that require API keys - # example_cross_encoder_relevance() - # example_primary_secondary_extraction() - - example_clustering_with_primary_secondary() - example_database_schema() - example_llm_prompt() - - print("\n" + "="*60) - print("For production usage, check main.py watch/backfill modes") - print("="*60) - - -if __name__ == "__main__": - main() diff --git a/scrapper/requirements.txt b/scrapper/requirements.txt index 5f8243c..ec999ae 100644 --- a/scrapper/requirements.txt +++ b/scrapper/requirements.txt @@ -6,4 +6,5 @@ beautifulsoup4==4.12.2 PyPDF2==3.0.1 openai==1.54.0 psycopg[binary]==3.2.1 -pgvector==0.2.5sentence-transformers==3.0.1 \ No newline at end of file +pgvector==0.2.5 +sentence-transformers==3.0.1 \ No newline at end of file diff --git a/scrapper/scrapers/arxiv_scraper.py b/scrapper/scrapers/arxiv_scraper.py index 4563d4d..8a2a7ba 100644 --- a/scrapper/scrapers/arxiv_scraper.py +++ b/scrapper/scrapers/arxiv_scraper.py @@ -123,8 +123,8 @@ def _normalize_result(self, paper) -> Dict: item_type="paper" ) - def scrape_latest(self, limit: int = 20) -> List[Dict]: - """Scrape latest articles.""" + def _perform_search(self, limit: int) -> List[Dict]: + """Perform search with given limit.""" try: search = arxiv.Search( query=f"cat:{self.category}", @@ -141,26 +141,13 @@ def scrape_latest(self, limit: int = 20) -> List[Dict]: return results except Exception as e: - print(f"[ERROR] ArXiv scrape_latest: {e}") + print(f"[ERROR] ArXiv search: {e}") return [] + + def scrape_latest(self, limit: int = 20) -> List[Dict]: + """Scrape latest articles.""" + return self._perform_search(limit) def scrape_all(self, limit: int = 100) -> List[Dict]: """Scrape all available articles (with limit).""" - try: - search = arxiv.Search( - query=f"cat:{self.category}", - max_results=limit, - sort_by=arxiv.SortCriterion.SubmittedDate, - sort_order=arxiv.SortOrder.Descending - ) - - results = [] - for paper in search.results(): - results.append(self._normalize_result(paper)) - - self.update_last_check() - return results - - except Exception as e: - print(f"[ERROR] ArXiv scrape_all: {e}") - return [] + return self._perform_search(limit) diff --git a/scrapper/scrapers/huggingface_scraper.py b/scrapper/scrapers/huggingface_scraper.py index 9a32abc..7d43ec2 100644 --- a/scrapper/scrapers/huggingface_scraper.py +++ b/scrapper/scrapers/huggingface_scraper.py @@ -23,12 +23,18 @@ def __init__(self): def _build_url(self, item: Dict, item_type: str) -> str: """Build public URL for item.""" base = "https://huggingface.co" - item_id = item.get("id") + item_id = item.get("id") or item.get("modelId") or item.get("name") if item_type == "model": - return f"{base}/{item.get('modelId')}" - elif item_type in ("dataset", "space", "collection", "paper"): return f"{base}/{item_id}" + elif item_type == "dataset": + return f"{base}/datasets/{item_id}" + elif item_type == "space": + return f"{base}/spaces/{item_id}" + elif item_type == "collection": + return f"{base}/collections/{item_id}" + elif item_type == "paper": + return f"{base}/papers/{item_id}" return base @@ -37,8 +43,12 @@ def _fetch_model_card(self, item_id: str, item_type: str) -> str: try: if item_type == "model": url = f"https://huggingface.co/{item_id}/raw/main/README.md" + elif item_type == "dataset": + url = f"https://huggingface.co/datasets/{item_id}/raw/main/README.md" + elif item_type == "space": + url = f"https://huggingface.co/spaces/{item_id}/raw/main/README.md" else: - url = f"https://huggingface.co/{item_id}/raw/main/README.md" + return "" resp = requests.get(url, timeout=10) if resp.status_code == 200: diff --git a/server/Cargo.toml b/server/Cargo.toml index bcf540b..82d40db 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -6,3 +6,12 @@ edition = "2021" [dependencies] axum = "0.7" tokio = { version = "1", features = ["full"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid"] } +dotenv = "0.15" +argon2 = "0.5" +jsonwebtoken = "9" +chrono = { version = "0.4", features = ["serde"] } +uuid = { version = "1.10", features = ["serde", "v4"] } +tower-http = { version = "0.5", features = ["cors"] } diff --git a/server/root.md b/server/root.md new file mode 100644 index 0000000..05bd733 --- /dev/null +++ b/server/root.md @@ -0,0 +1,120 @@ +# API Routes (Rust server) + +Base URL: http://localhost:3000 + +## GET / +Returns a simple root message. + +```bash +curl -s http://localhost:3000/ +``` + +## GET /articles/count +Returns the number of rows in the `articles` table. + +```bash +curl -s http://localhost:3000/articles/count +``` + +Response example: + +```json +{"count": 123} +``` + +## POST /auth/register +Creates a new user. + +Request body: + +```json +{ + "name": "Alice", + "email": "alice@example.com", + "password": "password123" +} +``` + +```bash +curl -s -X POST http://localhost:3000/auth/register \ + -H "Content-Type: application/json" \ + -d '{"name":"Alice","email":"alice@example.com","password":"password123"}' +``` + +Response example: + +```json +{"status":"success","user":{"id":"00000000-0000-0000-0000-000000000000"}} +``` + +## POST /auth/login +Authenticates a user and returns a JWT. + +Request body: + +```json +{ + "email": "alice@example.com", + "password": "password123" +} +``` + +```bash +curl -s -X POST http://localhost:3000/auth/login \ + -H "Content-Type: application/json" \ + -d '{"email":"alice@example.com","password":"password123"}' +``` + +Response example: + +```json +{"status":"success","token":""} +``` + +## GET /users/:id/keywords +Returns all keywords for a user. + +```bash +curl -s http://localhost:3000/users//keywords +``` + +Response example: + +```json +[ + { + "id": "00000000-0000-0000-0000-000000000000", + "user_id": "11111111-1111-1111-1111-111111111111", + "keyword": "machine learning", + "created_at": "2026-02-23T12:00:00Z" + } +] +``` + +## POST /users/:id/keywords +Adds a keyword for a user. + +Request body: + +```json +{ + "keyword": "machine learning" +} +``` + +```bash +curl -s -X POST http://localhost:3000/users//keywords \ + -H "Content-Type: application/json" \ + -d '{"keyword":"machine learning"}' +``` + +Response example: + +```json +{ + "id": "00000000-0000-0000-0000-000000000000", + "user_id": "11111111-1111-1111-1111-111111111111", + "keyword": "machine learning", + "created_at": "2026-02-23T12:00:00Z" +} +``` diff --git a/server/src/db.rs b/server/src/db.rs new file mode 100644 index 0000000..dcc9f12 --- /dev/null +++ b/server/src/db.rs @@ -0,0 +1,13 @@ +use sqlx::postgres::PgPoolOptions; +use sqlx::{Pool, Postgres}; +use std::env; + +pub async fn establish_connection() -> Pool { + let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + + PgPoolOptions::new() + .max_connections(5) + .connect(&database_url) + .await + .expect("Failed to connect to Postgres") +} diff --git a/server/src/handlers.rs b/server/src/handlers.rs new file mode 100644 index 0000000..e244d4d --- /dev/null +++ b/server/src/handlers.rs @@ -0,0 +1,161 @@ +use axum::{ + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde_json::json; +use argon2::{ + password_hash::{ + rand_core::OsRng, + PasswordHash, PasswordHasher, PasswordVerifier, SaltString + }, + Argon2 +}; +use jsonwebtoken::{encode, EncodingKey, Header}; +use chrono::{Utc, Duration}; +use uuid::Uuid; +use crate::models::*; +use crate::state::AppState; + +pub enum AppError { + InternalServerError, + UserAlreadyExists, + InvalidCredentials, + KeywordAlreadyExists, + InvalidKeyword, +} + +impl IntoResponse for AppError { + fn into_response(self) -> Response { + let (status, message) = match self { + AppError::InternalServerError => (StatusCode::INTERNAL_SERVER_ERROR, "Internal server error"), + AppError::UserAlreadyExists => (StatusCode::CONFLICT, "User with that email already exists"), + AppError::InvalidCredentials => (StatusCode::UNAUTHORIZED, "Invalid email or password"), + AppError::KeywordAlreadyExists => (StatusCode::CONFLICT, "Keyword already exists for this user"), + AppError::InvalidKeyword => (StatusCode::BAD_REQUEST, "Keyword must not be empty"), + }; + (status, Json(json!({"error": message}))).into_response() + } +} + +pub async fn get_article_count(State(state): State) -> Json { + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM articles") + .fetch_one(&state.pool) + .await + .unwrap_or(0); // Safer handling + + Json(ArticleCount { count }) +} + +pub async fn register_user_handler( + State(state): State, + Json(body): Json, +) -> Result { + let user_exists = sqlx::query("SELECT 1 FROM users WHERE email = $1") + .bind(&body.email.to_lowercase()) + .fetch_optional(&state.pool) + .await + .map_err(|_| AppError::InternalServerError)?; + + if user_exists.is_some() { + return Err(AppError::UserAlreadyExists); + } + + let salt = SaltString::generate(&mut OsRng); + let argon2 = Argon2::default(); + let password_hash = argon2.hash_password(body.password.as_bytes(), &salt) + .map_err(|_| AppError::InternalServerError)? + .to_string(); + + let user_id: uuid::Uuid = sqlx::query_scalar( + "INSERT INTO users (name, email, password_hash) VALUES ($1, $2, $3) RETURNING id", + ) + .bind(body.name) + .bind(body.email.to_lowercase()) + .bind(password_hash) + .fetch_one(&state.pool) + .await + .map_err(|_| AppError::InternalServerError)?; + + Ok((StatusCode::CREATED, Json(json!({"status": "success", "user": {"id": user_id}})))) +} + +pub async fn login_user_handler( + State(state): State, + Json(body): Json, +) -> Result { + let user = sqlx::query_as::<_, User>("SELECT * FROM users WHERE email = $1") + .bind(&body.email.to_lowercase()) + .fetch_optional(&state.pool) + .await + .map_err(|_| AppError::InternalServerError)?; + + let user = user.ok_or(AppError::InvalidCredentials)?; + + let parsed_hash = PasswordHash::new(&user.password_hash) + .map_err(|_| AppError::InternalServerError)?; + + Argon2::default().verify_password(body.password.as_bytes(), &parsed_hash) + .map_err(|_| AppError::InvalidCredentials)?; + + let secret = std::env::var("JWT_SECRET").unwrap_or_else(|_| "secret".to_string()); + + let now = Utc::now(); + let iat = now.timestamp() as usize; + let exp = (now + Duration::minutes(60)).timestamp() as usize; + let claims = json!({ + "sub": user.id, + "role": user.role, + "exp": exp, + "iat": iat, + }); + + let token = encode(&Header::default(), &claims, &EncodingKey::from_secret(secret.as_bytes())) + .map_err(|_| AppError::InternalServerError)?; + + Ok(Json(TokenResponse { + status: "success".to_string(), + token, + })) +} + +pub async fn list_user_keywords( + Path(user_id): Path, + State(state): State, +) -> Result>, AppError> { + let keywords = sqlx::query_as::<_, UserKeyword>( + "SELECT id, user_id, keyword, created_at FROM user_keywords WHERE user_id = $1 ORDER BY created_at DESC", + ) + .bind(user_id) + .fetch_all(&state.pool) + .await + .map_err(|_| AppError::InternalServerError)?; + + Ok(Json(keywords)) +} + +pub async fn add_user_keyword( + Path(user_id): Path, + State(state): State, + Json(body): Json, +) -> Result { + let keyword = body.keyword.trim().to_lowercase(); + if keyword.is_empty() { + return Err(AppError::InvalidKeyword); + } + + let inserted = sqlx::query_as::<_, UserKeyword>( + "INSERT INTO user_keywords (user_id, keyword) VALUES ($1, $2) \n ON CONFLICT (user_id, keyword) DO NOTHING \n RETURNING id, user_id, keyword, created_at", + ) + .bind(user_id) + .bind(keyword) + .fetch_optional(&state.pool) + .await + .map_err(|_| AppError::InternalServerError)?; + + match inserted { + Some(keyword) => Ok((StatusCode::CREATED, Json(keyword))), + None => Err(AppError::KeywordAlreadyExists), + } +} diff --git a/server/src/main.rs b/server/src/main.rs index ce454b3..52d2675 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,15 +1,32 @@ -use axum::{routing::get, Router}; +mod db; +mod handlers; +mod models; +mod routes; +mod state; + +use dotenv::dotenv; +use state::AppState; #[tokio::main] async fn main() { - let app = Router::new() - .route("/", get(|| async { "Route principale" })) - .route("/ping", get(|| async { "pong" })); + dotenv().ok(); + + let pool = db::establish_connection().await; + + // Run migrations + sqlx::migrate!("./migrations") + .run(&pool) + .await + .expect("Failed to run migrations"); + + let state = AppState { pool }; + + let app = routes::create_router(state); let listener = tokio::net::TcpListener::bind("127.0.0.1:3000") .await .unwrap(); - + println!("Server sur http://localhost:3000"); axum::serve(listener, app) diff --git a/server/src/models.rs b/server/src/models.rs new file mode 100644 index 0000000..6f7763d --- /dev/null +++ b/server/src/models.rs @@ -0,0 +1,59 @@ +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use uuid::Uuid; +use chrono::{DateTime, Utc}; + +#[derive(Serialize)] +pub struct ArticleCount { + pub count: i64, +} + +#[derive(Debug, FromRow, Serialize, Deserialize)] +pub struct User { + pub id: Uuid, + pub name: String, + pub email: String, + #[serde(skip_serializing)] + pub password_hash: String, + pub role: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug, Deserialize)] +pub struct CreateUserSchema { + pub name: String, + pub email: String, + pub password: String, +} + +#[derive(Debug, Deserialize)] +pub struct LoginUserSchema { + pub email: String, + pub password: String, +} + +#[derive(Debug, Serialize)] +pub struct TokenResponse { + pub token: String, + pub status: String, +} + +#[derive(Debug, Serialize)] +pub struct ErrorResponse { + pub status: String, + pub message: String, +} + +#[derive(Debug, FromRow, Serialize)] +pub struct UserKeyword { + pub id: Uuid, + pub user_id: Uuid, + pub keyword: String, + pub created_at: DateTime, +} + +#[derive(Debug, Deserialize)] +pub struct CreateKeywordSchema { + pub keyword: String, +} diff --git a/server/src/routes.rs b/server/src/routes.rs new file mode 100644 index 0000000..76c0507 --- /dev/null +++ b/server/src/routes.rs @@ -0,0 +1,20 @@ +use axum::{routing::{get, post}, Router}; +use crate::handlers::{ + add_user_keyword, + get_article_count, + list_user_keywords, + login_user_handler, + register_user_handler, +}; +use crate::state::AppState; + +pub fn create_router(state: AppState) -> Router { + Router::new() + .route("/", get(|| async { "API Root" })) + .route("/articles/count", get(get_article_count)) + .route("/auth/register", post(register_user_handler)) + .route("/auth/login", post(login_user_handler)) + .route("/users/:id/keywords", get(list_user_keywords)) + .route("/users/:id/keywords", post(add_user_keyword)) + .with_state(state) +} diff --git a/server/src/state.rs b/server/src/state.rs new file mode 100644 index 0000000..8ced403 --- /dev/null +++ b/server/src/state.rs @@ -0,0 +1,6 @@ +use sqlx::{Pool, Postgres}; + +#[derive(Clone)] +pub struct AppState { + pub pool: Pool, +}