Refract/scholar.py at main · LDY5388/Refract · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
"""
Refract - Semantic Scholar API Client v2
Works with ParsedReference dataclass from parser.py
"""

import time
import requests
from dataclasses import dataclass

BASE_URL = "https://api.semanticscholar.org/graph/v1"

PAPER_FIELDS = ",".join([
    "title", "abstract", "authors", "year",
    "citationCount", "referenceCount",
    "fieldsOfStudy", "url", "externalIds", "tldr",
])


@dataclass
class ScholarData:
    title: str | None = None
    abstract: str | None = None
    authors: list[str] | None = None
    year: int | None = None
    citation_count: int | None = None
    fields: list[str] | None = None
    url: str | None = None
    tldr: str | None = None
    doi: str | None = None


def search_paper(query: str, limit: int = 3) -> list[dict]:
    url = f"{BASE_URL}/paper/search"
    params = {"query": query, "limit": limit, "fields": PAPER_FIELDS}
    try:
        resp = requests.get(url, params=params, timeout=10)
        resp.raise_for_status()
        return resp.json().get("data", [])
    except requests.RequestException as e:
        return [{"error": str(e)}]


def enrich_references(parsed_refs: dict, delay: float = 0.5) -> dict:
    """
    For each ParsedReference, search Semantic Scholar.
    Returns dict with same keys, values are dicts with:
      - all original ParsedReference fields
      - scholar_data: ScholarData or None
      - match_status: "found" or "not_found"
    """
    enriched = {}

    for key, ref in parsed_refs.items():
        query = ref.search_query or ref.extracted_title or ref.raw_text[:120]
        query = query.strip()[:200]

        results = search_paper(query, limit=1)

        if results and "error" not in results[0]:
            best = results[0]
            sd = ScholarData(
                title=best.get("title"),
                abstract=best.get("abstract"),
                authors=[a.get("name", "") for a in best.get("authors", [])],
                year=best.get("year"),
                citation_count=best.get("citationCount"),
                fields=best.get("fieldsOfStudy"),
                url=best.get("url"),
                tldr=best.get("tldr", {}).get("text") if best.get("tldr") else None,
                doi=best.get("externalIds", {}).get("DOI"),
            )
            enriched[key] = {
                "key": key,
                "raw": ref.raw_text,
                "extracted_title": ref.extracted_title,
                "authors": ref.authors,
                "year": ref.year,
                "citation_contexts": ref.citation_contexts,
                "scholar_data": {
                    "title": sd.title,
                    "abstract": sd.abstract,
                    "authors": sd.authors,
                    "year": sd.year,
                    "citation_count": sd.citation_count,
                    "fields": sd.fields,
                    "url": sd.url,
                    "tldr": sd.tldr,
                    "doi": sd.doi,
                },
                "match_status": "found",
            }
        else:
            enriched[key] = {
                "key": key,
                "raw": ref.raw_text,
                "extracted_title": ref.extracted_title,
                "authors": ref.authors,
                "year": ref.year,
                "citation_contexts": ref.citation_contexts,
                "scholar_data": None,
                "match_status": "not_found",
            }

        time.sleep(delay)

    return enriched


def format_authors(authors: list[str], max_show: int = 3) -> str:
    if not authors:
        return "Unknown"
    if len(authors) <= max_show:
        return ", ".join(authors)
    return ", ".join(authors[:max_show]) + f" et al. (+{len(authors) - max_show})"