fix export to CSV for time series, collocation, and aggregation

Author: clovis

python/philologic/runtime/__init__.py

@@ -7,21 +7,27 @@
 from philologic.runtime.pages import page_interval
 from philologic.runtime.Query import parse_query
 from philologic.runtime.reports import (
+    aggregation_by_field,
+    aggregation_to_csv,
     bibliography_results,
+    bibliography_to_csv,
     collocation_results,
+    collocation_to_csv,
     concordance_results,
+    concordance_to_csv,
     frequency_results,
     generate_text_object,
     generate_time_series,
     generate_toc_object,
     generate_word_frequency,
     get_start_end_date,
-    kwic_hit_object,
-    kwic_results,
     group_by_metadata,
     group_by_range,
+    kwic_hit_object,
+    kwic_results,
+    kwic_to_csv,
     landing_page_bibliography,
-    aggregation_by_field,
+    time_series_to_csv,
 )
 from philologic.runtime.web_config import WebConfig
 from philologic.runtime.WSGIHandler import WSGIHandler

python/philologic/runtime/reports/__init__.py

@@ -1,13 +1,13 @@
 """Report exports"""
 
-from philologic.runtime.reports.concordance import concordance_results
-from philologic.runtime.reports.bibliography import bibliography_results
-from philologic.runtime.reports.time_series import generate_time_series, get_start_end_date
+from philologic.runtime.reports.concordance import concordance_results, concordance_to_csv
+from philologic.runtime.reports.bibliography import bibliography_results, bibliography_to_csv
+from philologic.runtime.reports.time_series import generate_time_series, get_start_end_date, time_series_to_csv
 from philologic.runtime.reports.navigation import generate_text_object
 from philologic.runtime.reports.table_of_contents import generate_toc_object
-from philologic.runtime.reports.kwic import kwic_results, kwic_hit_object
+from philologic.runtime.reports.kwic import kwic_results, kwic_hit_object, kwic_to_csv
 from philologic.runtime.reports.generate_word_frequency import generate_word_frequency
 from philologic.runtime.reports.frequency import frequency_results
-from philologic.runtime.reports.collocation import collocation_results
+from philologic.runtime.reports.collocation import collocation_results, collocation_to_csv
 from philologic.runtime.reports.landing_page import landing_page_bibliography, group_by_range, group_by_metadata
-from philologic.runtime.reports.aggregation import aggregation_by_field
+from philologic.runtime.reports.aggregation import aggregation_by_field, aggregation_to_csv

python/philologic/runtime/reports/aggregation.py

@@ -1,6 +1,9 @@
 # /usr/bin/env python3
 """Report designed to group results by metadata with additional breakdown optional"""
 
+import csv
+import io
+
 import numpy as np
 
 from philologic.runtime.DB import DB
@@ -137,6 +140,45 @@ def aggregation_by_field(request, config):
     }
 
 
+def aggregation_to_csv(results, break_up_field_name=""):
+    """Convert aggregation results to CSV string.
+
+    Each breakdown entry gets its own row. Rows from the same group
+    are contiguous, with the group-level metadata repeated.
+    """
+    if not results:
+        return ""
+    output = io.StringIO()
+    first = results[0]
+    group_keys = sorted(k for k in first["metadata_fields"].keys() if k not in ("field_name", "philo_id"))
+    has_breakdown = break_up_field_name and any(r["break_up_field"] for r in results)
+    if has_breakdown:
+        # Collect all metadata keys from breakdown entries
+        breakdown_keys = set()
+        for result in results:
+            for sub in result["break_up_field"]:
+                breakdown_keys.update(k for k in sub["metadata_fields"].keys() if k not in ("field_name", "philo_id"))
+        breakdown_keys = sorted(breakdown_keys - set(group_keys))
+        fieldnames = group_keys + ["group_count"] + breakdown_keys + ["count"]
+    else:
+        fieldnames = group_keys + ["count"]
+    writer = csv.DictWriter(output, fieldnames=fieldnames)
+    writer.writeheader()
+    for result in results:
+        group_fields = {k: result["metadata_fields"].get(k, "") for k in group_keys}
+        if has_breakdown and result["break_up_field"]:
+            for sub in result["break_up_field"]:
+                row = {**group_fields, "group_count": result["count"]}
+                for k in breakdown_keys:
+                    row[k] = sub["metadata_fields"].get(k, "")
+                row["count"] = sub["count"]
+                writer.writerow(row)
+        else:
+            row = {**group_fields, "count": result["count"]}
+            writer.writerow(row)
+    return output.getvalue()
+
+
 def __expand_hits_counted(hits, metadata_type):
     """Stream sorted hitlist with numpy, return per-ID hit counts.
 

python/philologic/runtime/reports/bibliography.py

@@ -1,6 +1,8 @@
 #!/var/lib/philologic5/philologic_env/bin/python3
 """Bibliography results"""
 
+import csv
+import io
 
 from philologic.runtime.citations import citation_links, citations
 from philologic.runtime.DB import DB
@@ -77,3 +79,19 @@ def bibliography_results(request, config):
     bibliography_object["query_done"] = hits.done
     bibliography_object["result_type"] = result_type
     return bibliography_object, hits
+
+
+def bibliography_to_csv(results):
+    """Convert bibliography results to CSV string."""
+    if not results:
+        return ""
+    output = io.StringIO()
+    metadata_keys = sorted(results[0]["metadata_fields"].keys())
+    fieldnames = ["philo_id"] + metadata_keys
+    writer = csv.DictWriter(output, fieldnames=fieldnames)
+    writer.writeheader()
+    for result in results:
+        row = {"philo_id": " ".join(str(x) for x in result["philo_id"])}
+        row.update(result["metadata_fields"])
+        writer.writerow(row)
+    return output.getvalue()

python/philologic/runtime/reports/collocation.py

@@ -580,6 +580,21 @@ def collocation_results(request, config):
     return collocation_object
 
 
+def collocation_to_csv(collocates):
+    """Convert collocation results (list of (word, count) tuples) to CSV string."""
+    import csv
+    import io
+
+    if not collocates:
+        return ""
+    output = io.StringIO()
+    writer = csv.DictWriter(output, fieldnames=["collocate", "count"])
+    writer.writeheader()
+    for word, count in collocates:
+        writer.writerow({"collocate": word, "count": count})
+    return output.getvalue()
+
+
 def atomic_pickle_dump(data, file_path):
     """Write pickle atomically to prevent truncated reads from concurrent requests."""
     dir_path = os.path.dirname(file_path)

python/philologic/runtime/reports/concordance.py

@@ -1,6 +1,9 @@
 #!/var/lib/philologic5/philologic_env/bin/python3
 """Concordance report"""
 
+import csv
+import io
+
 import regex as re
 from philologic.runtime.citations import citation_links, citations
 from philologic.runtime.DB import DB
@@ -60,3 +63,23 @@ def concordance_results(request, config):
     concordance_object["results_length"] = len(hits)
     concordance_object["query_done"] = hits.done
     return concordance_object
+
+
+def concordance_to_csv(results, filter_html=False):
+    """Convert concordance results to CSV string."""
+    if not results:
+        return ""
+    tags_re = re.compile(r"<[^>]+>")
+    output = io.StringIO()
+    metadata_keys = sorted(results[0]["metadata_fields"].keys())
+    fieldnames = ["philo_id", "context"] + metadata_keys
+    writer = csv.DictWriter(output, fieldnames=fieldnames)
+    writer.writeheader()
+    for result in results:
+        context = result["context"]
+        if filter_html:
+            context = tags_re.sub("", context).strip()
+        row = {"philo_id": " ".join(str(x) for x in result["philo_id"]), "context": context}
+        row.update(result["metadata_fields"])
+        writer.writerow(row)
+    return output.getvalue()

python/philologic/runtime/reports/kwic.py

@@ -1,6 +1,8 @@
 #!/var/lib/philologic5/philologic_env/bin/python3
 """KWIC results"""
 
+import csv
+import io
 
 import regex as re
 from philologic.runtime.citations import citation_links, citations
@@ -92,3 +94,23 @@ def kwic_hit_object(hit, config, db):
     }
 
     return kwic_result
+
+
+def kwic_to_csv(results, filter_html=False):
+    """Convert KWIC results to CSV string."""
+    if not results:
+        return ""
+    tags_re = re.compile(r"<[^>]+>")
+    output = io.StringIO()
+    metadata_keys = sorted(results[0]["metadata_fields"].keys())
+    fieldnames = ["philo_id", "context"] + metadata_keys
+    writer = csv.DictWriter(output, fieldnames=fieldnames)
+    writer.writeheader()
+    for result in results:
+        context = result["context"]
+        if filter_html:
+            context = tags_re.sub("", context).strip()
+        row = {"philo_id": " ".join(str(x) for x in result["philo_id"]), "context": context}
+        row.update(result["metadata_fields"])
+        writer.writerow(row)
+    return output.getvalue()

python/philologic/runtime/reports/time_series.py

@@ -2,7 +2,6 @@
 """Time series"""
 
 import os
-import time
 
 import numba
 import numpy as np
@@ -96,7 +95,6 @@ def _bucket_hits_by_year(doc_ids, year_array, start_date, interval, n_ranges):
 
 
 def generate_time_series(request, config):
-    t0 = time.time()
     db = DB(config.db_path + "/data/")
     year_field = validate_column(config.time_series_year_field, db)
     time_series_object = {"query": dict([i for i in request]), "query_done": False}
@@ -109,25 +107,24 @@ def generate_time_series(request, config):
         time_series_object["results"] = {"absolute_count": {}, "date_count": {}}
         return time_series_object
 
-    interval = int(request.year_interval)
+    try:
+        interval = int(request.year_interval)
+    except (ValueError, TypeError):
+        interval = int(config.time_series_interval)
 
     # Get cached doc→year mapping (SQL only on first request per worker)
-    t1 = time.time()
     year_array, year_word_counts, year_doc_counts, min_date, max_date = _get_doc_year_data(db, year_field)
-    print(f"[time_series] doc year data: {time.time()-t1:.3f}s", flush=True)
 
     # Resolve start/end dates
     start_date = int(request.start_date) if request.start_date else min_date
     end_date = int(request.end_date) if request.end_date else max_date
 
     # Fire the word query now that we have start/end dates
-    t1 = time.time()
     hits = None
     if request.q:
         metadata = dict(request.metadata)
         metadata[year_field] = "%d-%d" % (start_date, end_date)
         hits = db.query(request["q"], request["method"], request["arg"], raw_results=True, **metadata)
-    print(f"[time_series] db.query dispatch: {time.time()-t1:.3f}s", flush=True)
 
     # Generate date ranges for output
     date_ranges = []
@@ -150,27 +147,18 @@ def generate_time_series(request, config):
 
     # Absolute hit counts: wait for search, then vectorized bucketing
     if hits is not None:
-        t1 = time.time()
         hits.finish()
-        t_finish = time.time() - t1
         total_hits = len(hits)
-        print(f"[time_series] hits.finish() wait ({total_hits} hits): {t_finish:.3f}s", flush=True)
 
         if total_hits > 0:
-            t1 = time.time()
             hit_length = hits.length
             mm = np.memmap(hits.filename, dtype="u4", mode="r").reshape(-1, hit_length)
             doc_ids = np.ascontiguousarray(mm[:, 0])
             del mm  # release mmap immediately
-            t_read = time.time() - t1
 
-            # Single-pass JIT on contiguous doc_id column
-            t1 = time.time()
             bin_counts, total_hits = _bucket_hits_by_year(
                 doc_ids, year_array, start_date, interval, n_ranges
             )
-            t_jit = time.time() - t1
-            print(f"[time_series] mmap+extract doc_ids: {t_read:.3f}s, JIT bucket: {t_jit:.3f}s ({total_hits} hits in {n_ranges} bins)", flush=True)
         else:
             bin_counts = np.zeros(n_ranges, dtype=np.int64)
     else:
@@ -182,7 +170,6 @@ def generate_time_series(request, config):
             total_hits += int(bin_counts[i])
 
     # Build absolute_count output matching expected format
-    t1 = time.time()
     absolute_count = {}
     for i, (range_start, date_range) in enumerate(date_ranges):
         params = {"report": "concordance", "start": "0", "end": "0"}
@@ -193,7 +180,6 @@ def generate_time_series(request, config):
             "count": int(bin_counts[i]),
             "url": url,
         }
-    print(f"[time_series] build output ({n_ranges} ranges): {time.time()-t1:.3f}s", flush=True)
 
     time_series_object["results_length"] = int(total_hits)
     time_series_object["more_results"] = False
@@ -202,10 +188,31 @@ def generate_time_series(request, config):
         "date_count": {str(date): count for date, count in date_counts.items()},
     }
 
-    print(f"[time_series] TOTAL: {time.time()-t0:.3f}s", flush=True)
     return time_series_object
 
 
+def time_series_to_csv(results):
+    """Convert time series results to CSV string."""
+    import csv
+    import io
+
+    absolute_count = results.get("absolute_count", {})
+    date_count = results.get("date_count", {})
+    if not absolute_count:
+        return ""
+    output = io.StringIO()
+    writer = csv.DictWriter(output, fieldnames=["period", "count", "total_words"])
+    writer.writeheader()
+    for period_start in sorted(absolute_count.keys(), key=int):
+        entry = absolute_count[period_start]
+        writer.writerow({
+            "period": entry["label"],
+            "count": entry["count"],
+            "total_words": date_count.get(period_start, ""),
+        })
+    return output.getvalue()
+
+
 def get_start_end_date(db, config, start_date=None, end_date=None):
     """Get start and end date of dataset"""
     year_field = validate_column(config.time_series_year_field, db)