update writers and docs

Author: Gerit Wagner

colrev/loader/load_utils.py

@@ -6,10 +6,10 @@
     import colrev.loader.load_utils
 
     # Files
-    records = colrev.loader.load_utils.load(filename=filename, logger=logger)
+    records = colrev.loader.load_utils.load(filename=filename)
 
     # Strings
-    records = colrev.loader.load_utils.loads(load_str=load_str, logger=logger)
+    records = colrev.loader.load_utils.loads(load_str=load_str)
 
     returns: records (dict)
 
@@ -173,6 +173,9 @@ def load(  # type: ignore
 ) -> dict:
     """Load a file and return records as a dictionary"""
 
+    if isinstance(filename, str):
+        filename = Path(filename)
+
     if not filename.is_file():
         if empty_if_file_not_exists:
             return {}

colrev/writer/csv.py

@@ -23,23 +23,43 @@
 ]
 
 
-def to_dataframe(*, records_dict: dict) -> pd.DataFrame:
+def to_dataframe(
+    *,
+    records_dict: dict,
+    sort_fields_first: bool = True,
+    drop_empty_fields: bool = True,
+) -> pd.DataFrame:
     """Convert a records dict to a pandas DataFrame"""
+    all_keys = {k for v in records_dict.values() for k in v.keys()}
+    additional_fields = sorted(all_keys - set(FIELDS))
+    fields = FIELDS + additional_fields if sort_fields_first else sorted(all_keys)
+
     data = []
-    additional_fields = [x for x in list(records_dict) if x not in FIELDS]
     for record_id in sorted(records_dict.keys()):
         record_dict = records_dict[record_id]
-        row = {}
-        for field in FIELDS + additional_fields:
-            if field in record_dict:
-                row[field] = record_dict[field]
-            else:
-                row[field] = ""
+        row = {field: record_dict.get(field, "") for field in fields}
         data.append(row)
-    return pd.DataFrame(data)
+
+    df = pd.DataFrame(data)
+
+    if drop_empty_fields:
+        df = df.dropna(axis=1, how="all")
+        df = df.loc[:, (df != "").any(axis=0)]
+
+    return df
 
 
-def write_file(*, records_dict: dict, filename: str) -> None:
-    """Write a csv file from a records dict"""
-    data_frame = to_dataframe(records_dict=records_dict)
-    data_frame.to_csv(filename, index=False)
+def write_file(
+    *,
+    records_dict: dict,
+    filename: str,
+    sort_fields_first: bool = True,
+    drop_empty_fields: bool = True,
+) -> None:
+    """Write a CSV file from a records dict"""
+    df = to_dataframe(
+        records_dict=records_dict,
+        sort_fields_first=sort_fields_first,
+        drop_empty_fields=drop_empty_fields,
+    )
+    df.to_csv(filename, index=False, encoding="utf-8")

colrev/writer/excel.py

@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-"""Function to write excel files"""
+"""Function to write Excel files with flexible field handling"""
 from __future__ import annotations
 
 import pandas as pd
@@ -23,25 +23,45 @@
 ]
 
 
-def to_dataframe(*, records_dict: dict) -> pd.DataFrame:
+def to_dataframe(
+    *,
+    records_dict: dict,
+    sort_fields_first: bool = True,
+    drop_empty_fields: bool = True,
+) -> pd.DataFrame:
     """Convert a records dict to a pandas DataFrame"""
+    all_keys = {k for v in records_dict.values() for k in v.keys()}
+    additional_fields = sorted(all_keys - set(FIELDS))
+    fields = FIELDS + additional_fields if sort_fields_first else sorted(all_keys)
+
     data = []
     for record_id in sorted(records_dict.keys()):
         record_dict = records_dict[record_id]
-        row = {}
-        for field in FIELDS:
-            if field in record_dict:
-                row[field] = record_dict[field]
-            else:
-                row[field] = ""
+        row = {field: record_dict.get(field, "") for field in fields}
         data.append(row)
-    return pd.DataFrame(data)
+
+    df = pd.DataFrame(data)
+
+    if drop_empty_fields:
+        df = df.dropna(axis=1, how="all")
+        df = df.loc[:, (df != "").any(axis=0)]
+
+    return df
 
 
-def write_file(*, records_dict: dict, filename: str) -> None:
-    """Write an excel file from a records dict"""
-    data_frame = to_dataframe(records_dict=records_dict)
-    # pylint: disable=abstract-class-instantiated
+def write_file(
+    *,
+    records_dict: dict,
+    filename: str,
+    sort_fields_first: bool = True,
+    drop_empty_fields: bool = True,
+) -> None:
+    """Write an Excel file from a records dict"""
+    data_frame = to_dataframe(
+        records_dict=records_dict,
+        sort_fields_first=sort_fields_first,
+        drop_empty_fields=drop_empty_fields,
+    )
     writer = pd.ExcelWriter(filename, engine="xlsxwriter")
     data_frame.to_excel(writer, index=False)
 

colrev/writer/markdown.py

@@ -0,0 +1,91 @@
+#! /usr/bin/env python
+"""Function to write Markdown (table) files"""
+from __future__ import annotations
+
+import pandas as pd
+
+from colrev.constants import Fields
+
+FIELDS = [
+    Fields.ID,
+    Fields.ENTRYTYPE,
+    Fields.TITLE,
+    Fields.AUTHOR,
+    Fields.YEAR,
+    Fields.JOURNAL,
+    Fields.BOOKTITLE,
+    Fields.VOLUME,
+    Fields.NUMBER,
+    Fields.PAGES,
+    Fields.DOI,
+    Fields.URL,
+    Fields.FILE,
+]
+
+
+def to_dataframe(
+    *,
+    records_dict: dict,
+    sort_fields_first: bool = True,
+    drop_empty_fields: bool = True,
+) -> pd.DataFrame:
+    """Convert a records dict to a pandas DataFrame"""
+    all_keys = {k for v in records_dict.values() for k in v.keys()}
+    additional_fields = sorted(all_keys - set(FIELDS))
+    fields = FIELDS + additional_fields if sort_fields_first else sorted(all_keys)
+
+    data = []
+    for record_id in sorted(records_dict.keys()):
+        record_dict = records_dict[record_id]
+        row = {field: record_dict.get(field, "") for field in fields}
+        data.append(row)
+
+    df = pd.DataFrame(data)
+
+    if drop_empty_fields:
+        df = df.dropna(axis=1, how="all")
+        df = df.loc[:, (df != "").any(axis=0)]
+
+    return df
+
+
+def to_string(
+    *,
+    records_dict: dict,
+    sort_fields_first: bool = True,
+    drop_empty_fields: bool = True,
+) -> str:
+    """Convert a records dict to a markdown string with a table"""
+    data_frame = to_dataframe(
+        records_dict=records_dict,
+        sort_fields_first=sort_fields_first,
+        drop_empty_fields=drop_empty_fields,
+    )
+
+    headers = list(data_frame.columns)
+    md_lines = [
+        "| " + " | ".join(headers) + " |",
+        "|" + "|".join(["---"] * len(headers)) + "|",
+    ]
+    for _, row in data_frame.iterrows():
+        row_values = [str(row[h]).replace("\n", " ") for h in headers]
+        md_lines.append("| " + " | ".join(row_values) + " |")
+
+    return "\n".join(md_lines)
+
+
+def write_file(
+    *,
+    records_dict: dict,
+    filename: str,
+    sort_fields_first: bool = True,
+    drop_empty_fields: bool = True,
+) -> None:
+    """Write a markdown file with a table from a records dict"""
+    md_string = to_string(
+        records_dict=records_dict,
+        sort_fields_first=sort_fields_first,
+        drop_empty_fields=drop_empty_fields,
+    )
+    with open(filename, "w", encoding="utf-8") as f:
+        f.write(md_string)

colrev/writer/write_utils.py

@@ -3,7 +3,14 @@
 
 Usage::
 
-    TODO
+    import colrev.loader.load_utils
+    import colrev.writer.write_utils
+
+    # Load
+    records = colrev.loader.load_utils.load(filename=filename)
+
+    # Write
+    colrev.loader.write_utils.write_file(records, filename=filename)
 
 """
 from __future__ import annotations
@@ -13,11 +20,20 @@
 import colrev.writer.bib
 import colrev.writer.csv
 import colrev.writer.excel
+import colrev.writer.markdown
 import colrev.writer.ris
 
 
-def write_file(records_dict: dict, filename: Path, **kw) -> dict:  # type: ignore
-    """Write a file (BiBTex, RIS, or other) from a dictionary of records."""
+def write_file(records_dict: dict, *, filename: Path, **kw) -> dict:  # type: ignore
+    """Write a file (BiBTex, RIS, or other) from a dictionary of records.
+
+    Note:
+        For tabular formats (csv, xlsx, md), the following options are supported:
+            - sort_fields_first: list of fields to appear first in the output
+            - drop_empty_fields: if True, empty fields will be omitted
+    """
+    if isinstance(filename, str):
+        filename = Path(filename)
     if filename.suffix == ".bib":
         writer = colrev.writer.bib.write_file  # type: ignore
     elif filename.suffix == ".ris":
@@ -26,6 +42,8 @@ def write_file(records_dict: dict, filename: Path, **kw) -> dict:  # type: ignor
         writer = colrev.writer.csv.write_file  # type: ignore
     elif filename.suffix == ".xlsx":
         writer = colrev.writer.excel.write_file  # type: ignore
+    elif filename.suffix == ".md":
+        writer = colrev.writer.markdown.write_file  # type: ignore
     else:
         raise NotImplementedError
 

docs/source/dev_docs/_autosummary/colrev.writer.markdown.rst

@@ -0,0 +1,20 @@
+colrev.writer.markdown
+======================
+
+.. automodule:: colrev.writer.markdown
+
+
+
+
+
+
+
+   .. rubric:: Functions
+
+   .. autosummary::
+      :toctree:
+      :nosignatures:
+
+      to_dataframe
+      to_string
+      write_file

docs/source/dev_docs/_autosummary/colrev.writer.markdown.to_dataframe.rst

@@ -0,0 +1,6 @@
+colrev.writer.markdown.to\_dataframe
+====================================
+
+.. currentmodule:: colrev.writer.markdown
+
+.. autofunction:: to_dataframe

docs/source/dev_docs/_autosummary/colrev.writer.markdown.to_string.rst

@@ -0,0 +1,6 @@
+colrev.writer.markdown.to\_string
+=================================
+
+.. currentmodule:: colrev.writer.markdown
+
+.. autofunction:: to_string

docs/source/dev_docs/_autosummary/colrev.writer.markdown.write_file.rst

@@ -0,0 +1,6 @@
+colrev.writer.markdown.write\_file
+==================================
+
+.. currentmodule:: colrev.writer.markdown
+
+.. autofunction:: write_file

docs/source/dev_docs/_autosummary/colrev.writer.rst

@@ -29,5 +29,6 @@ colrev.writer
    colrev.writer.bib
    colrev.writer.csv
    colrev.writer.excel
+   colrev.writer.markdown
    colrev.writer.ris
    colrev.writer.write_utils