Better describe what up here with Import and Export

Author: rocky

mathics/builtin/fileformats/__init__.py

@@ -1,12 +1,40 @@
-"""
-Import/Export File Formats
+r"""Import/Export File Formats, Importers and Exporters
+
+The data of files on a filesystem or retrieved from the Internet often are structured \
+according to a specific structures and rules. For example, consider different kinds of \
+structuring used in a JSON file, versus an HTML files, or a compressed GZIP file.
+
+In some cases, such as archive files, e.g., ZIP, TAR, and JAR, the file contains component parts, \
+which in WMA terminology are called "members" which is part of the broader metadata items \
+called "elements".
 
-There various file formats can be used by 'Import' and 'Export' and related functions, \
-e.g. 'ImportString'.
+A MIME type is typically associated with each kind of format. \Mathics3, following WMA, \
+uses a shortend name for this MIME type. For example \Mathics3 uses "HTML" as a shorthand \
+for the MIME type "text/html".
+
+Below is a list of file supported file types that we have builtin importers or exporters written \
+in Python. Other importers, however, are written in \Mathics3.
+
+Variable <url>
+:\$ExportFormats:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/\$exportformats</url> \
+contains a list of file formats that are supported by <url>
+:Export:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/export</url>, \
+while <url>
+:\$ImportFormats:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/\$importformats</url> \
+does the corresponding thing for <url>
+:Import:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/import</url>.
 
 Many Import/Export functions are registered in SystemFiles/Formats/*.wl which is \
 autoloaded on startup.
 
 The Built-in Functions are defined in a separate context.
 For example, HTML` or Compress`.  This is done to not pollute the System` namespace.
 """
+
+# This tells documentation how to sort this module
+# Here we are also hiding "file_io" since this can erroneously appear at the top level.
+sort_order = "mathics.builtin.importing-export-file-formats"

mathics/builtin/import_export/importexport.py

@@ -1,8 +1,33 @@
 # -*- coding: utf-8 -*-
 
-"""
+r"""
 Import and Export Functions and Variables
 
+Many kinds data formats can be read into or written from \Mathics3.
+
+In contrast to reading or writing a file, <i>importing</i> and <i>exporting</i> imply some sort of \
+data restructuring into \Mathics3 and structuring into a filesystem that is not \
+just a stream of bytes, but instead also contains additional metadata and requires data reorganization \
+when stored in a filesystem.
+
+See <url>
+:Import/Export File Formats:
+/doc/reference-of-built-in-symbols/fileformats/</url> for documentation \
+on the specific kinds of File Formats \Mathics3 supports.
+
+
+Variable <url>
+:\$ExportFormats:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/\$exportformats</url> \
+contains a list of file formats that are supported by <url>
+:Export:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/export</url>, \
+while <url>
+:\$ImportFormats:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/\$importformats</url> \
+does the corresponding thing for <url>
+:Import:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/import</url>.
 """
 
 import base64
@@ -434,7 +459,13 @@ class Import(Builtin):
         "$OptionSyntax": "System`Ignore",
     }
 
-    summary_text = "import elements from a file"
+    rules = {
+        "Import[filename_]": "Import[filename, {}]",
+    }
+
+    summary_text = (
+        r"read and convert to \Mathics3 some or all elements of structured file"
+    )
 
     def eval_elements_query(self, source, evaluation, options={}):
         """Import[source_String, "Elements", OptionsPattern[]]"""
@@ -537,7 +568,9 @@ class ImportString(Builtin):
         "$OptionSyntax": "System`Ignore",
     }
 
-    summary_text = "import data or elements of data from a string"
+    summary_text = (
+        r"read and convert to \Mathics3 some or all elements of structured string"
+    )
 
     def eval_data_only(self, data, evaluation, options={}):
         "ImportString[data_, OptionsPattern[]]"
@@ -618,7 +651,9 @@ class Export(Builtin):
         "$OptionSyntax": "System`Ignore",
     }
 
-    summary_text = "export elements to a file"
+    summary_text = (
+        r"write and convert to \Mathics3 some or all elements of structured file"
+    )
 
     def eval(self, dest, expr, evaluation, options={}):
         "Export[dest_, expr_, OptionsPattern[Export]]"
@@ -767,7 +802,9 @@ class ExportString(Builtin):
     rules = {
         "ExportString[expr_, elems_?NotListQ]": ("ExportString[expr, {elems}]"),
     }
-    summary_text = "export elements to a string"
+    summary_text = (
+        r"write and convert to \Mathics3 some or all elements of structured string"
+    )
 
     def eval_element(self, expr, element: String, evaluation: Evaluation, **options):
         "ExportString[expr_, element_String, OptionsPattern[ExportString]]"

mathics/eval/fileformats/__init__.py

@@ -1,3 +1,8 @@
+"""
+Evaluation routines for handling data in some sort of archive format,
+e.g. ZIP, TAR, etc.
+"""
+
 import zipfile
 from typing import Optional
 
@@ -22,7 +27,7 @@ def eval_ImportZIP(
     """If `members` is empty, this function takes a ZIP file path and returns a
     list of file names/paths contained inside.
 
-    "If `members` is given, then extract those members from the ZIP file.
+    "If `members` is given, then extract those members (or files) from the ZIP file.
     """
 
     zip_path, is_temporary_file = resolve_file(zip_name, "r", evaluation)
@@ -36,6 +41,10 @@ def eval_ImportZIP(
             if members is None:
                 filenames = archive.namelist()
                 mathics_filenames = to_mathics_list(*filenames)
+
+                # Wrap metadata or "elements" of of the zip file into
+                # list of Rule. The caller can then use
+                # rules to pick out specific elements desired.
                 exprs = [
                     Expression(
                         SymbolRule,

mathics/eval/fileformats/compression.py

@@ -1,6 +1,8 @@
 """
 Functions for figuring out a filetype or MIME type a given
 file path.
+
+Following WMA, we use WMA's custom short name for a mime type.
 """
 
 import mimetypes
@@ -37,6 +39,7 @@
 # convert these mismatches
 MIME_SHORTNAME_TO_WMA: Final[Dict[str, str]] = {"JPG": "JPEG", "TXT": "Text"}
 
+# FIXME: elements of the below dict should be a dataclass.
 IMPORTERS = {}
 
 # TODO: This hard-coded dictionary should be
@@ -254,7 +257,7 @@ def eval_Import_general(
 
     elements = [el.value for el in elements]
 
-    # Determine file format
+    # Determine WMA version of the mime type.
     file_format = None
     for el in elements.copy():
         if el.upper() in IMPORTERS.keys():
@@ -270,7 +273,8 @@ def eval_Import_general(
         evaluation.predetermined_out = current_predetermined_out
         return SymbolFailed
 
-    # Load the importer
+    # Extract information about the loader used for this MIME type.
+    # FIXME: turn into dataclass
     conditionals, import_function_symbol, posts, importer_options = IMPORTERS[
         file_format
     ]
@@ -419,11 +423,11 @@ def eval_Import_general(
 
 
 def eval_Import_Elements(file_format: str, evaluation):
-    """
-    Basic implementation behind Import[fileformat, Elements].
+    """Basic implementation behind Import[fileformat, Elements].
+
     This returns the element names that can be used for a specific
-    file_format type. We get this from the AvailableElements field
-    mentioned when registering an importer.
+    file_format type. We get this from the
+    AvailableElements field mentioned when registering an importer.
     """
     filetype = MIME_SHORTNAME_TO_WMA.get(file_format, file_format).upper()
 
@@ -450,14 +454,23 @@ def perform_import(
     data: Optional[str],
     elements: Optional[list] = None,
 ):
-    """
-    This routine does the data import.
+    """ This routine does the import. "import" here means reading a  \
+    file or string which has been structured according to a format belonging to a mime type.
+
     "findfile", if not "None", is the path of a file where the unimported data resides.
-    If findfile is empty, then "data" will have the string data for that file, and
+    If "findfile" is empty, then "data" will have the string data for that file, and
     this routine will create a temporary file containing the data. The actual importer
     then uses this file.
 
-    "elements" when given contains the parts or kinds of things that should be extracted.
+    "elements", when given, contains the parts or kinds of things that should be extracted.
+    Usually, there are custom routines for retrieving an element.
+
+    It is also possible that when a custom element extraction does not
+    exist, that the caller will do the filtering after retrieving all of the information.
+
+    This is not advisable when the information inside an element is small compared
+    to the information of the entire importable file. For example consider asking
+    about the member names or contents of tar file compared to the entire tar file.
     """
     current_predetermined_out = evaluation.predetermined_out
     if function_channels == ListExpression(String("FileNames")):