Towards handling Import[xxx, "ZIP", member-name] (#1846)

Plumbing hooked up for Import zip with members.
    
Now we just gotta get ImportString[xxx, "JSON"] working on its own.
    
Also, other small fixes and improvements. 

More changes to `Import` will be made in a future PR.

Author: rocky

SYMBOLS_MANIFEST.txt

@@ -1,3 +1,4 @@
+Compress`ImportZIP
 HTML`DataImport
 HTML`FullDataImport
 HTML`HyperlinksImport
@@ -12,7 +13,7 @@ ImportExport`RegisterExport
 ImportExport`RegisterImport
 Internal`RealValuedNumberQ
 Internal`RealValuedNumericQ
-JSON`Import`JSONImport
+JSON`ImportJSON
 System`$Aborted
 System`$Assumptions
 System`$BaseDirectory
@@ -582,9 +583,7 @@ System`ImageTake
 System`ImageType
 System`Implies
 System`Import
-System`ImportJSON
 System`ImportString
-System`ImportZIP
 System`In
 System`Increment
 System`Indeterminate

mathics/SystemFiles/Formats/JSON/Import.wl

@@ -1,15 +1,19 @@
 (* ::Package:: *)
 
+(* JSON Javascript Object Notation or JSON web service description Importer.
+   This is used by Import[] and, ImportString[].
+ *)
+
 Begin["System`Convert`JSONDump`"]
 
 (* JSON legacy element is Data even if Expression would be better. *)
 $AvailableElements = {"Data", "Dataset"};
 
 ImportExport`RegisterImport[
-    "JSON",
-    ImportJSON,
+    "JSON",  (* WMA mime-type name *)
+    JSON`ImportJSON, (* Default Function name that handles this. *)
     {},
-    "AvailableElements" -> $AvailableElements,
+    "AvailableElements" -> $AvailableElements, (* names retuned by "Elements" query *)
     "FunctionChannels" -> {"FileNames"},
     "DefaultElement" -> "Data"
 ]

mathics/SystemFiles/Formats/ZIP/Import.wl

@@ -1,7 +1,7 @@
 (* ::Package:: *)
 
-(* ZIP compressed file and file archive Importer.
-   This is used by Import[].
+(* Windows ZIP archive, ZIP compressed file and file archive Importer.
+   This is used by Import[] and, ImportString[].
  *)
 
 Begin["System`Convert`CommonArchiveDump`"]
@@ -21,13 +21,13 @@ GetElements[___] :=
 		];
 
 ImportExport`RegisterImport[
-    "ZIP",
-    ImportZIP,
+    "ZIP", (* WMA mime-type name *)
+    Compress`ImportZIP,  (* Default Function name that handles this. *)
     {}, (* Post importer function(s) *)
     FunctionChannels -> {"FileNames"},
     (* WMA has this, but I (rocky) am not sure why or what it means:
     AvailableElements -> $ZIPAvailableElements, *)
-    AvailableElements -> {"Filenames", "Summary"},
+    AvailableElements -> {"Filenames", "Summary"},  (* names retuned by "Elements" query *)
     BinaryFormat -> True,
     DefaultElement -> "FileNames",
     HiddenElements -> $ZIPHiddenElements,

mathics/builtin/fileformats/__init__.py

@@ -1,10 +1,40 @@
-"""
-File Formats
+r"""Import/Export File Formats, Importers and Exporters
+
+The data of files on a filesystem or retrieved from the Internet often are structured \
+according to a specific structures and rules. For example, consider different kinds of \
+structuring used in a JSON file, versus an HTML files, or a compressed GZIP file.
+
+In some cases, such as archive files, e.g., ZIP, TAR, and JAR, the file contains component parts, \
+which in WMA terminology are called "members" which is part of the broader metadata items \
+called "elements".
+
+A MIME type is typically associated with each kind of format. \Mathics3, following WMA, \
+uses a shortend name for this MIME type. For example \Mathics3 uses "HTML" as a shorthand \
+for the MIME type "text/html".
+
+Below is a list of file supported file types that we have builtin importers or exporters written \
+in Python. Other importers, however, are written in \Mathics3.
+
+Variable <url>
+:\$ExportFormats:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/\$exportformats</url> \
+contains a list of file formats that are supported by <url>
+:Export:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/export</url>, \
+while <url>
+:\$ImportFormats:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/\$importformats</url> \
+does the corresponding thing for <url>
+:Import:
+/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/import</url>.
 
-Built-in Importers.
+Many Import/Export functions are registered in SystemFiles/Formats/*.wl which is \
+autoloaded on startup.
 
+The Built-in Functions are defined in a separate context.
+For example, HTML` or Compress`.  This is done to not pollute the System` namespace.
 """
 
-# The Built-in Functions are defined in a separate context under the
-# System`. For example System`HTML` and System`XML.  This is done to not
-# pollute the System` namespace.
+# This tells documentation how to sort this module
+# Here we are also hiding "file_io" since this can erroneously appear at the top level.
+sort_order = "mathics.builtin.importing-export-file-formats"

mathics/builtin/fileformats/compression.py

@@ -0,0 +1,32 @@
+"""
+Compression & Archive Formats
+"""
+
+from mathics.core.builtin import Builtin, String
+from mathics.core.evaluation import Evaluation
+from mathics.eval.fileformats.compression import eval_ImportZIP
+
+# See commit in __init__.py regarding the whacky way this gets called
+
+
+class ImportZIP(Builtin):
+    """
+    <url>:WMA link:https://reference.wolfram.com/language/ref/format/ZIP.html</url>
+
+    <dl>
+      <dt>'Compress`ImportZIP[path]'
+      <dd>Run zip for archive file $path$
+    </dl>
+
+    """
+
+    context = "Compress`"
+    summary_text = "import a ZIP file"
+
+    def eval(self, path: String, evaluation: Evaluation):
+        "Compress`ImportZIP[path_String]"
+        return eval_ImportZIP(path, evaluation)
+
+    def eval_with_elements(self, path: String, elements, evaluation: Evaluation):
+        "Compress`ImportZIP[path_String, elements_]"
+        return eval_ImportZIP(path, evaluation, elements)

mathics/builtin/fileformats/htmlformat.py

@@ -2,7 +2,7 @@
 """
 HTML
 
-Basic implementation for a HTML importer.
+HTML importer.
 """
 
 
@@ -15,6 +15,7 @@
 from mathics.core.builtin import Builtin, MessageException
 from mathics.core.convert.expression import to_expression, to_mathics_list
 from mathics.core.convert.python import from_python
+from mathics.core.evaluation import Evaluation
 from mathics.core.expression import Expression
 from mathics.core.list import ListExpression
 from mathics.core.symbols import Symbol
@@ -126,7 +127,7 @@ class _TagImport(_HTMLBuiltin):
     def _import(self, tree):
         raise NotImplementedError
 
-    def eval(self, text, evaluation):
+    def eval(self, text: String, evaluation: Evaluation):
         """%(name)s[text_String]"""
         tree = parse_html(parse_html_file, text, evaluation)
         if isinstance(tree, Symbol):  # $Failed?
@@ -135,6 +136,12 @@ def eval(self, text, evaluation):
             to_expression(SymbolRule, self.tag_name, self._import(tree))
         )
 
+    def eval_with_element(self, text, element, evaluation: Evaluation):
+        """%(name)s[text_String, element_]"""
+        # FIXME: right now we aren't using element, and should use this to more
+        # efficiently extract part of the XML file that we want.
+        return self.eval(text, evaluation)
+
 
 class _Get(_HTMLBuiltin):
     context = "HTML`Parser`"
@@ -401,7 +408,7 @@ class SourceImport(_HTMLBuiltin):
 
     summary_text = "import source code from a HTML file"
 
-    def eval(self, text, evaluation):
+    def eval(self, text, evaluation: Evaluation):
         """%(name)s[text_String]"""
 
         def source(filename):
@@ -412,6 +419,12 @@ def source(filename):
 
         return parse_html(source, text, evaluation)
 
+    def eval_with_element(self, text, element, evaluation: Evaluation):
+        """%(name)s[text_String, element_]"""
+        # FIXME: right now we aren't using element, and should use this to more
+        # efficiently extract part of the XML file that we want.
+        return self.eval(text, evaluation)
+
 
 class TitleImport(_TagImport):
     """
@@ -437,7 +450,7 @@ def _import(self, tree):
 
 class XMLObjectImport(_HTMLBuiltin):
     """
-    ## <url>:native internal:</url>
+    <url>:WMA link:https://reference.wolfram.com/language/ref/XMLObject.html</url>
 
     <dl>
     <dt>'HTML`XMLObjectImport["filename"]'
@@ -450,7 +463,13 @@ class XMLObjectImport(_HTMLBuiltin):
 
     summary_text = "import XML objects from a HTML file"
 
-    def eval(self, text, evaluation):
+    def eval(self, text, evaluation: Evaluation):
         """%(name)s[text_String]"""
         xml = to_expression("HTML`Parser`HTMLGet", text).evaluate(evaluation)
         return ListExpression(Expression(SymbolRule, String("XMLObject"), xml))
+
+    def eval_with_element(self, text, element, evaluation: Evaluation):
+        """%(name)s[text_String, element_]"""
+        # FIXME: right now we aren't using element, and should use this to more
+        # efficiently extract part of the HTML file that we want.
+        return self.eval(text, evaluation)

mathics/builtin/fileformats/jsonformat.py

@@ -1,32 +1,29 @@
-# -*- coding: utf-8 -*-
-
 """
-JSON
+JSON File Format
 
-Basic implementation for an JSON importer.
+JSON importer (via Python's "json" module).
 """
 
-from mathics.core.builtin import Builtin
-from mathics.core.expression import Evaluation
+from mathics.core.builtin import Builtin, String
+from mathics.core.evaluation import Evaluation
 from mathics.eval.fileformats.jsonformat import eval_JSONImport
 
 
-class JSONImport(Builtin):
+class ImportJSON(Builtin):
     """
-    ## <url>:native internal:</url>
+    <url>:WMA link:https://reference.wolfram.com/language/ref/format/JSON.html</url>
 
     <dl>
-      <dt>'JSON`Import`JSONImport["file"]'
-      <dd>parses "string" as a JSON file, and returns the data as a nested \
-          list of rules.
+      <dt>'JSON`ImportJSON[path]'
+      <dd>Read $path$ as JSON and convert that to its corresponding Mathics3 equivalent.
     </dl>
 
     """
 
-    summary_text = "import elements from json"
-    context = "JSON`Import`"
+    context = "JSON`"
     messages = {"dec": "Decoding Error at `1`"}
+    summary_text = "import JSON file"
 
-    def eval(self, filename, evaluation: Evaluation):
-        """%(name)s[filename_String]"""
-        return eval_JSONImport(filename.value, evaluation)
+    def eval(self, path: String, evaluation: Evaluation):
+        "JSON`ImportJSON[path_String]"
+        return eval_JSONImport(path, evaluation)

mathics/builtin/fileformats/xmlformat.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 
 """
-XML
+XML File Format
 
-Basic implementation for an XML importer.
+XML importer (via lxml).
 """
 
 
@@ -345,6 +345,12 @@ def lines():
         plaintext = String("\n".join(lines()))
         return to_mathics_list(to_expression("Rule", "Plaintext", plaintext))
 
+    def eval_with_element(self, text, element, evaluation: Evaluation):
+        """%(name)s[text_String, element_]"""
+        # FIXME: right now we aren't using element, and should use this to more
+        # efficiently extract part of the XML file that we want.
+        return self.eval(text, evaluation)
+
 
 class TagsImport(Builtin):
     """
@@ -381,6 +387,12 @@ def eval(self, text, evaluation: Evaluation):
             return root
         return to_mathics_list(to_expression("Rule", "Tags", self._tags(root)))
 
+    def eval_with_element(self, text, element, evaluation: Evaluation):
+        """%(name)s[text_String, element_]"""
+        # FIXME: right now we aren't using element, and should use this to more
+        # efficiently extract part of the XML file that we want.
+        return self.eval(text, evaluation)
+
 
 class XMLObjectImport(Builtin):
     """
@@ -405,3 +417,9 @@ def eval(self, text, evaluation: Evaluation):
         """%(name)s[text_String]"""
         xml = to_expression("XML`Parser`XMLGet", text).evaluate(evaluation)
         return to_mathics_list(to_expression("Rule", "XMLObject", xml))
+
+    def eval_with_element(self, text, element, evaluation: Evaluation):
+        """%(name)s[text_String, element_]"""
+        # FIXME: right now we aren't using element, and should use this to more
+        # efficiently extract part of the XML file that we want.
+        return self.eval(text, evaluation)