Merge pull request learningequality#6027 from rtibblesbot/issue-6005-a23526

feat: add QTI 3.0 schema validator

Author: rtibbles

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: check-added-large-files
-        exclude: '^.+?\.ttf$'
+        exclude: '^.+?\.ttf$|^contentcuration/contentcuration/utils/assessment/qti/schema/xsd/.+?\.xsd$'
       - id: debug-statements
       - id: end-of-file-fixer
         exclude: '^.+?(\.json|\.po)$'

contentcuration/contentcuration/tests/utils/qti/test_validation.py

@@ -0,0 +1,163 @@
+import os
+import tempfile
+import unittest
+
+from lxml import etree
+
+from contentcuration.utils.assessment.qti.validation import _compiled_schema
+from contentcuration.utils.assessment.qti.validation import validate_qti_item
+
+
+class CompiledSchemaTests(unittest.TestCase):
+    def test_returns_xml_schema_instance(self):
+        self.assertIsInstance(_compiled_schema(), etree.XMLSchema)
+
+    def test_is_cached_across_calls(self):
+        self.assertIs(_compiled_schema(), _compiled_schema())
+
+
+def _item_xml(identifier, title, response_declaration, item_body):
+    return (
+        '<?xml version="1.0" encoding="UTF-8"?>'
+        '<qti-assessment-item xmlns="http://www.imsglobal.org/xsd/imsqtiasi_v3p0" '
+        'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
+        'xsi:schemaLocation="http://www.imsglobal.org/xsd/imsqtiasi_v3p0 '
+        'https://purl.imsglobal.org/spec/qti/v3p0/schema/xsd/imsqti_asiv3p0p1_v1p0.xsd" '
+        'identifier="%s" title="%s" adaptive="false" time-dependent="false" '
+        'language="en-US" tool-name="kolibri" tool-version="0.1">'
+        "%s"
+        '<qti-outcome-declaration identifier="SCORE" cardinality="single" base-type="float" />'
+        "<qti-item-body>%s</qti-item-body>"
+        '<qti-response-processing template="https://purl.imsglobal.org/spec/qti/v3p0/rptemplates/match_correct" />'
+        "</qti-assessment-item>"
+    ) % (identifier, title, response_declaration, item_body)
+
+
+VALID_CHOICE_ITEM = _item_xml(
+    "item_1",
+    "Sample Item",
+    '<qti-response-declaration identifier="RESPONSE" cardinality="single" base-type="identifier">'
+    "<qti-correct-response><qti-value>choice_0</qti-value></qti-correct-response>"
+    "</qti-response-declaration>",
+    '<qti-choice-interaction response-identifier="RESPONSE" max-choices="1" min-choices="0" '
+    'orientation="vertical">'
+    "<qti-prompt>Select the correct answer.</qti-prompt>"
+    '<qti-simple-choice identifier="choice_0" show-hide="show" fixed="false">Option A</qti-simple-choice>'
+    '<qti-simple-choice identifier="choice_1" show-hide="show" fixed="false">Option B</qti-simple-choice>'
+    "</qti-choice-interaction>",
+)
+
+
+class ValidateQTIItemTests(unittest.TestCase):
+    def test_accepts_valid_item(self):
+        result = validate_qti_item(VALID_CHOICE_ITEM)
+        self.assertTrue(result.is_valid)
+        self.assertEqual(result.errors, [])
+
+    def test_accepts_valid_item_as_bytes(self):
+        result = validate_qti_item(VALID_CHOICE_ITEM.encode("utf-8"))
+        self.assertTrue(result.is_valid)
+
+    def test_rejects_invalid_enum_value(self):
+        xml = VALID_CHOICE_ITEM.replace(
+            'orientation="vertical"', 'orientation="sideways"'
+        )
+        result = validate_qti_item(xml)
+        self.assertFalse(result.is_valid)
+        self.assertTrue(
+            any(
+                "orientation" in e.message and "sideways" in e.message
+                for e in result.errors
+            )
+        )
+
+    def test_rejects_missing_required_attribute(self):
+        xml = VALID_CHOICE_ITEM.replace(' time-dependent="false"', "")
+        result = validate_qti_item(xml)
+        self.assertFalse(result.is_valid)
+        self.assertTrue(any("time-dependent" in e.message for e in result.errors))
+
+    def test_rejects_malformed_xml_without_raising(self):
+        result = validate_qti_item("<qti-assessment-item><unclosed>")
+        self.assertFalse(result.is_valid)
+        self.assertEqual(len(result.errors), 1)
+
+    def test_does_not_resolve_external_entities(self):
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
+            f.write("super-secret-value")
+            secret_path = f.name
+        self.addCleanup(os.remove, secret_path)
+        xml = (
+            '<?xml version="1.0"?>'
+            '<!DOCTYPE qti-assessment-item [<!ENTITY xxe SYSTEM "file://%s">]>'
+            '<qti-assessment-item title="&xxe;"></qti-assessment-item>' % secret_path
+        )
+        result = validate_qti_item(xml)
+        serialized = " ".join(e.message for e in result.errors)
+        self.assertNotIn("super-secret-value", serialized)
+
+
+MATCH_INTERACTION_ITEM = _item_xml(
+    "item_match",
+    "Match Item",
+    '<qti-response-declaration identifier="RESPONSE" cardinality="multiple" base-type="directedPair">'
+    "<qti-correct-response><qti-value>A X</qti-value><qti-value>B Y</qti-value></qti-correct-response>"
+    "</qti-response-declaration>",
+    '<qti-match-interaction response-identifier="RESPONSE" shuffle="false" max-associations="1">'
+    "<qti-prompt>Match the fruit to its color.</qti-prompt>"
+    "<qti-simple-match-set>"
+    '<qti-simple-associable-choice identifier="A" match-max="1">Apple</qti-simple-associable-choice>'
+    '<qti-simple-associable-choice identifier="B" match-max="1">Banana</qti-simple-associable-choice>'
+    "</qti-simple-match-set>"
+    "<qti-simple-match-set>"
+    '<qti-simple-associable-choice identifier="X" match-max="1">Red</qti-simple-associable-choice>'
+    '<qti-simple-associable-choice identifier="Y" match-max="1">Yellow</qti-simple-associable-choice>'
+    "</qti-simple-match-set>"
+    "</qti-match-interaction>",
+)
+
+ORDER_INTERACTION_ITEM = _item_xml(
+    "item_order",
+    "Order Item",
+    '<qti-response-declaration identifier="RESPONSE" cardinality="ordered" base-type="identifier">'
+    "<qti-correct-response>"
+    "<qti-value>step1</qti-value><qti-value>step2</qti-value><qti-value>step3</qti-value>"
+    "</qti-correct-response>"
+    "</qti-response-declaration>",
+    '<qti-order-interaction response-identifier="RESPONSE" shuffle="false" orientation="vertical">'
+    "<qti-prompt>Order the steps.</qti-prompt>"
+    '<qti-simple-choice identifier="step1" show-hide="show" fixed="false">First</qti-simple-choice>'
+    '<qti-simple-choice identifier="step2" show-hide="show" fixed="false">Second</qti-simple-choice>'
+    '<qti-simple-choice identifier="step3" show-hide="show" fixed="false">Third</qti-simple-choice>'
+    "</qti-order-interaction>",
+)
+
+
+class UncoveredInteractionTypeTests(unittest.TestCase):
+    """Interaction types the pydantic models / QTIExerciseGenerator never produce."""
+
+    def test_accepts_match_interaction_item(self):
+        result = validate_qti_item(MATCH_INTERACTION_ITEM)
+        self.assertTrue(result.is_valid)
+        self.assertEqual(result.errors, [])
+
+    def test_accepts_order_interaction_item(self):
+        result = validate_qti_item(ORDER_INTERACTION_ITEM)
+        self.assertTrue(result.is_valid)
+        self.assertEqual(result.errors, [])
+
+    def test_rejects_item_with_unknown_root_element(self):
+        result = validate_qti_item('<not-qti identifier="x" />')
+        self.assertFalse(result.is_valid)
+        self.assertTrue(result.errors)
+
+
+class SchemaReuseTests(unittest.TestCase):
+    def test_schema_compiled_once_across_multiple_validate_calls(self):
+        _compiled_schema.cache_clear()
+        self.addCleanup(_compiled_schema.cache_clear)
+        validate_qti_item(VALID_CHOICE_ITEM)
+        validate_qti_item(MATCH_INTERACTION_ITEM)
+        validate_qti_item(ORDER_INTERACTION_ITEM)
+        self.assertEqual(_compiled_schema.cache_info().misses, 1)
+        self.assertEqual(_compiled_schema.cache_info().hits, 2)

contentcuration/contentcuration/utils/assessment/qti/__init__.py

@@ -1,6 +1,12 @@
 from .base import ElementTreeBase
+from .validation import QTIValidationError
+from .validation import QTIValidationResult
+from .validation import validate_qti_item
 
 
 __all__ = [
     "ElementTreeBase",
+    "QTIValidationError",
+    "QTIValidationResult",
+    "validate_qti_item",
 ]

contentcuration/contentcuration/utils/assessment/qti/schema/README.md

@@ -0,0 +1,29 @@
+# Vendored QTI 3.0 schema
+
+XSD files backing `contentcuration.utils.assessment.qti.validation`. Source: QTI 3.0.1
+(IMS Global / 1EdTech), https://www.imsglobal.org/spec/qti/v3p0/impl.
+
+- `xsd/imsqti_itemv3p0p1_v1p0.xsd` — the QTI-item-scoped schema (validates a single
+  `qti-assessment-item` document; does not accept `qti-assessment-test`/`-section` roots).
+  Source: https://purl.imsglobal.org/spec/qti/v3p0/schema/xsd/imsqti_itemv3p0p1_v1p0.xsd
+- `xsd/xml.xsd`, `xsd/XInclude.xsd` — W3C schemas for the `xml:` namespace and XInclude.
+  Source: https://purl.imsglobal.org/spec/w3/2001/schema/xsd/{xml,XInclude}.xsd
+- `xsd/mathml3*.xsd` — MathML 3 schema (content, presentation, common, strict-content).
+  Source: https://purl.imsglobal.org/spec/mathml/v3p0/schema/xsd/
+- `xsd/ssmlv1p1-core.xsd`, `xsd/synthesis-nonamespace.xsd` — SSML 1.1 schema (QTI allows
+  SSML markup for text-to-speech hints).
+  Source: https://purl.imsglobal.org/spec/ssml/v1p1/schema/xsd/
+
+All `schemaLocation` attributes that originally pointed at `https://purl.imsglobal.org/...`
+have been rewritten to local relative filenames so the schema compiles with no network
+access at runtime — required since bulk publish and ricecooker upload validate many items
+per run. Do not restore the absolute URLs.
+
+To refresh to a newer QTI point release, run `refresh_schema.py` (re-downloads each
+file above and re-applies the `schemaLocation` rewrites), then re-run the full
+`test_validation.py` suite:
+
+```
+python contentcuration/contentcuration/utils/assessment/qti/schema/refresh_schema.py
+pytest contentcuration/contentcuration/tests/utils/qti/test_validation.py -v
+```

contentcuration/contentcuration/utils/assessment/qti/schema/refresh_schema.py

@@ -0,0 +1,122 @@
+"""Re-vendor the QTI 3.0 item XSD schema tree.
+
+Downloads the QTI-item XSD and its W3C/MathML/SSML dependencies, then
+rewrites their `schemaLocation` attributes from absolute
+`https://purl.imsglobal.org/...` URLs to local relative filenames so the
+schema tree compiles with no network access at runtime.
+
+Usage: python refresh_schema.py
+(Run from any directory; writes into the `xsd/` subdirectory next to this
+script.) Afterwards, re-run the validation test suite:
+
+    pytest contentcuration/contentcuration/tests/utils/qti/test_validation.py -v
+"""
+import codecs
+import socket
+import subprocess
+from pathlib import Path
+
+import requests
+from lxml import etree
+
+
+XSD_DIR = Path(__file__).parent / "xsd"
+
+SOURCES = {
+    "imsqti_itemv3p0p1_v1p0.xsd": "https://purl.imsglobal.org/spec/qti/v3p0/schema/xsd/imsqti_itemv3p0p1_v1p0.xsd",
+    "xml.xsd": "https://purl.imsglobal.org/spec/w3/2001/schema/xsd/xml.xsd",
+    "XInclude.xsd": "https://purl.imsglobal.org/spec/w3/2001/schema/xsd/XInclude.xsd",
+    "mathml3.xsd": "https://purl.imsglobal.org/spec/mathml/v3p0/schema/xsd/mathml3.xsd",
+    "mathml3-content.xsd": "https://purl.imsglobal.org/spec/mathml/v3p0/schema/xsd/mathml3-content.xsd",
+    "mathml3-presentation.xsd": "https://purl.imsglobal.org/spec/mathml/v3p0/schema/xsd/mathml3-presentation.xsd",
+    "mathml3-common.xsd": "https://purl.imsglobal.org/spec/mathml/v3p0/schema/xsd/mathml3-common.xsd",
+    "mathml3-strict-content.xsd": "https://purl.imsglobal.org/spec/mathml/v3p0/schema/xsd/mathml3-strict-content.xsd",
+    "ssmlv1p1-core.xsd": "https://purl.imsglobal.org/spec/ssml/v1p1/schema/xsd/ssmlv1p1-core.xsd",
+    "synthesis-nonamespace.xsd": "https://purl.imsglobal.org/spec/ssml/v1p1/schema/xsd/synthesis-nonamespace.xsd",
+}
+
+# schemaLocation rewrites: file -> its dependencies (keys into SOURCES),
+# each rewritten from the dependency's absolute URL to its local filename.
+REWRITES = {
+    "imsqti_itemv3p0p1_v1p0.xsd": [
+        "xml.xsd",
+        "XInclude.xsd",
+        "mathml3.xsd",
+        "ssmlv1p1-core.xsd",
+    ],
+    "ssmlv1p1-core.xsd": ["xml.xsd"],
+    "synthesis-nonamespace.xsd": ["xml.xsd"],
+}
+
+
+def _normalize_encoding(content):
+    # purl.imsglobal.org serves XInclude.xsd as UTF-16 with no XML declaration,
+    # unlike every other source file here (plain UTF-8); transcode rather than
+    # leaving it as the only non-UTF-8, non-diffable file in the vendored tree.
+    # Decoding/re-encoding (instead of round-tripping through lxml) preserves
+    # the original formatting instead of reflowing it.
+    if content.startswith(codecs.BOM_UTF16_LE) or content.startswith(
+        codecs.BOM_UTF16_BE
+    ):
+        text = content.decode("utf-16")
+        if not text.lstrip().startswith("<?xml"):
+            text = '<?xml version="1.0" encoding="UTF-8"?>\n' + text
+        return text.encode("utf-8")
+    return content
+
+
+def download():
+    XSD_DIR.mkdir(parents=True, exist_ok=True)
+    for filename, url in SOURCES.items():
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+        (XSD_DIR / filename).write_bytes(_normalize_encoding(response.content))
+
+
+def rewrite_schema_locations():
+    for filename, deps in REWRITES.items():
+        path = XSD_DIR / filename
+        text = path.read_text()
+        for dep in deps:
+            old = 'schemaLocation="%s"' % SOURCES[dep]
+            new = 'schemaLocation="%s"' % dep
+            assert old in text, "%r not found in %s" % (old, filename)
+            text = text.replace(old, new)
+        path.write_text(text)
+
+
+def run_pre_commit():
+    # Applies the repo's trailing-whitespace/end-of-file-fixer hooks (the only
+    # hooks whose `files:` patterns match .xsd) so the vendored tree matches
+    # the same convention already applied to the committed copies. Exit code
+    # 1 just means files were modified; that's expected, not a failure.
+    subprocess.run(
+        ["pre-commit", "run", "--files"]
+        + [str(path) for path in sorted(XSD_DIR.glob("*.xsd"))],
+        check=False,
+    )
+
+
+def verify_compiles_offline():
+    real_socket = socket.socket
+
+    def _blocked(*args, **kwargs):
+        raise RuntimeError("network access attempted while compiling vendored schema")
+
+    socket.socket = _blocked
+    try:
+        etree.XMLSchema(etree.parse(str(XSD_DIR / "imsqti_itemv3p0p1_v1p0.xsd")))
+    finally:
+        socket.socket = real_socket
+
+
+def main():
+    download()
+    rewrite_schema_locations()
+    run_pre_commit()
+    verify_compiles_offline()
+    print("Vendored schema refreshed in %s" % XSD_DIR)  # noqa: T201
+
+
+if __name__ == "__main__":
+    main()

contentcuration/contentcuration/utils/assessment/qti/schema/xsd/XInclude.xsd

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+           xmlns:xi="http://www.w3.org/2001/XInclude"
+           targetNamespace="http://www.w3.org/2001/XInclude"
+           finalDefault="extension">
+  <xs:annotation>
+   <xs:documentation>
+    Not normative, but may be useful.
+    See the REC http://www.w3.org/TR/XInclude for definitive
+    information about this namespace.
+   </xs:documentation>
+  </xs:annotation>
+
+  <xs:element name="include" type="xi:includeType" />
+
+  <xs:complexType name="includeType" mixed="true">
+    <xs:choice minOccurs='0' maxOccurs='unbounded' >
+      <xs:element ref='xi:fallback' />
+      <xs:any namespace='##other' processContents='lax' />
+      <xs:any namespace='##local' processContents='lax' />
+    </xs:choice>
+    <xs:attribute name="href" use="optional" type="xs:anyURI"/>
+    <xs:attribute name="parse" use="optional" default="xml"
+                  type="xi:parseType" />
+    <xs:attribute name="xpointer" use="optional" type="xs:string"/>
+    <xs:attribute name="encoding" use="optional" type="xs:string"/>
+    <xs:attribute name="accept" use="optional" type="xs:string"/>
+    <xs:attribute name="accept-language" use="optional" type="xs:string"/>
+    <xs:anyAttribute namespace="##other" processContents="lax"/>
+  </xs:complexType>
+
+  <xs:simpleType name="parseType">
+    <xs:restriction base="xs:token">
+      <xs:enumeration value="xml"/>
+      <xs:enumeration value="text"/>
+    </xs:restriction>
+  </xs:simpleType>
+
+  <xs:element name="fallback" type="xi:fallbackType" />
+
+  <xs:complexType name="fallbackType" mixed="true">
+    <xs:choice minOccurs="0" maxOccurs="unbounded">
+      <xs:element ref="xi:include"/>
+      <xs:any namespace="##other" processContents="lax"/>
+      <xs:any namespace="##local" processContents="lax"/>
+    </xs:choice>
+    <xs:anyAttribute namespace="##other" processContents="lax" />
+  </xs:complexType>
+
+</xs:schema>