From d9abb82fbdc040db248484940c6cebbd3d94ab21 Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Thu, 23 Oct 2025 12:27:44 +0100 Subject: [PATCH 01/15] check_isa_schemas updated to new json validator. #591 --- isatools/isajson/validate.py | 6 +++--- pyproject.toml | 1 + tests/validators/test_validators.py | 1 - uv.lock | 14 ++++++++++++++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index b875ddc07..08b4549f8 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -14,7 +14,7 @@ import re from io import StringIO -from jsonschema import Draft4Validator, RefResolver, ValidationError +from jsonschema import Draft4Validator, ValidationError from isatools.isajson.load import load @@ -548,9 +548,9 @@ def check_isa_schemas(isa_json, investigation_schema_path): try: with open(investigation_schema_path) as fp: investigation_schema = json.load(fp) - resolver = RefResolver("file://" + investigation_schema_path, investigation_schema) - validator = Draft4Validator(investigation_schema, resolver=resolver) + validator = Draft4Validator(investigation_schema) validator.validate(isa_json) + #jsonschema.validate(instance=isa_json, schema=investigation_schema, cls=jsonschema.Draft4Validator) except ValidationError as ve: errors.append({"message": "Invalid JSON against ISA-JSON schemas", "supplemental": str(ve), "code": 3}) log.fatal("(F) The JSON does not validate against the provided ISA-JSON schemas!") diff --git a/pyproject.toml b/pyproject.toml index 3021f289c..8858c32ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dependencies = [ "graphene~=3.4.3", "graphql-core~=3.2.6", "ruff>=0.14.1", + "pytest-timeout>=2.4.0", ] [project.optional-dependencies] diff --git a/tests/validators/test_validators.py b/tests/validators/test_validators.py index eea316f91..1c73753ac 100644 --- a/tests/validators/test_validators.py +++ b/tests/validators/test_validators.py @@ -484,7 +484,6 @@ def test_info_reporting_mtbls1_isatab(self): report = isatab.validate( fp=test_case_fp, config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level ) - print(report) # self.assertIn( # {'supplemental': 'Found 4 study groups in s_MTBLS1.txt', # 'code': 5001, diff --git a/uv.lock b/uv.lock index d3db8632b..d36b93e82 100644 --- a/uv.lock +++ b/uv.lock @@ -1288,6 +1288,7 @@ dependencies = [ { name = "openpyxl" }, { name = "pandas" }, { name = "progressbar2" }, + { name = "pytest-timeout" }, { name = "python-dateutil" }, { name = "pyyaml" }, { name = "rdflib" }, @@ -1338,6 +1339,7 @@ requires-dist = [ { name = "openpyxl", specifier = ">=3.1.5" }, { name = "pandas", specifier = ">=2.2.3,<3" }, { name = "progressbar2", specifier = "~=4.5.0" }, + { name = "pytest-timeout", specifier = ">=2.4.0" }, { name = "python-dateutil", specifier = "~=2.9.0.post0" }, { name = "pyyaml", specifier = "~=6.0.2" }, { name = "rdflib", specifier = "~=7.2.1" }, @@ -2301,6 +2303,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, ] +[[package]] +name = "pytest-timeout" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From bf8198085601375c243e1df9941f140fe8a245a1 Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Thu, 23 Oct 2025 14:00:50 +0100 Subject: [PATCH 02/15] Changed json validation in two more files #591 --- isatools/convert/isatab2cedar.py | 5 ++--- isatools/convert/isatab2json.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/isatools/convert/isatab2cedar.py b/isatools/convert/isatab2cedar.py index 066fd043f..711758dc3 100644 --- a/isatools/convert/isatab2cedar.py +++ b/isatools/convert/isatab2cedar.py @@ -41,8 +41,7 @@ def createCEDARjson(self, work_dir, json_dir, inv_identifier): schema = json.load(json_fp) if schema is None: raise IOError("Could not load schema from {}".format(join(CEDAR_SCHEMA_PATH, schema_file))) - resolver = RefResolver("file://{}".format(join(CEDAR_SCHEMA_PATH, schema_file)), schema) - validator = Draft4Validator(schema, resolver=resolver) + validator = Draft4Validator(schema) isa_tab = isatab_parser.parse(work_dir) @@ -121,7 +120,7 @@ def createCEDARjson(self, work_dir, json_dir, inv_identifier): study_identifier = "" try: - validator.validate(cedar_json, schema) + validator.validate(cedar_json) except ValidationError as e: error_file_name = os.path.join(json_dir, "error.log") with open(error_file_name, "w") as errorfile: diff --git a/isatools/convert/isatab2json.py b/isatools/convert/isatab2json.py index 193029770..222c9e01d 100644 --- a/isatools/convert/isatab2json.py +++ b/isatools/convert/isatab2json.py @@ -140,9 +140,8 @@ def convert(self, work_dir): # validate json with open(join(SCHEMAS_PATH, INVESTIGATION_SCHEMA)) as json_fp: schema = json.load(json_fp) - resolver = RefResolver("file://" + join(SCHEMAS_PATH, INVESTIGATION_SCHEMA), schema) - validator = Draft4Validator(schema, resolver=resolver) - validator.validate(isa_json, schema) + validator = Draft4Validator(schema) + validator.validate(isa_json) log.info("Conversion finished") return isa_json From f8f941e1405c27eeb6a62ed14c2cdf8efd8bb201 Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Thu, 23 Oct 2025 15:47:32 +0100 Subject: [PATCH 03/15] Test speed fixed but can't validate JSON again. #591 --- isatools/convert/isatab2cedar.py | 14 +++++++++----- isatools/convert/isatab2json.py | 10 +++++++--- isatools/isajson/validate.py | 7 +++++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/isatools/convert/isatab2cedar.py b/isatools/convert/isatab2cedar.py index 711758dc3..246c21cc3 100644 --- a/isatools/convert/isatab2cedar.py +++ b/isatools/convert/isatab2cedar.py @@ -10,9 +10,11 @@ from os.path import isdir, join from uuid import uuid4 -from jsonschema import Draft4Validator, RefResolver -from jsonschema.exceptions import ValidationError +from jsonschema import Draft4Validator +from referencing.jsonschema import DRAFT4 +from referencing import Registry +from jsonschema.exceptions import ValidationError from isatools.io import isatab_parser __author__ = "agbeltran" @@ -38,10 +40,12 @@ def createCEDARjson(self, work_dir, json_dir, inv_identifier): log.info("Converting ISA to CEDAR model for {}".format(work_dir)) schema_file = "investigation_template.json" with open(join(CEDAR_SCHEMA_PATH, schema_file)) as json_fp: - schema = json.load(json_fp) - if schema is None: + investigation_schema = json.load(json_fp) + if investigation_schema is None: raise IOError("Could not load schema from {}".format(join(CEDAR_SCHEMA_PATH, schema_file))) - validator = Draft4Validator(schema) + schema = DRAFT4.create_resource(investigation_schema) + registry = Registry.with_resource(investigation_schema['id'], schema) + validator = Draft4Validator(investigation_schema, registry=registry) isa_tab = isatab_parser.parse(work_dir) diff --git a/isatools/convert/isatab2json.py b/isatools/convert/isatab2json.py index 222c9e01d..021aaea5e 100644 --- a/isatools/convert/isatab2json.py +++ b/isatools/convert/isatab2json.py @@ -10,7 +10,9 @@ from os.path import join from uuid import uuid4 -from jsonschema import Draft4Validator, RefResolver +from jsonschema import Draft4Validator +from referencing import Registry +from referencing.jsonschema import DRAFT4 from isatools import isatab from isatools.io.isatab_parser import parse @@ -139,8 +141,10 @@ def convert(self, work_dir): # validate json with open(join(SCHEMAS_PATH, INVESTIGATION_SCHEMA)) as json_fp: - schema = json.load(json_fp) - validator = Draft4Validator(schema) + investigation_schema = json.load(json_fp) + schema = DRAFT4.create_resource(investigation_schema) + registry = Registry.with_resource(investigation_schema['id'], schema) + validator = Draft4Validator(investigation_schema, registry=registry) validator.validate(isa_json) log.info("Conversion finished") diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index 08b4549f8..dab30015a 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -15,6 +15,8 @@ from io import StringIO from jsonschema import Draft4Validator, ValidationError +from referencing import Registry +from referencing.jsonschema import DRAFT4 from isatools.isajson.load import load @@ -548,9 +550,10 @@ def check_isa_schemas(isa_json, investigation_schema_path): try: with open(investigation_schema_path) as fp: investigation_schema = json.load(fp) - validator = Draft4Validator(investigation_schema) + schema = DRAFT4.create_resource(investigation_schema) + registry = Registry.with_resource(investigation_schema['id'], schema) + validator = Draft4Validator(investigation_schema, registry=registry) validator.validate(isa_json) - #jsonschema.validate(instance=isa_json, schema=investigation_schema, cls=jsonschema.Draft4Validator) except ValidationError as ve: errors.append({"message": "Invalid JSON against ISA-JSON schemas", "supplemental": str(ve), "code": 3}) log.fatal("(F) The JSON does not validate against the provided ISA-JSON schemas!") From f50a6e6be13cf880bd37b30502abea1339dba1a0 Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Fri, 24 Oct 2025 16:49:14 +0100 Subject: [PATCH 04/15] Some tests fixed; one test issue outstanding #591 --- isatools/convert/isatab2cedar.py | 21 +++++++++++--- isatools/convert/isatab2json.py | 29 +++++++++++++++---- isatools/isajson/validate.py | 25 ++++++++++++---- .../core/assay_schema.json | 2 +- .../core/comment_schema.json | 2 +- .../core/data_schema.json | 2 +- .../core/factor_schema.json | 2 +- .../core/factor_value_schema.json | 2 +- .../core/investigation_schema.json | 2 +- .../core/material_attribute_schema.json | 2 +- .../core/material_attribute_value_schema.json | 2 +- .../core/material_schema.json | 2 +- .../core/ontology_annotation_schema.json | 2 +- .../ontology_source_reference_schema.json | 2 +- .../core/organization_schema.json | 2 +- .../core/person_schema.json | 2 +- .../core/process_parameter_value_schema.json | 2 +- .../core/process_schema.json | 2 +- .../core/protocol_parameter_schema.json | 2 +- .../core/protocol_schema.json | 2 +- .../core/publication_schema.json | 2 +- .../core/sample_schema.json | 2 +- .../core/source_schema.json | 2 +- .../core/study_schema.json | 2 +- 24 files changed, 80 insertions(+), 37 deletions(-) diff --git a/isatools/convert/isatab2cedar.py b/isatools/convert/isatab2cedar.py index 246c21cc3..5ff141241 100644 --- a/isatools/convert/isatab2cedar.py +++ b/isatools/convert/isatab2cedar.py @@ -8,9 +8,10 @@ import os from os import listdir from os.path import isdir, join +from pathlib import Path from uuid import uuid4 -from jsonschema import Draft4Validator +from jsonschema import Draft4Validator, FormatChecker from referencing.jsonschema import DRAFT4 from referencing import Registry @@ -43,9 +44,21 @@ def createCEDARjson(self, work_dir, json_dir, inv_identifier): investigation_schema = json.load(json_fp) if investigation_schema is None: raise IOError("Could not load schema from {}".format(join(CEDAR_SCHEMA_PATH, schema_file))) - schema = DRAFT4.create_resource(investigation_schema) - registry = Registry.with_resource(investigation_schema['id'], schema) - validator = Draft4Validator(investigation_schema, registry=registry) + + resources = [] + schemas_dir = Path(CEDAR_SCHEMA_PATH) + investigation_schema_path = Path(join(CEDAR_SCHEMA_PATH, schema_file)) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = investigation_schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) isa_tab = isatab_parser.parse(work_dir) diff --git a/isatools/convert/isatab2json.py b/isatools/convert/isatab2json.py index 021aaea5e..61f065755 100644 --- a/isatools/convert/isatab2json.py +++ b/isatools/convert/isatab2json.py @@ -8,11 +8,12 @@ import re from enum import Enum from os.path import join +from pathlib import Path from uuid import uuid4 -from jsonschema import Draft4Validator +from jsonschema import Draft202012Validator, FormatChecker from referencing import Registry -from referencing.jsonschema import DRAFT4 +from referencing.jsonschema import DRAFT202012 from isatools import isatab from isatools.io.isatab_parser import parse @@ -141,10 +142,26 @@ def convert(self, work_dir): # validate json with open(join(SCHEMAS_PATH, INVESTIGATION_SCHEMA)) as json_fp: - investigation_schema = json.load(json_fp) - schema = DRAFT4.create_resource(investigation_schema) - registry = Registry.with_resource(investigation_schema['id'], schema) - validator = Draft4Validator(investigation_schema, registry=registry) + #investigation_schema = json.load(json_fp) + #schema = DRAFT4.create_resource(investigation_schema) + #registry = Registry.with_resource(investigation_schema['id'], schema) + #validator = Draft4Validator(investigation_schema, registry=registry) + #validator.validate(isa_json) + + resources = [] + schemas_dir = Path(SCHEMAS_PATH) + investigation_schema_path = Path(join(SCHEMAS_PATH, INVESTIGATION_SCHEMA)) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT202012.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = investigation_schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "https://json-schema.org/draft/2020-12/schema"} + validator = Draft202012Validator(schema_ref, registry=registry, format_checker=FormatChecker()) validator.validate(isa_json) log.info("Conversion finished") diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index dab30015a..edde2e4ab 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -13,10 +13,11 @@ import os import re from io import StringIO +from pathlib import Path -from jsonschema import Draft4Validator, ValidationError +from jsonschema import Draft4Validator, ValidationError, Draft202012Validator, FormatChecker from referencing import Registry -from referencing.jsonschema import DRAFT4 +from referencing.jsonschema import DRAFT4, DRAFT202012 from isatools.isajson.load import load @@ -545,15 +546,27 @@ def check_utf8(fp): raise SystemError() + def check_isa_schemas(isa_json, investigation_schema_path): """Used for rule 0003 and 4003""" try: with open(investigation_schema_path) as fp: - investigation_schema = json.load(fp) - schema = DRAFT4.create_resource(investigation_schema) - registry = Registry.with_resource(investigation_schema['id'], schema) - validator = Draft4Validator(investigation_schema, registry=registry) + resources = [] + investigation_schema_path = Path(investigation_schema_path) + schemas_dir = investigation_schema_path.parent + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT202012.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = investigation_schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "https://json-schema.org/draft/2020-12/schema"} + validator = Draft202012Validator(schema_ref, registry=registry, format_checker=FormatChecker()) validator.validate(isa_json) + except ValidationError as ve: errors.append({"message": "Invalid JSON against ISA-JSON schemas", "supplemental": str(ve), "code": 3}) log.fatal("(F) The JSON does not validate against the provided ISA-JSON schemas!") diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json index 43a463809..91b97edf5 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Assay JSON Schema", "name": "ISA Assay JSON Schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json index 43689513c..be22d89da 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Comment schema - it corresponds to ISA Comment[] construct", "name" : "ISA Comment schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json index 5914d6cbc..ad430361f 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Data schema", "name" : "ISA Data schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json index b85d1bcf7..8c4fd04d3 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Factor schema", "name": "ISA Factor schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json index 05383d12a..cf646e6f7 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Factor Value schema", "name": "ISA Factor Value schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json index 3b2e1a7f0..50d92a901 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA investigation schema", "name" : "ISA Investigation schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json index c7b566486..933dd108d 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA material attribute schema", "name" : "ISA Material Attribute schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_value_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_value_schema.json index 3f67a82f5..f3336196a 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_value_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_value_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Material Attribute schema", "name" : "ISA Material Attribute schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json index c00f7216d..de0b30169 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Material schema", "name" : "ISA Material schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json index 25c5cd045..7848e6ea7 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA ontology reference schema", "name" : "ISA ontology reference schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_reference_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_reference_schema.json index 5e920666a..6663bb3dd 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_reference_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_reference_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA ontology source reference schema", "name" : "ISA ontology source reference schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json index 99fae9d17..309e2ea97 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Organization schema", "name" : "ISA Organization schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json index b81a3ed89..32fbf86b7 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Person schema", "name" : "ISA Person schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json index f9b63386c..a298ede16 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA process parameter value schema", "name" : "ISA Process Parameter Value schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json index 919ccda3d..476869120 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA process or protocol application schema, corresponds to 'Protocol REF' columns in the study and assay files", "name" : "ISA Process schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json index 74680a630..114accf77 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Protocol Parameter schema", "name" : "ISA Protocol Parameter schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json index ed625dba7..2775631b7 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Protocol schema", "name": "ISA Protocol schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json index c1c7fcdad..dd9495f24 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Publication schema", "name" : "ISA Publication schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json index b83f7b47e..571c1d82b 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Sample schema", "name" : "ISA Sample schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json index 3ba7f804f..b8f59d1f2 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Source schema", "name" : "ISA Source schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json index 884b8e779..93232552c 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Study JSON schema", "name" : "ISA Study JSON schema", From d58851f819a9c988758c34bb44a4f07952f388dc Mon Sep 17 00:00:00 2001 From: oerc0042 Date: Sat, 25 Oct 2025 15:56:19 +0100 Subject: [PATCH 05/15] downgrading requirement on email format attribute --- .../isa_model_version_1_0_schemas/core/person_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json index 32fbf86b7..4bc9e31a8 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json @@ -12,7 +12,7 @@ "lastName" : { "type" : "string"}, "firstName" : { "type" : "string"}, "midInitials" : { "type" : "string" }, - "email" : { "type" : "string", "format" : "email"}, + "email" : { "type" : "string"}, "phone" : { "type": "string"}, "fax" : { "type" : "string" }, "address" : { "type" : "string" }, From 50e7d2a62ea331362509cc22bbb2c66c158d8e6b Mon Sep 17 00:00:00 2001 From: oerc0042 Date: Sat, 25 Oct 2025 16:34:48 +0100 Subject: [PATCH 06/15] ruff linting and checking fixes --- isatools/convert/isatab2cedar.py | 4 ++-- isatools/isajson/validate.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/isatools/convert/isatab2cedar.py b/isatools/convert/isatab2cedar.py index 5ff141241..233bcd8b2 100644 --- a/isatools/convert/isatab2cedar.py +++ b/isatools/convert/isatab2cedar.py @@ -12,10 +12,10 @@ from uuid import uuid4 from jsonschema import Draft4Validator, FormatChecker -from referencing.jsonschema import DRAFT4 +from jsonschema.exceptions import ValidationError from referencing import Registry +from referencing.jsonschema import DRAFT4 -from jsonschema.exceptions import ValidationError from isatools.io import isatab_parser __author__ = "agbeltran" diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index edde2e4ab..b0c9dcab3 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -4,7 +4,6 @@ Don't forget to read the ISA-JSON spec: https://isa-specs.readthedocs.io/en/latest/isajson.html """ - from __future__ import absolute_import import glob @@ -15,7 +14,7 @@ from io import StringIO from pathlib import Path -from jsonschema import Draft4Validator, ValidationError, Draft202012Validator, FormatChecker +from jsonschema import Draft4Validator, Draft202012Validator, FormatChecker, ValidationError from referencing import Registry from referencing.jsonschema import DRAFT4, DRAFT202012 From b1116409a4ea6c18fd946ad744747a3fc3a74b8b Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Mon, 27 Oct 2025 10:37:10 +0000 Subject: [PATCH 07/15] Fixed person_schema.json. --- isatools/isajson/validate.py | 4 ++-- .../isa_model_version_1_0_schemas/core/person_schema.json | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index b0c9dcab3..c4232e4bb 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -14,9 +14,9 @@ from io import StringIO from pathlib import Path -from jsonschema import Draft4Validator, Draft202012Validator, FormatChecker, ValidationError +from jsonschema import Draft202012Validator, FormatChecker, ValidationError from referencing import Registry -from referencing.jsonschema import DRAFT4, DRAFT202012 +from referencing.jsonschema import DRAFT202012 from isatools.isajson.load import load diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json index 4bc9e31a8..7a928a4d1 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json @@ -12,7 +12,13 @@ "lastName" : { "type" : "string"}, "firstName" : { "type" : "string"}, "midInitials" : { "type" : "string" }, - "email" : { "type" : "string"}, + "email" : { + "anyOf": [ + { "type" : "string", "format": "email" }, + { "type" : "string", "maxLength": 0 }, + { "type": "null" } + ] + }, "phone" : { "type": "string"}, "fax" : { "type" : "string" }, "address" : { "type" : "string" }, From c61629b6ffea28baa0d69ff9893de4ff3f94b76a Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Mon, 27 Oct 2025 11:25:04 +0000 Subject: [PATCH 08/15] Fixed test_validate_test_data.py --- isatools/isajson/validate.py | 1 - isatools/utils.py | 1 - tests/validators/test_validate_test_data.py | 21 ++++++++++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index c4232e4bb..1f6e9a573 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -561,7 +561,6 @@ def check_isa_schemas(isa_json, investigation_schema_path): registry = Registry().with_resources(resources) main_uri = investigation_schema_path.resolve().as_uri() - print(registry.contents(main_uri)) schema_ref = {"$ref": main_uri, "$schema": "https://json-schema.org/draft/2020-12/schema"} validator = Draft202012Validator(schema_ref, registry=registry, format_checker=FormatChecker()) validator.validate(isa_json) diff --git a/isatools/utils.py b/isatools/utils.py index 45da0e013..a4e0c28aa 100644 --- a/isatools/utils.py +++ b/isatools/utils.py @@ -14,7 +14,6 @@ from zipfile import ZipFile import pandas as pd -import yaml from isatools import isatab from isatools.model import Process diff --git a/tests/validators/test_validate_test_data.py b/tests/validators/test_validate_test_data.py index 1a330b46e..f965a103b 100644 --- a/tests/validators/test_validate_test_data.py +++ b/tests/validators/test_validate_test_data.py @@ -4,7 +4,9 @@ import pathlib import unittest -from jsonschema import Draft4Validator, RefResolver +from jsonschema import Draft4Validator, RefResolver, FormatChecker +from referencing.jsonschema import DRAFT4 +from referencing import Registry from isatools import isajson, isatab from isatools.tests import utils @@ -356,13 +358,22 @@ def test_validate_testdata_sampleassayplan_json(self): with open(os.path.join(self.v2_create_schemas_path, "sample_assay_plan_schema.json")) as fp: sample_assay_plan_schema = json.load(fp) + resources = [] res_path = pathlib.Path( "file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json" - ).as_uri() - resolver = RefResolver(res_path, sample_assay_plan_schema) + ).resolve().as_uri() + schemas_dir = pathlib.Path("file://", self.v2_create_schemas_path) + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + schema_ref = {"$ref": res_path, "$schema": "https://json-schema.org/draft-04/schema"} + + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) + validator.validate(json.load(test_case_fp)) - validator = Draft4Validator(sample_assay_plan_schema, resolver=resolver) - validator.validate(json.load(test_case_fp)) def test_validate_testdata_sampleassayplan_qc_json(self): with open(os.path.join(utils.JSON_DATA_DIR, "create", "sampleassayplan_qc_test.json")) as test_case_fp: From b896ddcccc219b6875781d1c5f1b93d16b80c06c Mon Sep 17 00:00:00 2001 From: oerc0042 Date: Mon, 27 Oct 2025 11:36:20 +0000 Subject: [PATCH 09/15] fixing jsonschema resolver outstanding issues and addressing dependabot security issues --- isatools/isajson/validate.py | 6 +- isatools/net/storage_adapter.py | 25 +++++-- pyproject.toml | 6 +- tests/validators/test_validate_test_data.py | 74 +++++++++++++++++---- uv.lock | 6 +- 5 files changed, 91 insertions(+), 26 deletions(-) diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index b0c9dcab3..36fe06d92 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -14,9 +14,9 @@ from io import StringIO from pathlib import Path -from jsonschema import Draft4Validator, Draft202012Validator, FormatChecker, ValidationError +from jsonschema import Draft202012Validator, FormatChecker, ValidationError from referencing import Registry -from referencing.jsonschema import DRAFT4, DRAFT202012 +from referencing.jsonschema import DRAFT202012 from isatools.isajson.load import load @@ -209,7 +209,7 @@ def check_process_protocol_ids_usage(study_json): ) -def get_study_protocols_parameter_ids(study_json): +gdef get_study_protocols_parameter_ids(study_json): """Used for rule 1009""" return [ elem diff --git a/isatools/net/storage_adapter.py b/isatools/net/storage_adapter.py index 19f9b4fb1..6e1d50c20 100644 --- a/isatools/net/storage_adapter.py +++ b/isatools/net/storage_adapter.py @@ -5,15 +5,18 @@ import json import logging import os -import pathlib from abc import ABCMeta, abstractmethod from io import BytesIO, StringIO +from os.path import join +from pathlib import Path from urllib.parse import urljoin from zipfile import ZipFile import requests -from jsonschema import Draft4Validator, RefResolver +from jsonschema import Draft4Validator, FormatChecker # RefResolver from lxml import etree +from referencing import Registry +from referencing.jsonschema import DRAFT4 log = logging.getLogger("isatools") @@ -58,8 +61,22 @@ def validate_json_against_schema(json_dict, schema_src): """ with open(schema_src) as schema_file: schema = json.load(schema_file) - resolver = RefResolver(pathlib.Path(os.path.abspath(schema_src)).as_uri(), schema) - validator = Draft4Validator(schema, resolver=resolver) + resources = [] + schemas_dir = Path(schema_src) + investigation_schema_path = Path(join(schema_src, schema_file)) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = investigation_schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) + + # validator = Draft4Validator(schema) return validator.validate(json_dict, schema) diff --git a/pyproject.toml b/pyproject.toml index 8858c32ae..b40b7f30d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "pandas>=2.2.3,<3", "openpyxl>=3.1.5", "lxml~=6.0.2", - "requests~=2.32.3", + "requests~=2.32.4", "iso8601~=2.1.0", "chardet~=5.2.0", "jinja2~=3.1.4", @@ -51,7 +51,7 @@ dependencies = [ "PyYAML~=6.0.2", "rdflib~=7.2.1", "python-dateutil~=2.9.0.post0", - "Flask>=3.1.0", + "Flask>=3.1.1", "flask_sqlalchemy>=3.0.2", "SQLAlchemy~=2.0.31", #1.4.54 "graphene~=3.4.3", @@ -86,7 +86,7 @@ dev = [ "coveralls>=4.0.1", "sure>=2.0.1", "ddt>=1.7.2", - "deepdiff>=8.4.2", + "deepdiff>=8.6.1", "behave>=1.2.6", "httpretty>=1.1.4", "import-linter>=2.5.2", diff --git a/tests/validators/test_validate_test_data.py b/tests/validators/test_validate_test_data.py index 1a330b46e..bb2d62842 100644 --- a/tests/validators/test_validate_test_data.py +++ b/tests/validators/test_validate_test_data.py @@ -3,8 +3,12 @@ import os import pathlib import unittest +from os.path import join +from pathlib import Path -from jsonschema import Draft4Validator, RefResolver +from jsonschema import Draft4Validator, FormatChecker # RefResolver +from referencing import Registry +from referencing.jsonschema import DRAFT4 from isatools import isajson, isatab from isatools.tests import utils @@ -356,12 +360,25 @@ def test_validate_testdata_sampleassayplan_json(self): with open(os.path.join(self.v2_create_schemas_path, "sample_assay_plan_schema.json")) as fp: sample_assay_plan_schema = json.load(fp) - res_path = pathlib.Path( - "file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json" - ).as_uri() - resolver = RefResolver(res_path, sample_assay_plan_schema) - - validator = Draft4Validator(sample_assay_plan_schema, resolver=resolver) + # res_path = pathlib.Path( + # "file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json" + # ).as_uri() + # resolver = RefResolver(res_path, sample_assay_plan_schema) + resources = [] + schemas_dir = Path("file://", self.v2_create_schemas_path) + schema_path = Path(join(schemas_dir, "sample_assay_plan_schema.json")) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) + # validator = Draft4Validator(sample_assay_plan_schema, resolver=resolver) validator.validate(json.load(test_case_fp)) def test_validate_testdata_sampleassayplan_qc_json(self): @@ -372,18 +389,49 @@ def test_validate_testdata_sampleassayplan_qc_json(self): # os.path.join(self.v2_create_schemas_path, # 'sample_assay_plan_schema.json')), # sample_assay_plan_schema) - res_path = str(pathlib.Path("file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json")) - resolver = RefResolver(res_path, sample_assay_plan_schema) - validator = Draft4Validator(sample_assay_plan_schema, resolver=resolver) + # res_path = str(pathlib.Path("file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json")) + # resolver = RefResolver(res_path, sample_assay_plan_schema) + + resources = [] + schemas_dir = Path("file://", self.v2_create_schemas_path) + schema_path = Path(join(schemas_dir, "sample_assay_plan_schema.json")) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} + + # validator = Draft4Validator(sample_assay_plan_schema, resolver=resolver) + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) validator.validate(json.load(test_case_fp)) def test_validate_testdata_treatment_sequence_json(self): with open(os.path.join(utils.JSON_DATA_DIR, "create", "treatment_sequence_test.json")) as test_case_fp: with open(os.path.join(self.v2_create_schemas_path, "treatment_sequence_schema.json")) as fp: treatment_sequence_schema = json.load(fp) - res_path = pathlib.Path("file://", self.v2_create_schemas_path, "treatment_sequence_schema.json").as_uri() - resolver = RefResolver(res_path, treatment_sequence_schema) - validator = Draft4Validator(treatment_sequence_schema, resolver=resolver) + #res_path = pathlib.Path("file://", self.v2_create_schemas_path, "treatment_sequence_schema.json").as_uri() + # resolver = RefResolver(res_path, treatment_sequence_schema) + # validator = Draft4Validator(treatment_sequence_schema, resolver=resolver) + resources = [] + schemas_dir = Path("file://", self.v2_create_schemas_path) + schema_path = Path(join(schemas_dir, "treatment_sequence_schema.json")) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) + validator.validate(json.load(test_case_fp)) diff --git a/uv.lock b/uv.lock index d36b93e82..20397fbd1 100644 --- a/uv.lock +++ b/uv.lock @@ -1323,7 +1323,7 @@ requires-dist = [ { name = "biopython", specifier = ">=1.85,<1.86" }, { name = "bokeh", marker = "extra == 'notebook'", specifier = "~=3.4.2" }, { name = "chardet", specifier = "~=5.2.0" }, - { name = "flask", specifier = ">=3.1.0" }, + { name = "flask", specifier = ">=3.1.1" }, { name = "flask-sqlalchemy", specifier = ">=3.0.2" }, { name = "graphene", specifier = "~=3.4.3" }, { name = "graphql-core", specifier = "~=3.2.6" }, @@ -1343,7 +1343,7 @@ requires-dist = [ { name = "python-dateutil", specifier = "~=2.9.0.post0" }, { name = "pyyaml", specifier = "~=6.0.2" }, { name = "rdflib", specifier = "~=7.2.1" }, - { name = "requests", specifier = "~=2.32.3" }, + { name = "requests", specifier = "~=2.32.4" }, { name = "ruff", specifier = ">=0.14.1" }, { name = "setuptools", specifier = ">=77.0.3,<81" }, { name = "sqlalchemy", specifier = "~=2.0.31" }, @@ -1355,7 +1355,7 @@ dev = [ { name = "behave", specifier = ">=1.2.6" }, { name = "coveralls", specifier = ">=4.0.1" }, { name = "ddt", specifier = ">=1.7.2" }, - { name = "deepdiff", specifier = ">=8.4.2" }, + { name = "deepdiff", specifier = ">=8.6.1" }, { name = "httpretty", specifier = ">=1.1.4" }, { name = "import-linter", specifier = ">=2.5.2" }, { name = "pre-commit", specifier = ">=4.0.1,<5" }, From fb38486d21ee96cbebe36cc772ec2490208e0223 Mon Sep 17 00:00:00 2001 From: oerc0042 Date: Mon, 27 Oct 2025 11:43:07 +0000 Subject: [PATCH 10/15] sorting impports --- isatools/isajson/validate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index 36fe06d92..e86b70980 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -12,13 +12,13 @@ import os import re from io import StringIO -from pathlib import Path +from isatools.isajson.load import load from jsonschema import Draft202012Validator, FormatChecker, ValidationError +from pathlib import Path from referencing import Registry from referencing.jsonschema import DRAFT202012 -from isatools.isajson.load import load __author__ = "djcomlab@gmail.com (David Johnson)" @@ -209,7 +209,7 @@ def check_process_protocol_ids_usage(study_json): ) -gdef get_study_protocols_parameter_ids(study_json): +def get_study_protocols_parameter_ids(study_json): """Used for rule 1009""" return [ elem From 2a0f222159eae6081d0ada24e1ebd28532ca89eb Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Mon, 27 Oct 2025 11:45:39 +0000 Subject: [PATCH 11/15] Ruff formatting. --- isatools/convert/isatab2json.py | 10 +++++----- isatools/isajson/validate.py | 2 +- tests/validators/test_validate_test_data.py | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/isatools/convert/isatab2json.py b/isatools/convert/isatab2json.py index 61f065755..c770e60e1 100644 --- a/isatools/convert/isatab2json.py +++ b/isatools/convert/isatab2json.py @@ -142,11 +142,11 @@ def convert(self, work_dir): # validate json with open(join(SCHEMAS_PATH, INVESTIGATION_SCHEMA)) as json_fp: - #investigation_schema = json.load(json_fp) - #schema = DRAFT4.create_resource(investigation_schema) - #registry = Registry.with_resource(investigation_schema['id'], schema) - #validator = Draft4Validator(investigation_schema, registry=registry) - #validator.validate(isa_json) + # investigation_schema = json.load(json_fp) + # schema = DRAFT4.create_resource(investigation_schema) + # registry = Registry.with_resource(investigation_schema['id'], schema) + # validator = Draft4Validator(investigation_schema, registry=registry) + # validator.validate(isa_json) resources = [] schemas_dir = Path(SCHEMAS_PATH) diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index 1f6e9a573..4ca152f80 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -4,6 +4,7 @@ Don't forget to read the ISA-JSON spec: https://isa-specs.readthedocs.io/en/latest/isajson.html """ + from __future__ import absolute_import import glob @@ -545,7 +546,6 @@ def check_utf8(fp): raise SystemError() - def check_isa_schemas(isa_json, investigation_schema_path): """Used for rule 0003 and 4003""" try: diff --git a/tests/validators/test_validate_test_data.py b/tests/validators/test_validate_test_data.py index f965a103b..f3c265b5b 100644 --- a/tests/validators/test_validate_test_data.py +++ b/tests/validators/test_validate_test_data.py @@ -359,9 +359,11 @@ def test_validate_testdata_sampleassayplan_json(self): sample_assay_plan_schema = json.load(fp) resources = [] - res_path = pathlib.Path( - "file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json" - ).resolve().as_uri() + res_path = ( + pathlib.Path("file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json") + .resolve() + .as_uri() + ) schemas_dir = pathlib.Path("file://", self.v2_create_schemas_path) for p in sorted(schemas_dir.glob("*.json")): contents = json.loads(p.read_text(encoding="utf-8")) @@ -374,7 +376,6 @@ def test_validate_testdata_sampleassayplan_json(self): validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) validator.validate(json.load(test_case_fp)) - def test_validate_testdata_sampleassayplan_qc_json(self): with open(os.path.join(utils.JSON_DATA_DIR, "create", "sampleassayplan_qc_test.json")) as test_case_fp: with open(os.path.join(self.v2_create_schemas_path, "sample_assay_plan_schema.json")) as fp: From 69e7e397d0d28360bf1ba654cd9031df5c2a9cdf Mon Sep 17 00:00:00 2001 From: oerc0042 Date: Mon, 27 Oct 2025 11:54:35 +0000 Subject: [PATCH 12/15] uv linting error fixing --- isatools/isajson/validate.py | 3 ++- tests/validators/test_validate_test_data.py | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index 0cc994432..1f6e9a573 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -12,12 +12,13 @@ import os import re from io import StringIO -from isatools.isajson.load import load from pathlib import Path + from jsonschema import Draft202012Validator, FormatChecker, ValidationError from referencing import Registry from referencing.jsonschema import DRAFT202012 +from isatools.isajson.load import load __author__ = "djcomlab@gmail.com (David Johnson)" diff --git a/tests/validators/test_validate_test_data.py b/tests/validators/test_validate_test_data.py index 154e0eaca..9239f56ff 100644 --- a/tests/validators/test_validate_test_data.py +++ b/tests/validators/test_validate_test_data.py @@ -6,10 +6,7 @@ from os.path import join from pathlib import Path -from jsonschema import Draft4Validator, RefResolver, FormatChecker -from referencing.jsonschema import DRAFT4 -from referencing import Registry -from jsonschema import Draft4Validator, FormatChecker # RefResolver +from jsonschema import Draft4Validator, FormatChecker, RefResolver from referencing import Registry from referencing.jsonschema import DRAFT4 From 5eb1bde6bcbc28611ea4a4bd56e99be7d5602f84 Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Mon, 27 Oct 2025 13:05:24 +0000 Subject: [PATCH 13/15] Forced ruff to sort import block. --- tests/validators/test_validate_test_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/validators/test_validate_test_data.py b/tests/validators/test_validate_test_data.py index f3c265b5b..839a77a15 100644 --- a/tests/validators/test_validate_test_data.py +++ b/tests/validators/test_validate_test_data.py @@ -4,9 +4,9 @@ import pathlib import unittest -from jsonschema import Draft4Validator, RefResolver, FormatChecker -from referencing.jsonschema import DRAFT4 +from jsonschema import Draft4Validator, FormatChecker, RefResolver from referencing import Registry +from referencing.jsonschema import DRAFT4 from isatools import isajson, isatab from isatools.tests import utils From cc15896a4d23e35f48d7eaa28d93ecf2978bc46c Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Mon, 27 Oct 2025 13:56:59 +0000 Subject: [PATCH 14/15] Skipped two tests. --- isatools/net/ols.py | 5 +++++ tests/utils/test_isatools_utils.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/isatools/net/ols.py b/isatools/net/ols.py index dc447c834..4f3afa3ed 100644 --- a/isatools/net/ols.py +++ b/isatools/net/ols.py @@ -39,6 +39,7 @@ def get_ols_ontologies(): file=file, ) ) + return ontology_sources @@ -47,6 +48,7 @@ def get_ols_ontology(ontology_name): ontologiesUri = OLS_API_BASE_URI + "/ontologies?size=" + str(OLS_PAGINATION_SIZE) log.debug(ontologiesUri) J = json.loads(urlopen(ontologiesUri).read().decode("utf-8")) + print("EMBEDDED: ", J["_embedded"]["ontologies"]) ontology_sources = [] for ontology_source_json in J["_embedded"]["ontologies"]: ontology_sources.append( @@ -57,7 +59,10 @@ def get_ols_ontology(ontology_name): file=ontology_source_json["_links"]["self"]["href"], ) ) + print("NAME: ", ontology_name) + print("SOURCES: ", [o.name for o in ontology_sources]) hits = [o for o in ontology_sources if o.name == ontology_name] + print("HITS: ", hits) if len(hits) == 1: return hits[0] return None diff --git a/tests/utils/test_isatools_utils.py b/tests/utils/test_isatools_utils.py index 69d4b18b7..1b9366458 100644 --- a/tests/utils/test_isatools_utils.py +++ b/tests/utils/test_isatools_utils.py @@ -139,6 +139,7 @@ def test_get_ontologies(self): self.assertIsInstance(ontology_sources, list) self.assertIsInstance(ontology_sources[0], OntologySource) + @unittest.skip("efo is not available from https://www.ebi.ac.uk/ols4/api/ontologies") def test_get_ontology(self): ontology_source = ols.get_ols_ontology("efo") self.assertIsInstance(ontology_source, OntologySource) @@ -148,6 +149,7 @@ def test_get_ontology(self): self.assertIsInstance(ontology_source.version, str) self.assertEqual(ontology_source.description, "Experimental Factor Ontology") + @unittest.skip("efo is not available from https://www.ebi.ac.uk/ols4/api/ontologies") def test_search_for_term(self): ontology_source = ols.get_ols_ontology("efo") ontology_annotations = ols.search_ols("cell type", ontology_source) From 8767bae57d5a240ecdb41d82b265dc092b7a936f Mon Sep 17 00:00:00 2001 From: Milo Thurston Date: Mon, 27 Oct 2025 17:00:29 +0000 Subject: [PATCH 15/15] Code cleanup. --- tests/validators/test_validate_test_data.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/validators/test_validate_test_data.py b/tests/validators/test_validate_test_data.py index d2f7429d7..4033afe33 100644 --- a/tests/validators/test_validate_test_data.py +++ b/tests/validators/test_validate_test_data.py @@ -6,7 +6,7 @@ from os.path import join from pathlib import Path -from jsonschema import Draft4Validator, FormatChecker, RefResolver +from jsonschema import Draft4Validator, FormatChecker from referencing import Registry from referencing.jsonschema import DRAFT4 @@ -382,12 +382,6 @@ def test_validate_testdata_sampleassayplan_qc_json(self): with open(os.path.join(utils.JSON_DATA_DIR, "create", "sampleassayplan_qc_test.json")) as test_case_fp: with open(os.path.join(self.v2_create_schemas_path, "sample_assay_plan_schema.json")) as fp: sample_assay_plan_schema = json.load(fp) - # resolver = RefResolver('file://{}'.format( - # os.path.join(self.v2_create_schemas_path, - # 'sample_assay_plan_schema.json')), - # sample_assay_plan_schema) - # res_path = str(pathlib.Path("file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json")) - # resolver = RefResolver(res_path, sample_assay_plan_schema) resources = [] schemas_dir = Path("file://", self.v2_create_schemas_path) @@ -403,7 +397,6 @@ def test_validate_testdata_sampleassayplan_qc_json(self): print(registry.contents(main_uri)) schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} - # validator = Draft4Validator(sample_assay_plan_schema, resolver=resolver) validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) validator.validate(json.load(test_case_fp))