diff --git a/docs/cli-reference/general-options.md b/docs/cli-reference/general-options.md index 8aadbdbfc..92568220c 100644 --- a/docs/cli-reference/general-options.md +++ b/docs/cli-reference/general-options.md @@ -13,6 +13,7 @@ | [`--generate-pyproject-config`](#generate-pyproject-config) | Generate pyproject.toml configuration from CLI arguments. | | [`--http-headers`](#http-headers) | Fetch schema from URL with custom HTTP headers. | | [`--http-ignore-tls`](#http-ignore-tls) | Disable TLS certificate verification for HTTPS requests. | +| [`--http-local-ref-path`](#http-local-ref-path) | Resolve HTTP references from local schema files. | | [`--http-query-parameters`](#http-query-parameters) | Add query parameters to HTTP requests for remote schemas. | | [`--http-timeout`](#http-timeout) | Set timeout for HTTP requests to remote hosts. | | [`--ignore-pyproject`](#ignore-pyproject) | Ignore pyproject.toml configuration file. | @@ -1631,6 +1632,59 @@ environments with self-signed certificates. Not recommended for production. --- +## `--http-local-ref-path` {#http-local-ref-path} + +Resolve HTTP references from local schema files. + +The `--http-local-ref-path` flag maps HTTP(S) `$ref` URLs to files under +a local schema store instead of fetching them from the network. The host and +URL path are used as the relative path under the schema store. For example, +`https://api.example.com/schemas/pet.json` is read from +`schemas/api.example.com/schemas/pet.json`. + +!!! tip "Usage" + + ```bash + datamodel-codegen --input schema.json --url https://api.example.com/schema.json --http-local-ref-path schemas # (1)! + ``` + + 1. :material-arrow-left: `--http-local-ref-path` - the option documented here + +??? example "Examples" + + **Input Schema:** + + ```json + { + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "https://api.example.com/schemas/pet.json" + } + ``` + + **Output:** + + ```python + # generated by datamodel-codegen: + # filename: https://api.example.com/schema.json + # timestamp: 2019-07-26T00:00:00+00:00 + + from __future__ import annotations + + from pydantic import BaseModel, RootModel + + + class Pet(BaseModel): + id: int | None = None + name: str | None = None + tag: str | None = None + + + class Model(RootModel[Pet]): + root: Pet + ``` + +--- + ## `--http-query-parameters` {#http-query-parameters} Add query parameters to HTTP requests for remote schemas. diff --git a/docs/cli-reference/index.md b/docs/cli-reference/index.md index d481179dc..2e2e487dc 100644 --- a/docs/cli-reference/index.md +++ b/docs/cli-reference/index.md @@ -15,7 +15,7 @@ This documentation is auto-generated from test cases. | 🎨 [Template Customization](template-customization.md) | 21 | Output formatting and custom rendering | | 📘 [OpenAPI-only Options](openapi-only-options.md) | 7 | OpenAPI-specific features | | 📋 [GraphQL-only Options](graphql-only-options.md) | 1 | | -| ⚙️ [General Options](general-options.md) | 16 | Utilities and meta options | +| ⚙️ [General Options](general-options.md) | 17 | Utilities and meta options | | 📝 [Utility Options](utility-options.md) | 6 | Help, version, debug options | ## All Options @@ -105,6 +105,7 @@ This documentation is auto-generated from test cases. - [`--help`](utility-options.md#help) - [`--http-headers`](general-options.md#http-headers) - [`--http-ignore-tls`](general-options.md#http-ignore-tls) +- [`--http-local-ref-path`](general-options.md#http-local-ref-path) - [`--http-query-parameters`](general-options.md#http-query-parameters) - [`--http-timeout`](general-options.md#http-timeout) diff --git a/docs/cli-reference/model-customization.md b/docs/cli-reference/model-customization.md index 51bc94d1b..9aba2b115 100644 --- a/docs/cli-reference/model-customization.md +++ b/docs/cli-reference/model-customization.md @@ -5272,6 +5272,10 @@ Use default values from schema in generated models. The `--use-default` flag allows required fields with default values to be generated with their defaults, making them optional to provide when instantiating the model. +When `--strict-nullable` is enabled, the field type still follows the schema's +nullability. For example, a required string field with a default is generated +as `str = 'value'`, not `str | None = 'value'`, unless the schema allows null. + **Related:** [`--strict-nullable`](model-customization.md#strict-nullable) !!! tip "Usage" @@ -5282,21 +5286,6 @@ with their defaults, making them optional to provide when instantiating the mode 1. :material-arrow-left: `--use-default` - the option documented here -!!! warning "Fields with defaults become nullable" - When using `--use-default`, fields with default values are generated as nullable - types (e.g., `str | None` instead of `str`), even if the schema does not allow - null values. - - If you want fields to strictly follow the schema's type definition (non-nullable), - use `--strict-nullable` together with `--use-default`. - - -!!! note "Future behavior change" - In a future major version, the default behavior of `--use-default` may change to - generate non-nullable types that match the schema definition (equivalent to using - `--strict-nullable`). If you rely on the current nullable behavior, consider - explicitly handling this in your code. - ??? example "Examples" **Input Schema:** diff --git a/docs/cli-reference/quick-reference.md b/docs/cli-reference/quick-reference.md index 95acfb3d6..d57030904 100644 --- a/docs/cli-reference/quick-reference.md +++ b/docs/cli-reference/quick-reference.md @@ -191,6 +191,7 @@ datamodel-codegen [OPTIONS] | [`--generate-pyproject-config`](general-options.md#generate-pyproject-config) | Generate pyproject.toml configuration from CLI arguments. | | [`--http-headers`](general-options.md#http-headers) | Fetch schema from URL with custom HTTP headers. | | [`--http-ignore-tls`](general-options.md#http-ignore-tls) | Disable TLS certificate verification for HTTPS requests. | +| [`--http-local-ref-path`](general-options.md#http-local-ref-path) | Resolve HTTP references from local schema files. | | [`--http-query-parameters`](general-options.md#http-query-parameters) | Add query parameters to HTTP requests for remote schemas. | | [`--http-timeout`](general-options.md#http-timeout) | Set timeout for HTTP requests to remote hosts. | | [`--ignore-pyproject`](general-options.md#ignore-pyproject) | Ignore pyproject.toml configuration file. | @@ -275,6 +276,7 @@ All options sorted alphabetically: - [`--help`](utility-options.md#help) - Show help message and exit - [`--http-headers`](general-options.md#http-headers) - Fetch schema from URL with custom HTTP headers. - [`--http-ignore-tls`](general-options.md#http-ignore-tls) - Disable TLS certificate verification for HTTPS requests. +- [`--http-local-ref-path`](general-options.md#http-local-ref-path) - Resolve HTTP references from local schema files. - [`--http-query-parameters`](general-options.md#http-query-parameters) - Add query parameters to HTTP requests for remote schemas. - [`--http-timeout`](general-options.md#http-timeout) - Set timeout for HTTP requests to remote hosts. - [`--ignore-enum-constraints`](typing-customization.md#ignore-enum-constraints) - Ignore enum constraints and use base string type instead of ... diff --git a/src/datamodel_code_generator/__main__.py b/src/datamodel_code_generator/__main__.py index e7225ea13..bd0f29dc4 100644 --- a/src/datamodel_code_generator/__main__.py +++ b/src/datamodel_code_generator/__main__.py @@ -177,6 +177,7 @@ def validate_file(cls, value: Any) -> TextIOBase | None: # noqa: N805 "output", "custom_template_dir", "custom_file_header_path", + "http_local_ref_path", mode="before", ) def validate_path(cls, value: Any) -> Path | None: # noqa: N805 @@ -493,6 +494,7 @@ def validate_class_name_affix_scope(cls, v: str | ClassNameAffixScope | None) -> allof_class_hierarchy: AllOfClassHierarchy = AllOfClassHierarchy.IfNoConflict allow_remote_refs: Optional[bool] = None # noqa: UP045 http_headers: Optional[Sequence[tuple[str, str]]] = None # noqa: UP045 + http_local_ref_path: Optional[Path] = None # noqa: UP045 http_ignore_tls: bool = False http_timeout: Optional[float] = None # noqa: UP045 use_annotated: bool = False @@ -944,6 +946,7 @@ def run_generate_from_config( # noqa: PLR0913, PLR0917 allof_class_hierarchy=config.allof_class_hierarchy, allow_remote_refs=config.allow_remote_refs, http_headers=config.http_headers, + http_local_ref_path=config.http_local_ref_path, http_ignore_tls=config.http_ignore_tls, http_timeout=config.http_timeout, use_annotated=config.use_annotated, diff --git a/src/datamodel_code_generator/_types/generate_config_dict.py b/src/datamodel_code_generator/_types/generate_config_dict.py index c0c278d10..0e1701d89 100644 --- a/src/datamodel_code_generator/_types/generate_config_dict.py +++ b/src/datamodel_code_generator/_types/generate_config_dict.py @@ -118,6 +118,7 @@ class GenerateConfigDict(TypedDict, closed=True): allof_class_hierarchy: NotRequired[AllOfClassHierarchy] allow_remote_refs: NotRequired[bool | None] http_headers: NotRequired[Sequence[tuple[str, str]] | None] + http_local_ref_path: NotRequired[Path | None] http_ignore_tls: NotRequired[bool] http_timeout: NotRequired[float | None] use_annotated: NotRequired[bool] diff --git a/src/datamodel_code_generator/_types/parser_config_dicts.py b/src/datamodel_code_generator/_types/parser_config_dicts.py index c99912474..56b49e983 100644 --- a/src/datamodel_code_generator/_types/parser_config_dicts.py +++ b/src/datamodel_code_generator/_types/parser_config_dicts.py @@ -113,6 +113,7 @@ class ParserConfigDict(TypedDict): allof_class_hierarchy: NotRequired[AllOfClassHierarchy] allow_remote_refs: NotRequired[bool | None] http_headers: NotRequired[Sequence[tuple[str, str]] | None] + http_local_ref_path: NotRequired[Path | None] http_ignore_tls: NotRequired[bool] http_timeout: NotRequired[float | None] use_annotated: NotRequired[bool] diff --git a/src/datamodel_code_generator/arguments.py b/src/datamodel_code_generator/arguments.py index 7c8f7a7a0..a87dd3b56 100644 --- a/src/datamodel_code_generator/arguments.py +++ b/src/datamodel_code_generator/arguments.py @@ -193,6 +193,13 @@ def start_section(self, heading: str | None) -> None: default=None, help="Timeout in seconds for HTTP requests to remote hosts (default: 30)", ) +base_options.add_argument( + "--http-local-ref-path", + help=( + "Resolve HTTP(S) JSON Schema $ref URLs from a local directory instead of fetching them. " + "URLs are mapped under the directory by host and path; extensionless refs also try '.json'." + ), +) base_options.add_argument( "--input", help="Input file/directory (default: stdin)", diff --git a/src/datamodel_code_generator/cli_options.py b/src/datamodel_code_generator/cli_options.py index ad64860d3..39a8d0889 100644 --- a/src/datamodel_code_generator/cli_options.py +++ b/src/datamodel_code_generator/cli_options.py @@ -268,6 +268,7 @@ class CLIOptionMeta: "--allow-remote-refs": CLIOptionMeta(name="--allow-remote-refs", category=OptionCategory.GENERAL), "--no-allow-remote-refs": CLIOptionMeta(name="--no-allow-remote-refs", category=OptionCategory.GENERAL), "--http-headers": CLIOptionMeta(name="--http-headers", category=OptionCategory.GENERAL), + "--http-local-ref-path": CLIOptionMeta(name="--http-local-ref-path", category=OptionCategory.GENERAL), "--http-ignore-tls": CLIOptionMeta(name="--http-ignore-tls", category=OptionCategory.GENERAL), "--http-query-parameters": CLIOptionMeta(name="--http-query-parameters", category=OptionCategory.GENERAL), "--http-timeout": CLIOptionMeta(name="--http-timeout", category=OptionCategory.GENERAL), diff --git a/src/datamodel_code_generator/config.py b/src/datamodel_code_generator/config.py index 4bd1ed9a3..f8c9fbf12 100644 --- a/src/datamodel_code_generator/config.py +++ b/src/datamodel_code_generator/config.py @@ -143,6 +143,7 @@ class GenerateConfig(BaseModel): allof_class_hierarchy: AllOfClassHierarchy = AllOfClassHierarchy.IfNoConflict allow_remote_refs: bool | None = None http_headers: Sequence[tuple[str, str]] | None = None + http_local_ref_path: Path | None = None http_ignore_tls: bool = False http_timeout: float | None = None use_annotated: bool = False @@ -276,6 +277,7 @@ class ParserConfig(BaseModel): allof_class_hierarchy: AllOfClassHierarchy = AllOfClassHierarchy.IfNoConflict allow_remote_refs: bool | None = None http_headers: Sequence[tuple[str, str]] | None = None + http_local_ref_path: Path | None = None http_ignore_tls: bool = False http_timeout: float | None = None use_annotated: bool = False diff --git a/src/datamodel_code_generator/parser/base.py b/src/datamodel_code_generator/parser/base.py index ada2399b3..b4b293da5 100644 --- a/src/datamodel_code_generator/parser/base.py +++ b/src/datamodel_code_generator/parser/base.py @@ -1104,6 +1104,7 @@ def __init__( # noqa: PLR0912, PLR0915 self.wrap_string_literal: bool | None = config.wrap_string_literal self.allow_remote_refs: bool | None = config.allow_remote_refs self.http_headers: Sequence[tuple[str, str]] | None = config.http_headers + self.http_local_ref_path: Path | None = config.http_local_ref_path self.http_query_parameters: Sequence[tuple[str, str]] | None = config.http_query_parameters self.http_ignore_tls: bool = config.http_ignore_tls self.http_timeout: float | None = config.http_timeout diff --git a/src/datamodel_code_generator/parser/jsonschema.py b/src/datamodel_code_generator/parser/jsonschema.py index d02c0bfd7..e09b48ffb 100644 --- a/src/datamodel_code_generator/parser/jsonschema.py +++ b/src/datamodel_code_generator/parser/jsonschema.py @@ -16,7 +16,7 @@ from functools import cached_property, lru_cache from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union -from urllib.parse import ParseResult, unquote +from urllib.parse import ParseResult, unquote, urlparse from warnings import warn from pydantic import ( @@ -3822,7 +3822,7 @@ def create_enum(reference_: Reference) -> DataType: def _get_ref_body(self, resolved_ref: str) -> dict[str, YamlValue]: """Get the body of a reference from URL or remote file.""" if is_url(resolved_ref): - if not resolved_ref.startswith("file://"): + if not resolved_ref.startswith("file://") and self.http_local_ref_path is None: if self.allow_remote_refs is False: msg = ( f"Fetching remote $ref is disabled: {resolved_ref}\n" @@ -3842,10 +3842,41 @@ def _get_ref_body(self, resolved_ref: str) -> dict[str, YamlValue]: return self._get_ref_body_from_url(resolved_ref) return self._get_ref_body_from_remote(resolved_ref) + def _get_ref_body_from_local_http_path(self, ref: str) -> dict[str, YamlValue]: + assert self.http_local_ref_path is not None + parsed = urlparse(ref) + if parsed.scheme not in {"http", "https"}: # pragma: no cover + msg = f"Unsupported local HTTP $ref URL: {ref}" + raise Error(msg) + + parts = [unquote(part) for part in parsed.path.split("/") if part] + if not parsed.netloc or any(part in {".", ".."} or "/" in part or "\\" in part for part in parts): + msg = f"Unsupported local HTTP $ref URL path: {ref}" + raise Error(msg) + + base_path = self.http_local_ref_path.resolve() + relative_path = Path(parsed.netloc, *parts) + file_paths = [(base_path / relative_path).resolve()] + if not parts or not Path(parts[-1]).suffix: + file_paths.append((base_path / relative_path.with_name(f"{relative_path.name}.json")).resolve()) + + if any(not file_path.is_relative_to(base_path) for file_path in file_paths): + msg = f"Unsupported local HTTP $ref URL path: {ref}" + raise Error(msg) + + for file_path in file_paths: + if file_path.is_file(): + return self.remote_object_cache.get_or_put( + str(file_path), + default_factory=lambda _, file_path=file_path: load_data_from_path(file_path, self.encoding), + ) + + msg = f"$ref local file not found for {ref}: tried {', '.join(str(path) for path in file_paths)}" + raise Error(msg) + def _get_ref_body_from_url(self, ref: str) -> dict[str, YamlValue]: """Get reference body from a URL (HTTP, HTTPS, or file scheme).""" if ref.startswith("file://"): - from urllib.parse import urlparse # noqa: PLC0415 from urllib.request import url2pathname # noqa: PLC0415 parsed = urlparse(ref) @@ -3858,6 +3889,8 @@ def _get_ref_body_from_url(self, ref: str) -> dict[str, YamlValue]: return self.remote_object_cache.get_or_put( ref, default_factory=lambda _: load_data_from_path(file_path, self.encoding) ) + if self.http_local_ref_path is not None and urlparse(ref).scheme in {"http", "https"}: + return self._get_ref_body_from_local_http_path(ref) return self.remote_object_cache.get_or_put( ref, default_factory=lambda key: load_data(self._get_text_from_url(key)) ) diff --git a/src/datamodel_code_generator/prompt_data.py b/src/datamodel_code_generator/prompt_data.py index 21d442f9f..dc7cd6f78 100644 --- a/src/datamodel_code_generator/prompt_data.py +++ b/src/datamodel_code_generator/prompt_data.py @@ -63,6 +63,7 @@ "--graphql-no-typename": "Exclude __typename field from generated GraphQL models.", "--http-headers": "Fetch schema from URL with custom HTTP headers.", "--http-ignore-tls": "Disable TLS certificate verification for HTTPS requests.", + "--http-local-ref-path": "Resolve HTTP references from local schema files.", "--http-query-parameters": "Add query parameters to HTTP requests for remote schemas.", "--http-timeout": "Set timeout for HTTP requests to remote hosts.", "--ignore-enum-constraints": "Ignore enum constraints and use base string type instead of Enum classes.", diff --git a/tests/data/expected/main/help/color.txt b/tests/data/expected/main/help/color.txt index f34ce56d9..bf2afe21a 100644 --- a/tests/data/expected/main/help/color.txt +++ b/tests/data/expected/main/help/color.txt @@ -39,6 +39,11 @@ For detailed usage, see: https://datamodel-code-generator.koxudaxi.dev (example: "Authorization: Basic dXNlcjpwYXNz") --http-ignore-tls Disable verification of the remote host's TLS certificate + --http-local-ref-path HTTP_LOCAL_REF_PATH + Resolve HTTP(S) JSON Schema $ref URLs from a local + directory instead of fetching them. URLs are mapped + under the directory by host and path; extensionless + refs also try '.json'. --http-query-parameters HTTP_QUERY_PARAMETERS [HTTP_QUERY_PARAMETERS ...] Set query parameters in HTTP requests to the remote host. (example: "ref=branch") diff --git a/tests/data/expected/main/help/no_color.txt b/tests/data/expected/main/help/no_color.txt index d31bbb398..59b4ffd77 100644 --- a/tests/data/expected/main/help/no_color.txt +++ b/tests/data/expected/main/help/no_color.txt @@ -39,6 +39,11 @@ Options: (example: "Authorization: Basic dXNlcjpwYXNz") --http-ignore-tls Disable verification of the remote host's TLS certificate + --http-local-ref-path HTTP_LOCAL_REF_PATH + Resolve HTTP(S) JSON Schema $ref URLs from a local + directory instead of fetching them. URLs are mapped + under the directory by host and path; extensionless + refs also try '.json'. --http-query-parameters HTTP_QUERY_PARAMETERS [HTTP_QUERY_PARAMETERS ...] Set query parameters in HTTP requests to the remote host. (example: "ref=branch") diff --git a/tests/data/expected/main/input_model/config_class.py b/tests/data/expected/main/input_model/config_class.py index 02437ce39..696e69a1b 100644 --- a/tests/data/expected/main/input_model/config_class.py +++ b/tests/data/expected/main/input_model/config_class.py @@ -195,6 +195,7 @@ class GenerateConfig(TypedDict, closed=True): allof_class_hierarchy: NotRequired[AllOfClassHierarchy] allow_remote_refs: NotRequired[bool | None] http_headers: NotRequired[Sequence[tuple[str, str]] | None] + http_local_ref_path: NotRequired[str | None] http_ignore_tls: NotRequired[bool] http_timeout: NotRequired[float | None] use_annotated: NotRequired[bool] diff --git a/tests/data/expected/main_kr/generate_prompt/basic.txt b/tests/data/expected/main_kr/generate_prompt/basic.txt index 90ebc7f90..3cbe390f2 100644 --- a/tests/data/expected/main_kr/generate_prompt/basic.txt +++ b/tests/data/expected/main_kr/generate_prompt/basic.txt @@ -162,6 +162,7 @@ - `--generate-pyproject-config`: Generate pyproject.toml configuration from CLI arguments. - `--http-headers`: Fetch schema from URL with custom HTTP headers. - `--http-ignore-tls`: Disable TLS certificate verification for HTTPS requests. +- `--http-local-ref-path`: Resolve HTTP references from local schema files. - `--http-query-parameters`: Add query parameters to HTTP requests for remote schemas. - `--http-timeout`: Set timeout for HTTP requests to remote hosts. - `--ignore-pyproject`: Ignore pyproject.toml configuration file. @@ -215,6 +216,11 @@ For detailed usage, see: https://datamodel-code-generator.koxudaxi.dev (example: "Authorization: Basic dXNlcjpwYXNz") --http-ignore-tls Disable verification of the remote host's TLS certificate + --http-local-ref-path HTTP_LOCAL_REF_PATH + Resolve HTTP(S) JSON Schema $ref URLs from a local + directory instead of fetching them. URLs are mapped + under the directory by host and path; extensionless + refs also try '.json'. --http-query-parameters HTTP_QUERY_PARAMETERS [HTTP_QUERY_PARAMETERS ...] Set query parameters in HTTP requests to the remote host. (example: "ref=branch") diff --git a/tests/data/expected/main_kr/generate_prompt/with_list_options.txt b/tests/data/expected/main_kr/generate_prompt/with_list_options.txt index 820550c3a..789988f82 100644 --- a/tests/data/expected/main_kr/generate_prompt/with_list_options.txt +++ b/tests/data/expected/main_kr/generate_prompt/with_list_options.txt @@ -162,6 +162,7 @@ - `--generate-pyproject-config`: Generate pyproject.toml configuration from CLI arguments. - `--http-headers`: Fetch schema from URL with custom HTTP headers. - `--http-ignore-tls`: Disable TLS certificate verification for HTTPS requests. +- `--http-local-ref-path`: Resolve HTTP references from local schema files. - `--http-query-parameters`: Add query parameters to HTTP requests for remote schemas. - `--http-timeout`: Set timeout for HTTP requests to remote hosts. - `--ignore-pyproject`: Ignore pyproject.toml configuration file. @@ -215,6 +216,11 @@ For detailed usage, see: https://datamodel-code-generator.koxudaxi.dev (example: "Authorization: Basic dXNlcjpwYXNz") --http-ignore-tls Disable verification of the remote host's TLS certificate + --http-local-ref-path HTTP_LOCAL_REF_PATH + Resolve HTTP(S) JSON Schema $ref URLs from a local + directory instead of fetching them. URLs are mapped + under the directory by host and path; extensionless + refs also try '.json'. --http-query-parameters HTTP_QUERY_PARAMETERS [HTTP_QUERY_PARAMETERS ...] Set query parameters in HTTP requests to the remote host. (example: "ref=branch") diff --git a/tests/data/expected/main_kr/generate_prompt/with_options.txt b/tests/data/expected/main_kr/generate_prompt/with_options.txt index 01f4ffb63..25176c6b9 100644 --- a/tests/data/expected/main_kr/generate_prompt/with_options.txt +++ b/tests/data/expected/main_kr/generate_prompt/with_options.txt @@ -168,6 +168,7 @@ What other options should I use? - `--generate-pyproject-config`: Generate pyproject.toml configuration from CLI arguments. - `--http-headers`: Fetch schema from URL with custom HTTP headers. - `--http-ignore-tls`: Disable TLS certificate verification for HTTPS requests. +- `--http-local-ref-path`: Resolve HTTP references from local schema files. - `--http-query-parameters`: Add query parameters to HTTP requests for remote schemas. - `--http-timeout`: Set timeout for HTTP requests to remote hosts. - `--ignore-pyproject`: Ignore pyproject.toml configuration file. @@ -221,6 +222,11 @@ For detailed usage, see: https://datamodel-code-generator.koxudaxi.dev (example: "Authorization: Basic dXNlcjpwYXNz") --http-ignore-tls Disable verification of the remote host's TLS certificate + --http-local-ref-path HTTP_LOCAL_REF_PATH + Resolve HTTP(S) JSON Schema $ref URLs from a local + directory instead of fetching them. URLs are mapped + under the directory by host and path; extensionless + refs also try '.json'. --http-query-parameters HTTP_QUERY_PARAMETERS [HTTP_QUERY_PARAMETERS ...] Set query parameters in HTTP requests to the remote host. (example: "ref=branch") diff --git a/tests/data/expected/main_kr/generate_prompt/with_question.txt b/tests/data/expected/main_kr/generate_prompt/with_question.txt index d6285a325..6b8c0db45 100644 --- a/tests/data/expected/main_kr/generate_prompt/with_question.txt +++ b/tests/data/expected/main_kr/generate_prompt/with_question.txt @@ -166,6 +166,7 @@ How do I convert enums to Literal types? - `--generate-pyproject-config`: Generate pyproject.toml configuration from CLI arguments. - `--http-headers`: Fetch schema from URL with custom HTTP headers. - `--http-ignore-tls`: Disable TLS certificate verification for HTTPS requests. +- `--http-local-ref-path`: Resolve HTTP references from local schema files. - `--http-query-parameters`: Add query parameters to HTTP requests for remote schemas. - `--http-timeout`: Set timeout for HTTP requests to remote hosts. - `--ignore-pyproject`: Ignore pyproject.toml configuration file. @@ -219,6 +220,11 @@ For detailed usage, see: https://datamodel-code-generator.koxudaxi.dev (example: "Authorization: Basic dXNlcjpwYXNz") --http-ignore-tls Disable verification of the remote host's TLS certificate + --http-local-ref-path HTTP_LOCAL_REF_PATH + Resolve HTTP(S) JSON Schema $ref URLs from a local + directory instead of fetching them. URLs are mapped + under the directory by host and path; extensionless + refs also try '.json'. --http-query-parameters HTTP_QUERY_PARAMETERS [HTTP_QUERY_PARAMETERS ...] Set query parameters in HTTP requests to the remote host. (example: "ref=branch") diff --git a/tests/data/expected/main_kr/help_shows_new_options.txt b/tests/data/expected/main_kr/help_shows_new_options.txt index f34ce56d9..bf2afe21a 100644 --- a/tests/data/expected/main_kr/help_shows_new_options.txt +++ b/tests/data/expected/main_kr/help_shows_new_options.txt @@ -39,6 +39,11 @@ For detailed usage, see: https://datamodel-code-generator.koxudaxi.dev (example: "Authorization: Basic dXNlcjpwYXNz") --http-ignore-tls Disable verification of the remote host's TLS certificate + --http-local-ref-path HTTP_LOCAL_REF_PATH + Resolve HTTP(S) JSON Schema $ref URLs from a local + directory instead of fetching them. URLs are mapped + under the directory by host and path; extensionless + refs also try '.json'. --http-query-parameters HTTP_QUERY_PARAMETERS [HTTP_QUERY_PARAMETERS ...] Set query parameters in HTTP requests to the remote host. (example: "ref=branch") diff --git a/tests/data/expected/main_kr/http_local_ref_path/output.py b/tests/data/expected/main_kr/http_local_ref_path/output.py new file mode 100644 index 000000000..03e4c60a5 --- /dev/null +++ b/tests/data/expected/main_kr/http_local_ref_path/output.py @@ -0,0 +1,17 @@ +# generated by datamodel-codegen: +# filename: https://api.example.com/schema.json +# timestamp: 2019-07-26T00:00:00+00:00 + +from __future__ import annotations + +from pydantic import BaseModel, RootModel + + +class Pet(BaseModel): + id: int | None = None + name: str | None = None + tag: str | None = None + + +class Model(RootModel[Pet]): + root: Pet diff --git a/tests/data/jsonschema/http_local_ref_path_root.json b/tests/data/jsonschema/http_local_ref_path_root.json new file mode 100644 index 000000000..17fd77c9d --- /dev/null +++ b/tests/data/jsonschema/http_local_ref_path_root.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "https://api.example.com/schemas/pet.json" +} diff --git a/tests/main/jsonschema/test_main_jsonschema.py b/tests/main/jsonschema/test_main_jsonschema.py index e18f48209..6328e23dd 100644 --- a/tests/main/jsonschema/test_main_jsonschema.py +++ b/tests/main/jsonschema/test_main_jsonschema.py @@ -510,9 +510,9 @@ def test_main_null_and_array(output_file: Path) -> None: The `--use-default` flag allows required fields with default values to be generated with their defaults, making them optional to provide when instantiating the model. - The field type still follows the schema's nullability. For example, a required - string field with a default is generated as `str = 'value'`, not - `str | None = 'value'`, unless the schema allows null.""", +When `--strict-nullable` is enabled, the field type still follows the schema's +nullability. For example, a required string field with a default is generated +as `str = 'value'`, not `str | None = 'value'`, unless the schema allows null.""", input_schema="jsonschema/use_default_with_const.json", cli_args=["--output-model-type", "pydantic_v2.BaseModel", "--use-default"], golden_output="jsonschema/use_default_with_const.py", @@ -524,9 +524,9 @@ def test_use_default_pydantic_v2_with_json_schema_const(output_file: Path) -> No The `--use-default` flag allows required fields with default values to be generated with their defaults, making them optional to provide when instantiating the model. - The field type still follows the schema's nullability. For example, a required - string field with a default is generated as `str = 'value'`, not - `str | None = 'value'`, unless the schema allows null. + When `--strict-nullable` is enabled, the field type still follows the schema's + nullability. For example, a required string field with a default is generated + as `str = 'value'`, not `str | None = 'value'`, unless the schema allows null. """ run_main_and_assert( input_path=JSON_SCHEMA_DATA_PATH / "use_default_with_const.json", diff --git a/tests/main/test_public_api_signature_baseline.py b/tests/main/test_public_api_signature_baseline.py index b5511a070..d163b9b09 100644 --- a/tests/main/test_public_api_signature_baseline.py +++ b/tests/main/test_public_api_signature_baseline.py @@ -133,6 +133,7 @@ def _baseline_generate( allof_class_hierarchy: AllOfClassHierarchy = AllOfClassHierarchy.IfNoConflict, allow_remote_refs: bool | None = None, http_headers: Sequence[tuple[str, str]] | None = None, + http_local_ref_path: Path | None = None, http_ignore_tls: bool = False, http_timeout: float | None = None, use_annotated: bool = False, @@ -268,6 +269,7 @@ def __init__( allof_class_hierarchy: AllOfClassHierarchy = AllOfClassHierarchy.IfNoConflict, allow_remote_refs: bool | None = None, http_headers: Sequence[tuple[str, str]] | None = None, + http_local_ref_path: Path | None = None, http_ignore_tls: bool = False, http_timeout: float | None = None, use_annotated: bool = False, diff --git a/tests/parser/test_jsonschema.py b/tests/parser/test_jsonschema.py index 824ab4215..022a7cbb4 100644 --- a/tests/parser/test_jsonschema.py +++ b/tests/parser/test_jsonschema.py @@ -11,7 +11,7 @@ import pytest import yaml -from datamodel_code_generator import AllOfMergeMode +from datamodel_code_generator import AllOfMergeMode, Error from datamodel_code_generator.imports import Import from datamodel_code_generator.model import DataModelFieldBase from datamodel_code_generator.model.dataclass import DataClass @@ -200,6 +200,145 @@ class Pet(BaseModel): ) +def test_json_schema_ref_url_from_local_http_path(tmp_path: Path, mocker: MockerFixture) -> None: + """Test HTTP JSON schema references resolved from a local schema store.""" + schema_store = tmp_path / "schemas" + local_schema = schema_store / "example.com" / "application" / "package" / "element" / "sub-element.json" + local_schema.parent.mkdir(parents=True) + local_schema.write_text( + json.dumps( + { + "$id": "http://example.com/application/package/element/sub-element", + "title": "SubElement", + "type": "object", + "properties": { + "name": { + "type": "string", + }, + }, + }, + ), + encoding="utf-8", + ) + + parser = JsonSchemaParser("", allow_remote_refs=False, http_local_ref_path=schema_store) + mock_get = mocker.patch("httpx.get") + + parser.parse_raw_obj( + "Model", + { + "type": "object", + "properties": { + "sub_element": { + "$ref": "http://example.com/application/package/element/sub-element", + }, + }, + }, + ["Model"], + ) + + assert ( + dump_templates(list(parser.results)) + == """class Model(BaseModel): + sub_element: Optional[SubElement] = None + + +class SubElement(BaseModel): + name: Optional[str] = None""" + ) + mock_get.assert_not_called() + + +def test_json_schema_ref_url_from_local_http_path_with_extension(tmp_path: Path, mocker: MockerFixture) -> None: + """Test HTTP JSON schema references with an extension resolved from a local schema store.""" + schema_store = tmp_path / "schemas" + local_schema = schema_store / "example.com" / "application" / "package" / "element" / "sub-element.json" + local_schema.parent.mkdir(parents=True) + local_schema.write_text( + json.dumps( + { + "title": "SubElement", + "type": "object", + "properties": { + "name": { + "type": "string", + }, + }, + }, + ), + encoding="utf-8", + ) + + parser = JsonSchemaParser("", allow_remote_refs=False, http_local_ref_path=schema_store) + mock_get = mocker.patch("httpx.get") + + assert parser._get_ref_body_from_url("http://example.com/application/package/element/sub-element.json") == { + "title": "SubElement", + "type": "object", + "properties": { + "name": { + "type": "string", + }, + }, + } + mock_get.assert_not_called() + + +@pytest.mark.parametrize( + "ref", + [ + "http:///application/package/element/sub-element", + "http://example.com/application/package/../sub-element", + "http://example.com/..%5C..%5CWindows%5Cwin.ini", + "http://example.com/path%2Fwith-slash", + ], +) +def test_json_schema_ref_url_from_local_http_path_invalid_path(tmp_path: Path, ref: str) -> None: + """Test invalid local HTTP JSON schema reference paths are rejected.""" + parser = JsonSchemaParser("", allow_remote_refs=False, http_local_ref_path=tmp_path) + + with pytest.raises(Error, match="Unsupported local HTTP \\$ref URL path"): + parser._get_ref_body_from_url(ref) + + +def test_json_schema_ref_url_from_local_http_path_missing_file(tmp_path: Path) -> None: + """Test missing local HTTP JSON schema references show the attempted local paths.""" + parser = JsonSchemaParser("", allow_remote_refs=False, http_local_ref_path=tmp_path) + + with pytest.raises(Error, match=r"\$ref local file not found for http://example.com/schema"): + parser._get_ref_body_from_url("http://example.com/schema") + + +def test_json_schema_ref_url_from_local_http_path_ignores_non_http_scheme( + tmp_path: Path, mocker: MockerFixture +) -> None: + """Test local HTTP path resolution does not handle non-HTTP URL schemes.""" + parser = JsonSchemaParser("", http_local_ref_path=tmp_path) + mocker.patch.object(parser, "_get_text_from_url", return_value='{"type": "object"}') + local_http_path = mocker.patch.object(parser, "_get_ref_body_from_local_http_path") + + assert parser._get_ref_body_from_url("ftp://example.com/schema.json") == {"type": "object"} + local_http_path.assert_not_called() + + +def test_json_schema_ref_url_from_local_http_path_symlink_escape(tmp_path: Path) -> None: + """Test local HTTP JSON schema references cannot escape the schema store through symlinks.""" + schema_store = tmp_path / "schemas" + local_schema = schema_store / "example.com" / "schema.json" + local_schema.parent.mkdir(parents=True) + outside_schema = tmp_path / "outside.json" + outside_schema.write_text('{"type": "object"}', encoding="utf-8") + try: + local_schema.symlink_to(outside_schema) + except OSError as exc: # pragma: no cover + pytest.skip(f"symlink creation is not supported: {exc}") + + parser = JsonSchemaParser("", allow_remote_refs=False, http_local_ref_path=schema_store) + + with pytest.raises(Error, match="Unsupported local HTTP \\$ref URL path"): + parser._get_ref_body_from_url("http://example.com/schema.json") + + @pytest.mark.parametrize( ("source_obj", "generated_classes"), [ diff --git a/tests/test_main_kr.py b/tests/test_main_kr.py index 416c52821..d0465cbd5 100644 --- a/tests/test_main_kr.py +++ b/tests/test_main_kr.py @@ -1537,6 +1537,54 @@ def test_url_with_http_headers(mock_httpx_get: HttpxGetMockFactory, output_file: assert_httpx_get_kwargs(mock_get, headers=[("Authorization", "Bearer token")]) +@pytest.mark.cli_doc( + options=["--http-local-ref-path"], + option_description="""Resolve HTTP references from local schema files. + +The `--http-local-ref-path` flag maps HTTP(S) `$ref` URLs to files under +a local schema store instead of fetching them from the network. The host and +URL path are used as the relative path under the schema store. For example, +`https://api.example.com/schemas/pet.json` is read from +`schemas/api.example.com/schemas/pet.json`.""", + input_schema="jsonschema/http_local_ref_path_root.json", + cli_args=[ + "--url", + "https://api.example.com/schema.json", + "--http-local-ref-path", + "schemas", + ], + golden_output="main_kr/http_local_ref_path/output.py", +) +@freeze_time("2019-07-26") +def test_http_local_ref_path_cli_doc(mock_httpx_get: HttpxGetMockFactory, output_file: Path, tmp_path: Path) -> None: + """Resolve HTTP references from local schema files. + + The `--http-local-ref-path` flag maps HTTP(S) `$ref` URLs to files under + a local schema store instead of fetching them from the network. The host and + URL path are used as the relative path under the schema store. + """ + schema_store = tmp_path / "schemas" + local_schema = schema_store / "api.example.com" / "schemas" / "pet.json" + local_schema.parent.mkdir(parents=True) + local_schema.write_text((JSON_SCHEMA_DATA_PATH / "pet_simple.json").read_text(), encoding="utf-8") + mock_get = mock_httpx_get( + MockHttpxResponse( + "https://api.example.com/schema.json", + JSON_SCHEMA_DATA_PATH / "http_local_ref_path_root.json", + ) + ) + + run_main_url_and_assert( + url="https://api.example.com/schema.json", + output_path=output_file, + input_file_type="jsonschema", + assert_func=assert_file_content, + expected_file=EXPECTED_MAIN_KR_PATH / "http_local_ref_path" / "output.py", + extra_args=["--http-local-ref-path", str(schema_store)], + ) + assert_httpx_get_kwargs(mock_get) + + @pytest.mark.cli_doc( options=["--input"], option_description="""Specify the input schema file path.