From a4c9d2fcec7d49e5b91aa18f94bf0f58935699c3 Mon Sep 17 00:00:00 2001 From: Salman Mashayekh Date: Tue, 22 Apr 2025 16:12:47 -0700 Subject: [PATCH 01/12] Refactor boolean conversion in audit and health conventions Replaced usage of distutils.util.strtobool with a custom str_to_bool function in audit.py and health.py for improved clarity and consistency. Updated landing.py to utilize importlib.metadata for package metadata retrieval. --- microcosm_flask/audit.py | 7 +++---- microcosm_flask/conventions/health.py | 4 ++-- microcosm_flask/conventions/landing.py | 13 ++++--------- microcosm_flask/converters.py | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/microcosm_flask/audit.py b/microcosm_flask/audit.py index 925f6f3f..d289c202 100644 --- a/microcosm_flask/audit.py +++ b/microcosm_flask/audit.py @@ -4,17 +4,16 @@ """ from collections import namedtuple from contextlib import contextmanager -from distutils.util import strtobool from functools import wraps from json import loads from logging import DEBUG, getLogger from traceback import format_exc from uuid import UUID - from flask import current_app, g, request from inflection import underscore from microcosm.api import defaults, typed from microcosm.config.types import boolean +from microcosm_flask.converters import str_to_bool from microcosm_logging.timing import elapsed_time from microcosm_flask.errors import ( @@ -63,7 +62,7 @@ def should_skip_logging(func): Should we skip logging for this handler? """ - disabled = strtobool(request.headers.get("x-request-nolog", "false")) + disabled = str_to_bool(request.headers.get("x-request-nolog", "false")) return disabled or getattr(func, SKIP_LOGGING, False) @@ -75,7 +74,7 @@ def logging_levels(): Supports setting per-request debug logging using the `X-Request-Debug` header. """ - enabled = strtobool(request.headers.get("x-request-debug", "false")) + enabled = str_to_bool(request.headers.get("x-request-debug", "false")) level = None try: if enabled: diff --git a/microcosm_flask/conventions/health.py b/microcosm_flask/conventions/health.py index fda118f7..dc59616c 100644 --- a/microcosm_flask/conventions/health.py +++ b/microcosm_flask/conventions/health.py @@ -5,7 +5,6 @@ using HTTP 200/503 status codes to indicate healthiness. """ -from distutils.util import strtobool from functools import wraps from itertools import chain from logging import Logger @@ -18,6 +17,7 @@ from microcosm_flask.conventions.base import Convention from microcosm_flask.conventions.build_info import BuildInfo from microcosm_flask.conventions.encoding import load_query_string_data, make_response +from microcosm_flask.converters import str_to_bool from microcosm_flask.errors import extract_error_message from microcosm_flask.namespaces import Namespace from microcosm_flask.operations import Operation @@ -158,7 +158,7 @@ def configure_health(graph): subject=Health, ) - include_build_info = strtobool(graph.config.health_convention.include_build_info) + include_build_info = str_to_bool(graph.config.health_convention.include_build_info) convention = HealthConvention(graph, include_build_info) convention.configure(ns, retrieve=tuple()) return convention.health diff --git a/microcosm_flask/conventions/landing.py b/microcosm_flask/conventions/landing.py index 256d12dc..d1d835a3 100644 --- a/microcosm_flask/conventions/landing.py +++ b/microcosm_flask/conventions/landing.py @@ -2,10 +2,8 @@ Landing Page convention. """ -from distutils import dist -from io import StringIO from json import dumps -from pkg_resources import DistributionNotFound, get_distribution +from importlib.metadata import metadata, PackageNotFoundError from jinja2 import Template @@ -29,12 +27,9 @@ def get_properties_and_version(): """ try: - distribution = get_distribution(graph.metadata.name) - metadata_str = distribution.get_metadata(distribution.PKG_INFO) - package_info = dist.DistributionMetadata() - package_info.read_pkg_file(StringIO(metadata_str)) - return package_info - except DistributionNotFound: + package_metadata = metadata(graph.metadata.name) + return package_metadata + except PackageNotFoundError: return None def get_swagger_versions(): diff --git a/microcosm_flask/converters.py b/microcosm_flask/converters.py index 7ab70289..708c42d7 100644 --- a/microcosm_flask/converters.py +++ b/microcosm_flask/converters.py @@ -11,3 +11,17 @@ def configure_uuid(graph): """ return FlaskUUID(graph.flask) + + +def str_to_bool(value): + """ + Convert a string value to boolean. + Similar to distutils.util.strtobool but returns a boolean instead of an int. + """ + if isinstance(value, bool): + return value + if str(value).lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif str(value).lower() in ('no', 'false', 'f', 'n', '0'): + return False + raise ValueError(f"Invalid boolean value: {value}") \ No newline at end of file From 751c20534c53b4009b627cf54b250684e85438ce Mon Sep 17 00:00:00 2001 From: Salman Mashayekh Date: Tue, 22 Apr 2025 16:34:33 -0700 Subject: [PATCH 02/12] update str_to_bool to match return type --- microcosm_flask/converters.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/microcosm_flask/converters.py b/microcosm_flask/converters.py index 708c42d7..b58dc10a 100644 --- a/microcosm_flask/converters.py +++ b/microcosm_flask/converters.py @@ -16,12 +16,10 @@ def configure_uuid(graph): def str_to_bool(value): """ Convert a string value to boolean. - Similar to distutils.util.strtobool but returns a boolean instead of an int. + Similar to distutils.util.strtobool, it returns an int. """ - if isinstance(value, bool): - return value if str(value).lower() in ('yes', 'true', 't', 'y', '1'): - return True + return 1 elif str(value).lower() in ('no', 'false', 'f', 'n', '0'): - return False + return 0 raise ValueError(f"Invalid boolean value: {value}") \ No newline at end of file From 6bb70444eaf566603aecb67daca7aa5f37b9835b Mon Sep 17 00:00:00 2001 From: Salman Mashayekh Date: Tue, 22 Apr 2025 16:51:59 -0700 Subject: [PATCH 03/12] update build files --- .drone.yml | 4 ++-- .globality/build.json | 5 ++++- Dockerfile | 4 ++-- entrypoint.sh | 16 ++++++++++------ 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.drone.yml b/.drone.yml index a83e0e0d..7a8eba09 100644 --- a/.drone.yml +++ b/.drone.yml @@ -79,7 +79,7 @@ steps: - sonar-scanner commands: - pip install -U pip==24.0 - - pip install --quiet awscli twine==4.0.2 packaging==24.0 importlib-metadata==4.8.1 + - pip install --quiet awscli twine==4.0.2 packaging==24.0 - export version=$(cat .bumpversion.cfg | awk '/current_version / {print $3}') - aws codeartifact login --tool pip --repository globality-pypi-local --domain globality --domain-owner $AWS_ACCOUNT_ID --region us-east-1 - python setup.py sdist bdist_wheel @@ -99,7 +99,7 @@ steps: TWINE_REPOSITORY: https://upload.pypi.org/legacy/ commands: - pip install -U pip==24.0 - - pip install --quiet awscli twine==4.0.2 importlib-metadata==4.8.1 + - pip install --quiet awscli twine==4.0.2 - export version=$(cat .bumpversion.cfg | awk '/current_version / {print $3}') - echo "Publishing ${version}" - python setup.py sdist bdist_wheel diff --git a/.globality/build.json b/.globality/build.json index 9c24fa42..9af3c881 100644 --- a/.globality/build.json +++ b/.globality/build.json @@ -1,10 +1,13 @@ { "params": { + "docker": { + "docker_tag": "python:3.11-slim-buster" + }, "name": "microcosm-flask", "pypi": { "repository": "pypi" } }, "type": "python-library", - "version": "2024.52.0" + "version": "2024.54.0" } diff --git a/Dockerfile b/Dockerfile index e82cedf5..43f86401 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ FROM python:3.11-slim-buster as deps ARG EXTRA_INDEX_URL ENV EXTRA_INDEX_URL ${EXTRA_INDEX_URL} -ENV CORE_PACKAGES locales libpq-dev +ENV CORE_PACKAGES locales ENV BUILD_PACKAGES build-essential libffi-dev ENV OTHER_PACKAGES libssl-dev @@ -108,4 +108,4 @@ ARG SHA1 ENV MICROCOSM_FLASK__BUILD_INFO_CONVENTION__BUILD_NUM ${BUILD_NUM} ENV MICROCOSM_FLASK__BUILD_INFO_CONVENTION__SHA1 ${SHA1} COPY $NAME /src/$NAME/ -RUN pip install --no-cache-dir --extra-index-url "${EXTRA_INDEX_URL}" -e . +RUN pip install --no-cache-dir --extra-index-url $EXTRA_INDEX_URL -e . diff --git a/entrypoint.sh b/entrypoint.sh index 50a0adc0..afda096d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -24,15 +24,19 @@ if [ "$1" = "test" ]; then - pip --quiet install .\[test\] - pip --quiet install . - pytest ${NAME} + # Install standard test dependencies; YMMV + pip --quiet install \ + .[test] pytest pytest-cov PyHamcrest + pytest elif [ "$1" = "lint" ]; then - pip --quiet install .\[lint\] + # Install standard linting dependencies; YMMV + pip --quiet install \ + .[lint] flake8 ${NAME} elif [ "$1" = "typehinting" ]; then - pip --quiet install .\[typehinting\] - mypy ${NAME} + # Install standard type-linting dependencies + pip --quiet install mypy + exec mypy ${NAME} --ignore-missing-imports else echo "Cannot execute $@" exit 3 From f0f416eb017158e5d141581f8adea469b24f076c Mon Sep 17 00:00:00 2001 From: Salman Mashayekh Date: Wed, 23 Apr 2025 09:19:36 -0700 Subject: [PATCH 04/12] empty commit From c8f3f3140922a54f608a241692fe17529dfb1dd0 Mon Sep 17 00:00:00 2001 From: Salman Mashayekh Date: Wed, 23 Apr 2025 17:01:34 -0700 Subject: [PATCH 05/12] lint --- microcosm_flask/audit.py | 3 ++- microcosm_flask/conventions/landing.py | 2 +- microcosm_flask/converters.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/microcosm_flask/audit.py b/microcosm_flask/audit.py index d289c202..f165bed4 100644 --- a/microcosm_flask/audit.py +++ b/microcosm_flask/audit.py @@ -9,13 +9,14 @@ from logging import DEBUG, getLogger from traceback import format_exc from uuid import UUID + from flask import current_app, g, request from inflection import underscore from microcosm.api import defaults, typed from microcosm.config.types import boolean -from microcosm_flask.converters import str_to_bool from microcosm_logging.timing import elapsed_time +from microcosm_flask.converters import str_to_bool from microcosm_flask.errors import ( extract_context, extract_error_message, diff --git a/microcosm_flask/conventions/landing.py b/microcosm_flask/conventions/landing.py index d1d835a3..e91dad5d 100644 --- a/microcosm_flask/conventions/landing.py +++ b/microcosm_flask/conventions/landing.py @@ -2,8 +2,8 @@ Landing Page convention. """ +from importlib.metadata import PackageNotFoundError, metadata from json import dumps -from importlib.metadata import metadata, PackageNotFoundError from jinja2 import Template diff --git a/microcosm_flask/converters.py b/microcosm_flask/converters.py index b58dc10a..a2ead014 100644 --- a/microcosm_flask/converters.py +++ b/microcosm_flask/converters.py @@ -22,4 +22,4 @@ def str_to_bool(value): return 1 elif str(value).lower() in ('no', 'false', 'f', 'n', '0'): return 0 - raise ValueError(f"Invalid boolean value: {value}") \ No newline at end of file + raise ValueError(f"Invalid boolean value: {value}") From f3a89a380dc1c273e98fb8f909560f9be611833b Mon Sep 17 00:00:00 2001 From: Salman Mashayekh Date: Wed, 23 Apr 2025 17:16:26 -0700 Subject: [PATCH 06/12] change `default` to `dump_default`) --- microcosm_flask/errors.py | 6 +++--- microcosm_flask/tests/swagger/parameters/test_default.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/microcosm_flask/errors.py b/microcosm_flask/errors.py index c99b5946..5fdead92 100644 --- a/microcosm_flask/errors.py +++ b/microcosm_flask/errors.py @@ -22,9 +22,9 @@ class ErrorContextSchema(Schema): class ErrorSchema(Schema): - message = fields.String(required=True, default="Unknown Error") - code = fields.Integer(required=True, default=500) - retryable = fields.Boolean(required=True, default=False) + message = fields.String(required=True, dump_default="Unknown Error") + code = fields.Integer(required=True, dump_default=500) + retryable = fields.Boolean(required=True, dump_default=False) context = fields.Nested(ErrorContextSchema, required=False) # type: ignore diff --git a/microcosm_flask/tests/swagger/parameters/test_default.py b/microcosm_flask/tests/swagger/parameters/test_default.py index 89b0c624..6e0d7740 100644 --- a/microcosm_flask/tests/swagger/parameters/test_default.py +++ b/microcosm_flask/tests/swagger/parameters/test_default.py @@ -6,7 +6,7 @@ class FooSchema(Schema): id = fields.UUID() - foo = fields.String(metadata={"description": "Foo"}, default="bar") + foo = fields.String(metadata={"description": "Foo"}, dump_default="bar") payload = fields.Dict() datetime = fields.DateTime() From 84d21ae4e8d05e4308b77606677f0b55f10dad0b Mon Sep 17 00:00:00 2001 From: adamhadani <139888+adamhadani@users.noreply.github.com> Date: Wed, 23 Apr 2025 19:33:44 -0500 Subject: [PATCH 07/12] add missing types-setuptools and types-simplejson deps for mypy tests --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index afda096d..dea826c2 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -35,7 +35,7 @@ elif [ "$1" = "lint" ]; then flake8 ${NAME} elif [ "$1" = "typehinting" ]; then # Install standard type-linting dependencies - pip --quiet install mypy + pip --quiet install mypy types-setuptools types-simplejson exec mypy ${NAME} --ignore-missing-imports else echo "Cannot execute $@" From 1d7c31edec0ea83e6c77e248dc08062a2f5d67af Mon Sep 17 00:00:00 2001 From: adamhadani <139888+adamhadani@users.noreply.github.com> Date: Wed, 23 Apr 2025 19:38:13 -0500 Subject: [PATCH 08/12] add types-date-util to entrypoint.sh mypy --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index dea826c2..af7010b6 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -35,7 +35,7 @@ elif [ "$1" = "lint" ]; then flake8 ${NAME} elif [ "$1" = "typehinting" ]; then # Install standard type-linting dependencies - pip --quiet install mypy types-setuptools types-simplejson + pip --quiet install mypy types-setuptools types-simplejson types-python-dateutil exec mypy ${NAME} --ignore-missing-imports else echo "Cannot execute $@" From f3e0d4abb8ec9b2da20a21380074f542d4bd3574 Mon Sep 17 00:00:00 2001 From: adamhadani <139888+adamhadani@users.noreply.github.com> Date: Wed, 23 Apr 2025 20:39:33 -0500 Subject: [PATCH 09/12] remove use of pkg_resources in favor of importlib, remove reference to types-setuptools now not needed in entrypoint.sh mypy entrypoint --- entrypoint.sh | 2 +- microcosm_flask/swagger/parameters/__init__.py | 4 ++-- setup.cfg | 1 - setup.py | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index af7010b6..5fc81c76 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -35,7 +35,7 @@ elif [ "$1" = "lint" ]; then flake8 ${NAME} elif [ "$1" = "typehinting" ]; then # Install standard type-linting dependencies - pip --quiet install mypy types-setuptools types-simplejson types-python-dateutil + pip --quiet install mypy types-simplejson types-python-dateutil exec mypy ${NAME} --ignore-missing-imports else echo "Cannot execute $@" diff --git a/microcosm_flask/swagger/parameters/__init__.py b/microcosm_flask/swagger/parameters/__init__.py index 1d7c0f48..54f90186 100644 --- a/microcosm_flask/swagger/parameters/__init__.py +++ b/microcosm_flask/swagger/parameters/__init__.py @@ -1,6 +1,6 @@ from collections.abc import Mapping from functools import lru_cache -from pkg_resources import iter_entry_points +from importlib.metadata import entry_points from typing import Any from marshmallow.fields import Field @@ -54,7 +54,7 @@ def builder_types(cls) -> list[type[ParameterBuilder]]: Define the available builder types. """ - return [entry_point.load() for entry_point in iter_entry_points(ENTRY_POINT)] + return [entry_point.load() for entry_point in entry_points(ENTRY_POINT)] @classmethod def default_builder_type(cls) -> type[ParameterBuilder]: diff --git a/setup.cfg b/setup.cfg index 6c7e15ec..ea1ddb18 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,7 +9,6 @@ force_grid_wrap = 4 float_to_top = True include_trailing_comma = True known_first_party = microcosm_flask -extra_standard_library = pkg_resources line_length = 99 lines_after_imports = 2 multi_line_output = 3 diff --git a/setup.py b/setup.py index 4ef9691a..1bad0cd7 100644 --- a/setup.py +++ b/setup.py @@ -53,9 +53,9 @@ "lint": [ "mypy", "flake8", - "flake8-print", - "flake8-logging-format>=1.0.0", "flake8-isort", + "flake8-logging-format>=1.0.0", + "flake8-print", "types-python-dateutil", "types-setuptools", ], From 64a04825b1e8df0ed3fc75166d1a7953f7beec00 Mon Sep 17 00:00:00 2001 From: adamhadani <139888+adamhadani@users.noreply.github.com> Date: Wed, 23 Apr 2025 20:51:21 -0500 Subject: [PATCH 10/12] ignore remaining mypy errors --- microcosm_flask/cloning.py | 3 ++- microcosm_flask/swagger/parameters/__init__.py | 2 +- microcosm_flask/tests/swagger/parameters/test_constant.py | 7 ++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/microcosm_flask/cloning.py b/microcosm_flask/cloning.py index 24d0b331..596195df 100644 --- a/microcosm_flask/cloning.py +++ b/microcosm_flask/cloning.py @@ -50,8 +50,9 @@ class DAGSchema(Schema): Nodes should be overridden with a non-raw schema. """ + # Nb. using fields.Raw inside fields.Nested trips up mypy. documentation doesnt clarify. nodes = fields.Nested( - fields.Raw, + fields.Raw, # type: ignore[arg-type] required=True, attribute="nodes_map", ) diff --git a/microcosm_flask/swagger/parameters/__init__.py b/microcosm_flask/swagger/parameters/__init__.py index 54f90186..727861c8 100644 --- a/microcosm_flask/swagger/parameters/__init__.py +++ b/microcosm_flask/swagger/parameters/__init__.py @@ -54,7 +54,7 @@ def builder_types(cls) -> list[type[ParameterBuilder]]: Define the available builder types. """ - return [entry_point.load() for entry_point in entry_points(ENTRY_POINT)] + return [entry_point.load() for entry_point in entry_points(group=ENTRY_POINT)] @classmethod def default_builder_type(cls) -> type[ParameterBuilder]: diff --git a/microcosm_flask/tests/swagger/parameters/test_constant.py b/microcosm_flask/tests/swagger/parameters/test_constant.py index 81ffec27..164ab586 100644 --- a/microcosm_flask/tests/swagger/parameters/test_constant.py +++ b/microcosm_flask/tests/swagger/parameters/test_constant.py @@ -5,9 +5,10 @@ class FooSchema(Schema): - deprecated_constant_list = fields.Constant(constant=[], dump_only=True) - deprecated_constant_string = fields.Constant(constant="HELLO", dump_only=True) - deprecated_constant_int = fields.Constant(constant=123, dump_only=True) + # Nb. mypy wants type annotations for these fields, unclear what those would be. Disable checks below. + deprecated_constant_list = fields.Constant(constant=[], dump_only=True) # type: ignore[var-annotated] + deprecated_constant_string = fields.Constant(constant="HELLO", dump_only=True) # type: ignore[var-annotated] + deprecated_constant_int = fields.Constant(constant=123, dump_only=True) # type: ignore[var-annotated] def test_field_constant_list(): From f4b211a6d6bfa6f11f3e8b9cb5c6d306dca33db5 Mon Sep 17 00:00:00 2001 From: adamhadani <139888+adamhadani@users.noreply.github.com> Date: Wed, 23 Apr 2025 21:13:00 -0500 Subject: [PATCH 11/12] update gitignore file to more modern template --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1740f054..e480c812 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ nosetests.xml coverage.xml *,cover cover +junit.xml # Translations *.mo @@ -98,10 +99,12 @@ target/ .mypy_cache -/coverage/ +# direnv +.envrc # Python Virtual Environments venv/ +.venv/ /coverate/ **/coverage/**/* From 962fc09515655897281d409c67431f86b763496c Mon Sep 17 00:00:00 2001 From: adamhadani <139888+adamhadani@users.noreply.github.com> Date: Wed, 23 Apr 2025 21:18:43 -0500 Subject: [PATCH 12/12] bump min required microcosm-logging version to 2.1.0 in anticipation of that release which fixes remaining failures --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1bad0cd7..d8629187 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "jsonschema>=3.2.0", "marshmallow>=3.0.0", "microcosm>=4.0.0", - "microcosm-logging>=2.0.0", + "microcosm-logging>=2.1.0", "openapi>=2.0.0", "python-dateutil>=2.7.3", "PyYAML>=3.13",