diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4caaa77 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +.env +.coverage +.gitignore +.idea +.mypy_cache +.ruff_cache +.vscode +.git +.pytest_cache +.DS_Store +*.yml +Dockerfile +**/__pycache__ +.hypothesis +.venv diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f5ae9ed --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: main + +on: + push: + branches: + - main + pull_request: {} + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: extractions/setup-just@v2 + - uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: "**/pyproject.toml" + - run: uv python install 3.10 + - run: just install lint-ci + + pytest: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + steps: + - name: install libmagic + run: sudo apt install -y libmagic-dev + - uses: actions/checkout@v4 + - uses: extractions/setup-just@v2 + - uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: "**/pyproject.toml" + - run: uv python install ${{ matrix.python-version }} + - run: just install + - run: just test-ci . --cov=. --cov-report xml + - uses: codecov/codecov-action@v4.0.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + files: ./coverage.xml + flags: unittests + name: codecov-${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..b637272 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,17 @@ +name: Publish Package + +on: + release: + types: + - published + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: extractions/setup-just@v2 + - uses: astral-sh/setup-uv@v3 + - run: just publish + env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml deleted file mode 100644 index 894d465..0000000 --- a/.github/workflows/workflow.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: CI Pipeline - -on: - push: - branches: - - main - pull_request: - branches: - - main - release: - types: - - published - -jobs: - ci: - uses: community-of-python/community-workflow/.github/workflows/preset.yml@main - with: - python-version: '["3.10","3.11","3.12","3.13"]' - os: '["ubuntu-latest"]' - secrets: inherit diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0040201 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.13-slim + +# required for python-magic +RUN apt update \ + && apt install -y --no-install-recommends \ + libmagic-dev \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv +RUN useradd --no-create-home --gid root runner + +ENV UV_PYTHON_PREFERENCE=only-system +ENV UV_NO_CACHE=true + +WORKDIR /code + +COPY pyproject.toml . +COPY uv.lock . + +RUN uv sync --all-extras --frozen --no-install-project + +COPY . . + +RUN chown -R runner:root /code && chmod -R g=u /code + +USER runner diff --git a/Justfile b/Justfile index 51148c6..51176a7 100644 --- a/Justfile +++ b/Justfile @@ -1,5 +1,17 @@ default: install lint test +down: + docker compose down --remove-orphans + +sh: + docker compose run --service-ports application bash + +test *args: down && down + docker compose run application uv run --no-sync pytest {{ args }} + +build: + docker compose build application + install: uv lock --upgrade uv sync --frozen --all-groups @@ -16,7 +28,7 @@ lint-ci: uv run --group lint ruff check --no-fix uv run --group lint mypy . -test *args: +test-ci *args: uv run pytest {{ args }} publish: diff --git a/README.md b/README.md index f306662..373d899 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # safe-s3-storage -S3 tools for uploading files to S3 safely (antivirus check, etc) as well as downloading and deleting files. +S3 tools for uploading files to S3 safely (antivirus check, etc.) as well as downloading and deleting files. ## How To Use diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1dfe0d8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +services: + application: + build: + context: . + dockerfile: ./Dockerfile + restart: always + volumes: + - .:/code + - /code/.venv + stdin_open: true + tty: true diff --git a/pyproject.toml b/pyproject.toml index 23a6c61..0efa1be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ keywords = ["s3", "kaspersky", "antivirus", "upload"] classifiers = [ "Natural Language :: English", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Software Development :: Libraries", @@ -22,12 +23,22 @@ dependencies = [ "pydantic", "pyvips", "pyvips-binary", - "magika", + "python-magic", ] [dependency-groups] -dev = ["anyio", "faker", "pytest", "pytest-cov"] -lint = [{ include-group = "dev" }, "auto-typing-final", "mypy", "ruff"] +dev = [ + "anyio", + "faker", + "pytest", + "pytest-cov", +] +lint = [ + { include-group = "dev" }, + "auto-typing-final", + "mypy", + "ruff", +] [build-system] requires = ["uv_build"] diff --git a/safe_s3_storage/file_validator.py b/safe_s3_storage/file_validator.py index 3a6af6c..16172ff 100644 --- a/safe_s3_storage/file_validator.py +++ b/safe_s3_storage/file_validator.py @@ -2,8 +2,8 @@ import enum import typing +import magic import pyvips # type: ignore[import-untyped] -from magika import Magika from safe_s3_storage import exceptions from safe_s3_storage.kaspersky_scan_engine import KasperskyScanEngineClient @@ -49,13 +49,7 @@ class FileValidator: excluded_conversion_formats: list[str] | None = None def _validate_mime_type(self, *, file_name: str, file_content: bytes) -> str: - mime_type_prediction: typing.Final = Magika().identify_bytes(file_content) - if mime_type_prediction.output.is_text and file_name.endswith(".txt"): - mime_type = "text/plain" - elif mime_type_prediction.dl.extensions: - mime_type = mime_type_prediction.dl.mime_type - else: - mime_type = mime_type_prediction.output.mime_type + mime_type: typing.Final = magic.from_buffer(file_content, mime=True) if self.allowed_mime_types is None or mime_type in self.allowed_mime_types: return mime_type diff --git a/tests/test_file_validator.py b/tests/test_file_validator.py index 804e51a..4505ce4 100644 --- a/tests/test_file_validator.py +++ b/tests/test_file_validator.py @@ -129,7 +129,7 @@ async def test_ok_image( == _IMAGE_CONVERSION_FORMAT_TO_MIME_TYPE_AND_EXTENSION_MAP[image_conversion_format][0] ) - @pytest.mark.parametrize("file_content", ["'", "test'", "abracadabra", "python script"]) + @pytest.mark.parametrize("file_content", ["test'", "abracadabra", "python script"]) async def test_txt_file_validate(self, file_content: str) -> None: file_name: typing.Final = "file_name.txt"