🎉 Initial MVP (#3)

jerr0328 · sinanislekdemir · ragecryx · web-flow · commit e970b5edf000 · 2024-05-17T16:08:22.000+03:00
* [wip] Setup initial project structure - Add pyproject.toml with ruff and setuptools config - Add schema file with some initial enums - Add initial requirements-dev.txt (not pinned/hashed for now to avoid noise) - Add core audit log fields (type, timestamp, level, version) - Add demo output with `python -m audit_log.log` * Add support for headers parsing * Complete audit logging initial API Co-authored-by: Sinan ISLEKDEMIR <sinan@islekdemir.com> Co-authored-by: George Koutsikos <ragecryx@users.noreply.github.com> * Add pre-commit hooks * Enforce principal by dataclass (#1) * Enforce principal by dataclass * Fix for the broken ruff state and type * Update audit_log/log.py Co-authored-by: Jeremy Mayeres <1524722+jerr0328@users.noreply.github.com> * Add import * Better ruff config * Remove JWT processing (#2) * Remove JWT processing * Rebase from enforce principal * Fix type_ * Change hte logic to JWT first --------- Co-authored-by: Jeremy Mayeres <1524722+jerr0328@users.noreply.github.com> * Wrap exceptions with common base exception * Change version to integer * add unit tests (#4) * add unit tests * fix PR comments * Add case-insensitive support * add workflow for lint and unit tests (#5) * add workflow for lint and unit tests * fix * add pythonpath * add python 3.11 for test job * update workflow * Prepare for public release * Address mypy issue * Add security file and codeowners * Add release workflow * Cleanup, improve comments --------- Co-authored-by: Sinan ISLEKDEMIR <sinan@islekdemir.com> Co-authored-by: George Koutsikos <ragecryx@users.noreply.github.com> Co-authored-by: Tatiana Smirnova <tatiana.smirnova@iceye.fi> Co-authored-by: Sinan Islekdemir <sinan.islekdemir@iceye.fi>
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @iceye-ltd/role-python-logging-maintainer
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+version: 2
+updates:
+  # Enable version updates for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: monday
+      time: "09:00"
+      timezone: "Europe/Helsinki"
+    commit-message:
+      prefix: "⬆️"
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -0,0 +1,18 @@
+---
+name: pre-commit
+
+"on":
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: "3.12"
+      - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,28 @@
+---
+name: Upload Python Package
+
+"on":
+  release:
+    types: [created]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - name: Set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: "3.12"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+      - name: Build
+        run: python -m build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,39 @@
+---
+name: Test
+
+"on":
+  pull_request:
+  push:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+          # Need fetch-depth 0 for generating version based on tags/commits since tag
+          fetch-depth: 0
+
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Deps
+        run: make setup
+
+      - name: Lint
+        run: make lint
+
+      - name: Run tests
+        run: make test
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,28 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_language_version:
+  python: python3
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-json
+      - id: check-yaml
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+      - id: file-contents-sorter
+        files: ^requirements-dev.txt$
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.4.3
+    hooks:
+      - id: ruff
+        args: [ --fix, --exit-non-zero-on-fix ]
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.4.2
+    hooks:
+      - id: black
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Iceye Oy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/Makefile b/Makefile
@@ -0,0 +1,23 @@
+SHELL := bash
+
+PYTHONPATH = .
+
+.PHONY: test
+test:
+	PYTHONPATH=. pytest --cov=audit_log -vv tests
+
+
+.PHONY: fmt
+fmt: ## Format the source code using pre-commit hooks
+	pre-commit run --all-files
+
+
+.PHONY: setup
+setup: ## Install project dependencies from requirements-dev.txt
+	pip install -r requirements-dev.txt
+
+
+.PHONY: lint
+lint:
+	ruff check .
+	mypy audit_log
diff --git a/README.md b/README.md
@@ -1,2 +1,19 @@
 # python-audit-log
+
 Audit logging library for Python
+
+## Usage
+
+`pip install iceye-audit-log`
+
+Then in your applications:
+
+`from audit_log.log import log`
+
+And invoke it with the proper arguments.
+
+## Contributing
+
+This project is mostly for internal use, however it is made public as the concept of audit logging is quite generic.
+We are careful about accepting contributions and those would have to align with our needs.
+Reviews and issues will be considered time-permitting.
diff --git a/SECURITY.md b/SECURITY.md
@@ -0,0 +1,15 @@
+# Security Policy
+
+## Responsible Disclosure
+
+If you think you've found a security vulnerability, please adhere to responsible
+disclosure practices by allowing us a reasonable time to investigate and address
+the issue before making any information public.
+
+## Reporting a Vulnerability
+
+Please report security issues by sending email to: <security_reports@iceye.fi>.
+Try to be as explicit as possible, describing all the steps and example code to
+reproduce the issue.
+
+The ICEYE security team will review the issue and get back to you.
diff --git a/audit_log/__init__.py b/audit_log/__init__.py
diff --git a/audit_log/exceptions.py b/audit_log/exceptions.py
@@ -0,0 +1,6 @@
+class AuditError(Exception):
+    """General error in audit library"""
+
+
+class AuditPrincipalError(AuditError):
+    """Error with the principal"""
diff --git a/audit_log/headers.py b/audit_log/headers.py
@@ -0,0 +1,101 @@
+import re
+from dataclasses import dataclass
+from urllib.parse import urlparse
+
+from audit_log.exceptions import AuditPrincipalError
+
+from .schema import Principal, PrincipalType
+
+MTLS_CERT_HEADER = "x-forwarded-client-cert"
+SPIFFE_PATH_RE = re.compile(r"/ns/(?P<ns>[0-9a-z\-]+)/sa/(?P<sa>[0-9a-z\-]+)/?")
+SUB_HEADER = "x-jwt-claim-sub"
+ISS_HEADER = "x-jwt-claim-iss"
+SUB_TYPE_HEADER = "x-jwt-claim-sub-type"
+
+
+@dataclass
+class ParsedSPIFFE:
+    domain: str
+    namespace: str
+    service_account: str
+    # SPIFFE ID parsed from header
+    spiffe_id: str
+
+
+def parse_spiffe(xfcc_header: str) -> ParsedSPIFFE:
+    """Parse the X-Forwarded-Client-Cert header string and return the namespace, service account, and cluster internal hostname.
+
+    Raises exception if header is invalid.
+
+    Note: When using this with Istio, the ingress gateway will also include a SPIFFE header. However, this does not represent the actual principal.
+
+    Args:
+        xfcc_header (str): X-Forwarded-Client-Cert header contents
+
+    Returns:
+        ParsedSPIFFE: Data parsed from SPIFFE header
+    """
+    try:
+        # Split the header into a dictionary
+        pairs = (pair.split("=") for pair in xfcc_header.split(";"))
+        spiffe_dict = dict(pairs)
+        # Only checking URI for now
+        uri = spiffe_dict["URI"]
+        parsed_uri = urlparse(uri)
+        # Make sure it's a proper SPIFFE URI
+        if parsed_uri.scheme.lower() != "spiffe":
+            raise ValueError("URI scheme must be spiffe://")
+        # Need to get namespace and service account from the URI
+        parsed_path = SPIFFE_PATH_RE.search(parsed_uri.path)
+        # Regex not matching would be returning `None`
+        if not parsed_path:
+            raise ValueError("Could not parse SPIFFE header")
+        parsed_path_dict = parsed_path.groupdict()
+        namespace = parsed_path_dict["ns"]
+        service_account = parsed_path_dict["sa"]
+    except (KeyError, ValueError) as e:
+        raise ValueError("Invalid SPIFFE header") from e
+    else:
+        return ParsedSPIFFE(
+            domain=parsed_uri.netloc,
+            namespace=namespace,
+            service_account=service_account,
+            spiffe_id=uri,
+        )
+
+
+def get_principal_from_headers(
+    headers: dict[str, str],
+) -> Principal:
+    """Get principal from headers, supports mTLS, headers set in Istio, and JWTs.
+
+    Note: Do not use this to handle your auth, it expects auth to already be handled elsewhere and this is just to help get principals.
+
+    Args:
+        headers (dict[str, str]): Headers with all keys lowercase
+
+    Raises:
+        AuditPrincipalError: Cannot get a principal from the headers
+
+    Returns:
+        dict[str, str]: Principal dictionary in proper format
+    """
+    headers = {k.lower(): v for k, v in headers.items()}
+    if all(header in headers for header in (ISS_HEADER, SUB_HEADER, SUB_TYPE_HEADER)):
+        iss = headers[ISS_HEADER]
+        sub = headers[SUB_HEADER]
+        sub_type = headers[SUB_TYPE_HEADER]
+
+        try:
+            return Principal(type=PrincipalType(sub_type), authority=iss, id=sub)
+        except ValueError as e:
+            raise AuditPrincipalError("Invalid JWT headers") from e
+
+    try:
+        spiffe = parse_spiffe(headers[MTLS_CERT_HEADER])
+    except Exception as e:
+        raise AuditPrincipalError("Invalid SPIFFE header") from e
+    else:
+        return Principal(
+            type=PrincipalType.SERVICE, authority=spiffe.domain, id=spiffe.spiffe_id
+        )
diff --git a/audit_log/log.py b/audit_log/log.py
@@ -0,0 +1,76 @@
+import dataclasses
+import functools
+import json
+import uuid
+from collections.abc import Callable
+from contextvars import ContextVar
+from datetime import UTC, datetime
+from functools import singledispatch
+from typing import Any
+
+from audit_log.schema import (
+    SCHEMA_VERSION,
+    ActionType,
+    OutcomeResult,
+    Principal,
+)
+
+
+@singledispatch
+def to_serializable(val) -> dict | list | str:
+    """Default serialization"""
+    return str(val)
+
+
+@to_serializable.register
+def serialize_sets(val: set) -> list:
+    """Convert sets to lists for serialization"""
+    return list(val)
+
+
+@to_serializable.register
+def serialize_exceptions(val: Exception) -> str:
+    """Convert sets to lists for serialization"""
+    return repr(val)
+
+
+json_dumps = functools.partial(json.dumps, default=to_serializable)
+
+# Example use of ContextVar, TBD if this works well
+REQ_ID: ContextVar[str | uuid.UUID] = ContextVar("request_id")
+
+
+def log(
+    action_type: ActionType,
+    resource_type: str,
+    resource_id: Any,
+    result: OutcomeResult,
+    principal: Principal,
+    request_id: str | uuid.UUID | None = None,
+    outcome_reason: str | None = None,
+    before: Any | None = None,
+    after: Any | None = None,
+    serializer: Callable[[dict], str | bytes] = json_dumps,
+):
+    now = datetime.now(tz=UTC).isoformat()
+    request_id = request_id or REQ_ID.get()
+    print(
+        serializer(
+            {
+                "type": "audit-log",
+                "timestamp": now,
+                "level": "INFO",
+                "version": SCHEMA_VERSION,
+                "resource": {"type": resource_type, "id": resource_id},
+                "action": {"type": action_type},
+                "outcome": {
+                    "result": result,
+                    "reason": outcome_reason,
+                    "before": before,
+                    "after": after,
+                },
+                "context": {"request": {"id": request_id}},
+                "principal": dataclasses.asdict(principal),
+            }
+        )
+    )
diff --git a/audit_log/schema.py b/audit_log/schema.py
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/requirements-dev.txt b/requirements-dev.txt
diff --git a/tests/test_headers.py b/tests/test_headers.py
diff --git a/tests/test_log.py b/tests/test_log.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+* @iceye-ltd/role-python-logging-maintainer`