diff --git a/INSTALL.md b/INSTALL.md index cf669297..2cd83308 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -140,6 +140,93 @@ is running +## Report Protocols + +Patchman supports two report protocols: + +### Protocol 1 (Text) +The original form-based protocol. Uses multipart form data to upload package +and repository information. No additional dependencies required on the client. + +### Protocol 2 (JSON) +A JSON-based REST API. Provides better error handling, structured validation, +and easier debugging. Requires `jq` on the client. + +To use Protocol 2, update your `patchman-client.conf`: + +```shell +protocol=2 +``` + +Or use the `-p 2` command line option: + +```shell +$ patchman-client -s http://patchman.example.org -p 2 +``` + + +## API Key Authentication + +For Protocol 2, API key authentication is available using +[djangorestframework-api-key](https://florimondmanca.github.io/djangorestframework-api-key/). +Keys are hashed in the database and cannot be retrieved after creation. + +### Server-side setup + +1. Run migrations (first time only): + +```shell +$ patchman-manage migrate +``` + +2. Create an API key: + +```shell +$ patchman-manage create_api_key "clients" +Created API key: clients + + Key: abc123... + +Add this to your patchman-client.conf: + api_key=abc123... + +Save this key as it cannot be retrieved later. +``` + +3. List existing keys: + +```shell +$ patchman-manage list_api_keys +``` + +4. Revoke a key: + +```shell +$ patchman-manage revoke_api_key +``` + +5. To require API keys for all Protocol 2 uploads, set in `local_settings.py`: + +```python +REQUIRE_API_KEY = True +``` + +API keys can also be managed via the Django admin interface. + +### Client-side setup + +Add the API key to `patchman-client.conf`: + +```shell +protocol=2 +api_key=abc123... +``` + +Or use the `-k` command line option: + +```shell +$ patchman-client -s http://patchman.example.org -p 2 -k abc123... +``` ## Configure Database diff --git a/README.md b/README.md index d425c5fd..11088556 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,9 @@ required to report packages, `yum`, `dnf`, `zypper` and/or `apt` are required to report repositories. These packages are normally installed by default on most systems. +For Protocol 2 (JSON-based reports), `jq` is required. If `jq` is not available, +the client will automatically fall back to Protocol 1 (text-based reports). + deb-based OS's do not always change the kernel version when a kernel update is installed, so the `update-notifier-common` package can optionally be installed to enable this functionality. rpm-based OS's can tell if a reboot is required diff --git a/client/patchman-client b/client/patchman-client index bf4bc5c9..29b3b869 100755 --- a/client/patchman-client +++ b/client/patchman-client @@ -12,9 +12,10 @@ report=false local_updates=false repo_check=true tags='' +api_key='' usage() { - echo "${0} [-v] [-d] [-n] [-u] [-r] [-s SERVER] [-c FILE] [-t TAGS] [-h HOSTNAME]" + echo "${0} [-v] [-d] [-n] [-u] [-r] [-s SERVER] [-c FILE] [-t TAGS] [-h HOSTNAME] [-p PROTOCOL] [-k API_KEY]" echo "-v: verbose output (default is silent)" echo "-d: debug output" echo "-n: no repo check (required when used as an apt or yum plugin)" @@ -24,13 +25,15 @@ usage() { echo "-c FILE: config file location (default is /etc/patchman/patchman-client.conf)" echo "-t TAGS: comma-separated list of tags, e.g. -t www,dev" echo "-h HOSTNAME: specify the hostname of the local host" + echo "-p PROTOCOL: protocol version (1 or 2, default is 1)" + echo "-k API_KEY: API key for protocol 2 authentication" echo echo "Command line options override config file options." exit 0 } parseopts() { - while getopts "vdnurs:c:t:h:" opt; do + while getopts "vdnurs:c:t:h:p:k:" opt; do case ${opt} in v) verbose=true @@ -60,6 +63,12 @@ parseopts() { h) cli_hostname=${OPTARG} ;; + p) + cli_protocol=${OPTARG} + ;; + k) + cli_api_key=${OPTARG} + ;; *) usage ;; @@ -74,6 +83,14 @@ cleanup() { echo "Debug: not deleting ${tmpfile_sec} (security updates)" echo "Debug: not deleting ${tmpfile_bug} (updates)" echo "Debug: not deleting ${tmpfile_mod} (modules)" + if [ ! -z "${tmpfile_packages_json}" ] ; then + echo "Debug: not deleting ${tmpfile_packages_json} (packages json)" + echo "Debug: not deleting ${tmpfile_repos_json} (repos json)" + echo "Debug: not deleting ${tmpfile_modules_json} (modules json)" + echo "Debug: not deleting ${tmpfile_sec_json} (security updates json)" + echo "Debug: not deleting ${tmpfile_bug_json} (bug updates json)" + echo "Debug: not deleting ${tmpfile_report_json} (full report json)" + fi elif ${verbose} && ! ${debug} ; then echo "Deleting ${tmpfile_pkg}" echo "Deleting ${tmpfile_rep}" @@ -87,6 +104,12 @@ cleanup() { rm -fr "${tmpfile_sec}" rm -fr "${tmpfile_bug}" rm -fr "${tmpfile_mod}" + rm -fr "${tmpfile_packages_json}" + rm -fr "${tmpfile_repos_json}" + rm -fr "${tmpfile_modules_json}" + rm -fr "${tmpfile_sec_json}" + rm -fr "${tmpfile_bug_json}" + rm -fr "${tmpfile_report_json}" fi flock -u 200 rm -fr "${lock_dir}/patchman.lock" @@ -130,6 +153,14 @@ check_conf() { tags="${cli_tags}" fi + if [ ! -z "${cli_protocol}" ] ; then + protocol=${cli_protocol} + fi + + if [ ! -z "${cli_api_key}" ] ; then + api_key=${cli_api_key} + fi + if [ -z "${hostname}" ] && [ -z "${cli_hostname}" ] ; then get_hostname else @@ -140,6 +171,14 @@ check_conf() { check_booleans + # Check if protocol 2 is requested but jq is not available + if [ "${protocol}" == "2" ] ; then + if ! check_command_exists jq ; then + echo "Warning: jq not found, falling back to protocol 1" + protocol=1 + fi + fi + if ${verbose} ; then echo "Patchman configuration seems OK:" if [ -f ${conf} ] ; then @@ -148,6 +187,10 @@ check_conf() { echo "Patchman Server: ${server}" echo "Hostname: ${hostname}" echo "Tags: ${tags}" + echo "Protocol: ${protocol}" + if [ ! -z "${api_key}" ] ; then + echo "API Key: ${api_key:0:12}..." + fi for var in report local_updates repo_check verbose debug ; do eval val=\$${var} echo "${var}: ${val}" @@ -642,6 +685,302 @@ reboot_required() { fi } +build_packages_json() { + # Convert packages file to JSON array, writing to temp file + local outfile="${1}" + echo "[" > "${outfile}" + local first=true + while IFS= read -r line ; do + if [ -z "${line}" ] ; then + continue + fi + # Parse: 'name' 'epoch' 'version' 'release' 'arch' 'type' ['category'] ['repo'] + # Extract quoted fields properly + local parts=() + while IFS= read -r part ; do + parts+=("${part}") + done < <(echo "${line}" | grep -oP "'[^']*'" | tr -d "'") + + local name="${parts[0]:-}" + local epoch="${parts[1]:-}" + local version="${parts[2]:-}" + local release="${parts[3]:-}" + local arch="${parts[4]:-}" + local ptype="${parts[5]:-}" + local category="${parts[6]:-}" + local repo="${parts[7]:-}" + + # Skip packages without a valid type + if [ -z "${ptype}" ] ; then + continue + fi + + # Skip gpg-pubkey entries (they're not real packages) + if [ "${name}" == "gpg-pubkey" ] ; then + continue + fi + + if ! ${first} ; then + echo "," >> "${outfile}" + fi + first=false + + jq -n -c \ + --arg name "${name}" \ + --arg epoch "${epoch}" \ + --arg version "${version}" \ + --arg release "${release}" \ + --arg arch "${arch}" \ + --arg type "${ptype}" \ + --arg category "${category}" \ + --arg repo "${repo}" \ + '{name: $name, epoch: $epoch, version: $version, release: $release, arch: $arch, type: $type, category: $category, repo: $repo}' >> "${outfile}" + done < "${tmpfile_pkg}" + echo "]" >> "${outfile}" +} + +build_repos_json() { + # Convert repos file to JSON array, writing to temp file + local outfile="${1}" + echo "[" > "${outfile}" + local first=true + while IFS= read -r line ; do + if [ -z "${line}" ] ; then + continue + fi + # Parse: 'type' 'name' 'id' 'priority' 'url1' ['url2' ...] + local parts=() + while IFS= read -r part ; do + parts+=("${part}") + done < <(echo "${line}" | grep -oP "'[^']*'" | tr -d "'") + + local rtype="${parts[0]}" + local name="${parts[1]}" + local rid="${parts[2]}" + local priority="${parts[3]}" + + # Collect URLs (all remaining parts) + local urls_json="[" + local url_first=true + for ((i=4; i<${#parts[@]}; i++)); do + if ! ${url_first} ; then + urls_json="${urls_json}," + fi + url_first=false + urls_json="${urls_json}\"${parts[i]}\"" + done + urls_json="${urls_json}]" + + if ! ${first} ; then + echo "," >> "${outfile}" + fi + first=false + + jq -n -c \ + --arg type "${rtype}" \ + --arg name "${name}" \ + --arg id "${rid}" \ + --argjson priority "${priority:-0}" \ + --argjson urls "${urls_json}" \ + '{type: $type, name: $name, id: $id, priority: $priority, urls: $urls}' >> "${outfile}" + done < "${tmpfile_rep}" + echo "]" >> "${outfile}" +} + +build_modules_json() { + # Convert modules file to JSON array, writing to temp file + local outfile="${1}" + echo "[" > "${outfile}" + local first=true + while IFS= read -r line ; do + if [ -z "${line}" ] ; then + continue + fi + # Parse: 'name' 'stream' 'version' 'context' 'arch' 'repo' ['pkg1' 'pkg2' ...] + local parts=() + while IFS= read -r part ; do + parts+=("${part}") + done < <(echo "${line}" | grep -oP "'[^']*'" | tr -d "'") + + local name="${parts[0]}" + local stream="${parts[1]}" + local version="${parts[2]}" + local context="${parts[3]}" + local arch="${parts[4]}" + local repo="${parts[5]}" + + # Collect packages (all remaining parts) + local pkgs_json="[" + local pkg_first=true + for ((i=6; i<${#parts[@]}; i++)); do + if ! ${pkg_first} ; then + pkgs_json="${pkgs_json}," + fi + pkg_first=false + pkgs_json="${pkgs_json}\"${parts[i]}\"" + done + pkgs_json="${pkgs_json}]" + + if ! ${first} ; then + echo "," >> "${outfile}" + fi + first=false + + jq -n -c \ + --arg name "${name}" \ + --arg stream "${stream}" \ + --arg version "${version}" \ + --arg context "${context}" \ + --arg arch "${arch}" \ + --arg repo "${repo}" \ + --argjson packages "${pkgs_json}" \ + '{name: $name, stream: $stream, version: $version, context: $context, arch: $arch, repo: $repo, packages: $packages}' >> "${outfile}" + done < "${tmpfile_mod}" + echo "]" >> "${outfile}" +} + +build_updates_json() { + # Convert updates file to JSON array, writing to temp file + local updates_file="${1}" + local outfile="${2}" + echo "[" > "${outfile}" + local first=true + while IFS= read -r line ; do + if [ -z "${line}" ] ; then + continue + fi + # Parse: name.arch version repo + local parts=(${line}) + local name_arch="${parts[0]}" + local version="${parts[1]}" + local repo="${parts[2]}" + + # Split name.arch + local name="${name_arch%.*}" + local arch="${name_arch##*.}" + + if ! ${first} ; then + echo "," >> "${outfile}" + fi + first=false + + jq -n -c \ + --arg name "${name}" \ + --arg version "${version}" \ + --arg arch "${arch}" \ + --arg repo "${repo}" \ + '{name: $name, version: $version, arch: $arch, repo: $repo}' >> "${outfile}" + done < "${updates_file}" + echo "]" >> "${outfile}" +} + +build_json_report() { + # Build complete JSON report using temp files + # Temp files are created by post_json_data before calling this function + + build_packages_json "${tmpfile_packages_json}" + build_repos_json "${tmpfile_repos_json}" + build_modules_json "${tmpfile_modules_json}" + build_updates_json "${tmpfile_sec}" "${tmpfile_sec_json}" + build_updates_json "${tmpfile_bug}" "${tmpfile_bug_json}" + + # Convert tags to JSON array + local tags_json="[]" + if [ ! -z "${tags}" ] && [ "${tags}" != "Default" ] ; then + tags_json=$(echo "${tags}" | tr ',' '\n' | jq -R . | jq -s .) + fi + + # Convert reboot to boolean + local reboot_bool="false" + if [ "${reboot}" == "True" ] ; then + reboot_bool="true" + fi + + jq -n \ + --argjson protocol 2 \ + --arg hostname "${hostname}" \ + --arg arch "${host_arch}" \ + --arg kernel "${host_kernel}" \ + --arg os "${os}" \ + --argjson tags "${tags_json}" \ + --argjson reboot_required "${reboot_bool}" \ + --slurpfile packages "${tmpfile_packages_json}" \ + --slurpfile repos "${tmpfile_repos_json}" \ + --slurpfile modules "${tmpfile_modules_json}" \ + --slurpfile sec_updates "${tmpfile_sec_json}" \ + --slurpfile bug_updates "${tmpfile_bug_json}" \ + '{ + protocol: $protocol, + hostname: $hostname, + arch: $arch, + kernel: $kernel, + os: $os, + tags: $tags, + reboot_required: $reboot_required, + packages: $packages[0], + repos: $repos[0], + modules: $modules[0], + sec_updates: $sec_updates[0], + bug_updates: $bug_updates[0] + }' +} + +post_json_data() { + # Post data using protocol 2 (JSON) + curl_opts=${curl_options} + + if ${verbose} ; then + curl_opts="${curl_opts} -i" + echo "Sending JSON data to ${server} with curl (protocol 2):" + else + curl_opts="${curl_opts} -s -S" + fi + + # Create temp files for JSON (global so cleanup can handle them) + tmpfile_packages_json=$(mktemp) + tmpfile_repos_json=$(mktemp) + tmpfile_modules_json=$(mktemp) + tmpfile_sec_json=$(mktemp) + tmpfile_bug_json=$(mktemp) + tmpfile_report_json=$(mktemp) + + # Build JSON report to temp file + build_json_report > "${tmpfile_report_json}" + + if ${debug} ; then + echo "JSON Report:" + jq . "${tmpfile_report_json}" + fi + + curl_opts="${curl_opts} -H 'Content-Type: application/json'" + if [ ! -z "${api_key}" ] ; then + curl_opts="${curl_opts} -H 'Authorization: Api-Key ${api_key}'" + fi + curl_opts="${curl_opts} -d @${tmpfile_report_json}" + + post_command="curl ${curl_opts} ${server%/}/api/report/" + + if ${verbose} ; then + echo "${post_command}" + fi + + result=$(eval "${post_command}") + retval=${?} + + if [ ! ${retval} -eq 0 ] ; then + echo 'Failed to upload report.' + exit ${retval} + fi + + if ${report} || ${verbose} ; then + if [ ! -z "${result}" ] ; then + echo "${result}" | jq . 2>/dev/null || echo "${result}" + else + echo "No output returned." + fi + fi +} + post_data() { curl_opts=${curl_options} @@ -736,4 +1075,10 @@ if ${repo_check} ; then get_repos fi reboot_required -post_data + +# Use protocol 2 (JSON) or protocol 1 (form data) based on config +if [ "${protocol}" == "2" ] ; then + post_json_data +else + post_data +fi diff --git a/client/patchman-client.conf b/client/patchman-client.conf index fdca5663..666dd54c 100644 --- a/client/patchman-client.conf +++ b/client/patchman-client.conf @@ -9,3 +9,10 @@ tags="Server" # Does the client output a report of the upload (e.g. for cronjob output) report=false + +# Protocol version (1 = text, 2 = json) +# Protocol 2 requires jq to be installed +protocol=1 + +# API key for protocol 2 authentication +#api_key=pm_your_api_key_here diff --git a/debian/control b/debian/control index 34610549..53121399 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,8 @@ Architecture: all Homepage: https://github.com/furlongm/patchman Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-django-extensions, python3-django-bootstrap3, python3-cvss, - python3-djangorestframework, python3-django-filters, python3-debian, + python3-djangorestframework, python3-djangorestframework-api-key, + python3-django-filters, python3-debian, python3-rpm, python3-tqdm, python3-defusedxml, python3-pip, python3-tenacity, python3-requests, python3-colorama, python3-magic, python3-humanize, python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, @@ -44,7 +45,7 @@ Description: Django-based patch status monitoring tool for linux systems. Package: patchman-client Architecture: all Homepage: https://github.com/furlongm/patchman -Depends: ${misc:Depends}, curl, debianutils, util-linux, coreutils, mawk +Depends: ${misc:Depends}, curl, debianutils, util-linux, coreutils, mawk, jq Description: Client for the patchman monitoring system. . The client will send a list of packages and repositories to the upstream diff --git a/patchman-client.spec b/patchman-client.spec index f2f8279a..ed04ac12 100644 --- a/patchman-client.spec +++ b/patchman-client.spec @@ -8,7 +8,7 @@ License: GPLv3 URL: http://patchman.openbytes.ie Source: %{expand:%%(pwd)} BuildArch: noarch -Requires: curl which coreutils util-linux gawk +Requires: curl which coreutils util-linux gawk jq %define _binary_payload w9.gzdio diff --git a/patchman/settings.py b/patchman/settings.py index 76c28b3f..20ac82e8 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -100,15 +100,24 @@ 'security.apps.SecurityConfig', 'reports.apps.ReportsConfig', 'util.apps.UtilConfig', + 'rest_framework_api_key', ] REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': ['rest_framework.permissions.IsAuthenticatedOrReadOnly'], # noqa + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.BasicAuthentication', + ], 'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'], # noqa 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', # noqa 'PAGE_SIZE': 100, } +# API Key authentication settings +# Set to False to allow unauthenticated protocol 2 report uploads +REQUIRE_API_KEY = True + TAGGIT_CASE_INSENSITIVE = True DJANGO_TABLES2_TEMPLATE = 'table.html' diff --git a/patchman/urls.py b/patchman/urls.py index b66a0de3..4cc22900 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -28,6 +28,7 @@ from hosts import views as host_views from operatingsystems import views as os_views from packages import views as package_views +from reports import views as report_views from repos import views as repo_views from security import views as security_views @@ -48,6 +49,7 @@ router.register(r'repo', repo_views.RepositoryViewSet) router.register(r'mirror', repo_views.MirrorViewSet) router.register(r'mirror-package', repo_views.MirrorPackageViewSet) +router.register(r'report', report_views.ReportViewSet, basename='report') admin.autodiscover() diff --git a/reports/models.py b/reports/models.py index f1e0f6f3..bda18fed 100644 --- a/reports/models.py +++ b/reports/models.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +import json + from django.db import models from django.urls import reverse @@ -53,6 +55,91 @@ def __str__(self): def get_absolute_url(self): return reverse('reports:report_detail', args=[str(self.id)]) + @property + def packages_parsed(self): + """Parse packages JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.packages: + try: + return json.loads(self.packages) + except json.JSONDecodeError: + return [] + return [] + + @property + def repos_parsed(self): + """Parse repos JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.repos: + try: + return json.loads(self.repos) + except json.JSONDecodeError: + return [] + return [] + + @property + def modules_parsed(self): + """Parse modules JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.modules: + try: + return json.loads(self.modules) + except json.JSONDecodeError: + return [] + return [] + + @property + def sec_updates_parsed(self): + """Parse security updates JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.sec_updates: + try: + return json.loads(self.sec_updates) + except json.JSONDecodeError: + return [] + return [] + + @property + def bug_updates_parsed(self): + """Parse bug updates JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.bug_updates: + try: + return json.loads(self.bug_updates) + except json.JSONDecodeError: + return [] + return [] + + @property + def has_packages(self): + """Check if report has packages data.""" + if self.protocol == '2': + return bool(self.packages_parsed) + return bool(self.packages and self.packages.strip()) + + @property + def has_repos(self): + """Check if report has repos data.""" + if self.protocol == '2': + return bool(self.repos_parsed) + return bool(self.repos and self.repos.strip()) + + @property + def has_modules(self): + """Check if report has modules data.""" + if self.protocol == '2': + return bool(self.modules_parsed) + return bool(self.modules and self.modules.strip()) + + @property + def has_sec_updates(self): + """Check if report has security updates.""" + if self.protocol == '2': + return bool(self.sec_updates_parsed) + return bool(self.sec_updates and self.sec_updates.strip()) + + @property + def has_bug_updates(self): + """Check if report has bug updates.""" + if self.protocol == '2': + return bool(self.bug_updates_parsed) + return bool(self.bug_updates and self.bug_updates.strip()) + def parse(self, data, meta): """ Parse a report and save the object """ @@ -113,13 +200,34 @@ def process(self, find_updates=True, verbose=False): if verbose: info_message(text=f'Processing report {self.id} - {self.host}') - from reports.utils import ( - process_modules, process_packages, process_repos, process_updates, - ) - process_repos(report=self, host=host) - process_modules(report=self, host=host) - process_packages(report=self, host=host) - process_updates(report=self, host=host) + if self.protocol == '2': + # Protocol 2: JSON data + import json + + from reports.utils import ( + process_modules_json, process_packages_json, + process_repos_json, process_updates_json, + ) + packages_json = json.loads(self.packages) if self.packages else [] + repos_json = json.loads(self.repos) if self.repos else [] + modules_json = json.loads(self.modules) if self.modules else [] + sec_updates_json = json.loads(self.sec_updates) if self.sec_updates else [] + bug_updates_json = json.loads(self.bug_updates) if self.bug_updates else [] + + process_repos_json(repos_json, host, self.arch) + process_modules_json(modules_json, host) + process_packages_json(packages_json, host) + process_updates_json(sec_updates_json, bug_updates_json, host) + else: + # Protocol 1: Text data + from reports.utils import ( + process_modules, process_packages, process_repos, + process_updates, + ) + process_repos(report=self, host=host) + process_modules(report=self, host=host) + process_packages(report=self, host=host) + process_updates(report=self, host=host) self.processed = True self.save() diff --git a/reports/serializers.py b/reports/serializers.py new file mode 100644 index 00000000..6822afa7 --- /dev/null +++ b/reports/serializers.py @@ -0,0 +1,91 @@ +# Copyright 2013-2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from rest_framework import serializers + + +class PackageSerializer(serializers.Serializer): + """Serializer for a single package in a report.""" + name = serializers.CharField(max_length=255) + epoch = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + version = serializers.CharField(max_length=255) + release = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + arch = serializers.CharField(max_length=255) + type = serializers.ChoiceField(choices=['deb', 'rpm', 'arch', 'gentoo']) + # Gentoo-specific fields + category = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + repo = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + + +class RepoSerializer(serializers.Serializer): + """Serializer for a single repository in a report.""" + type = serializers.ChoiceField(choices=['deb', 'rpm', 'arch', 'gentoo']) + name = serializers.CharField(max_length=255) + id = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + priority = serializers.IntegerField(required=False, default=0) + urls = serializers.ListField( + child=serializers.URLField(max_length=512), + required=False, + default=list + ) + + +class ModuleSerializer(serializers.Serializer): + """Serializer for a single module in a report.""" + name = serializers.CharField(max_length=255) + stream = serializers.CharField(max_length=255) + version = serializers.CharField(max_length=255) + context = serializers.CharField(max_length=255) + arch = serializers.CharField(max_length=255) + repo = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + packages = serializers.ListField( + child=serializers.CharField(max_length=255), + required=False, + default=list + ) + + +class UpdateSerializer(serializers.Serializer): + """Serializer for a single update (security or bugfix) in a report.""" + name = serializers.CharField(max_length=255) + version = serializers.CharField(max_length=255) + arch = serializers.CharField(max_length=255) + repo = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + + +class ReportUploadSerializer(serializers.Serializer): + """Serializer for protocol 2 JSON report uploads.""" + protocol = serializers.IntegerField(default=2) + hostname = serializers.CharField(max_length=255) + arch = serializers.CharField(max_length=255) + kernel = serializers.CharField(max_length=255) + os = serializers.CharField(max_length=255) + tags = serializers.ListField( + child=serializers.CharField(max_length=255), + required=False, + default=list + ) + reboot_required = serializers.BooleanField(required=False, default=False) + packages = PackageSerializer(many=True, required=False, default=list) + repos = RepoSerializer(many=True, required=False, default=list) + modules = ModuleSerializer(many=True, required=False, default=list) + sec_updates = UpdateSerializer(many=True, required=False, default=list) + bug_updates = UpdateSerializer(many=True, required=False, default=list) + + def validate_protocol(self, value): + if value != 2: + raise serializers.ValidationError('This endpoint only accepts protocol 2') + return value diff --git a/reports/tables.py b/reports/tables.py index 52f077e0..bc43e49e 100644 --- a/reports/tables.py +++ b/reports/tables.py @@ -58,3 +58,57 @@ class ReportTable(BaseTable): class Meta(BaseTable.Meta): model = Report fields = ('selection', 'report_id', 'host', 'created', 'report_ip', 'processed') + + +class ReportPackageTable(tables.Table): + """Table for displaying packages in Protocol 2 reports.""" + name = tables.Column() + epoch = tables.Column(default='') + version = tables.Column() + release = tables.Column() + arch = tables.Column() + type = tables.Column() + + class Meta: + attrs = {'class': 'table table-striped table-bordered table-hover table-condensed'} + orderable = True + + +class ReportRepoTable(tables.Table): + """Table for displaying repos in Protocol 2 reports.""" + type = tables.Column() + name = tables.Column() + id = tables.Column(verbose_name='ID') + priority = tables.Column() + + class Meta: + attrs = {'class': 'table table-striped table-bordered table-hover table-condensed'} + orderable = True + + +class ReportModuleTable(tables.Table): + """Table for displaying modules in Protocol 2 reports.""" + name = tables.Column() + stream = tables.Column() + version = tables.Column() + context = tables.Column() + arch = tables.Column() + repo = tables.Column(default='') + + class Meta: + attrs = {'class': 'table table-striped table-bordered table-hover table-condensed'} + orderable = True + + +class ReportUpdateTable(tables.Table): + """Table for displaying updates in Protocol 2 reports.""" + name = tables.Column() + epoch = tables.Column(default='') + version = tables.Column() + release = tables.Column() + arch = tables.Column() + repo = tables.Column(default='') + + class Meta: + attrs = {'class': 'table table-striped table-bordered table-hover table-condensed'} + orderable = True diff --git a/reports/templates/reports/report_detail.html b/reports/templates/reports/report_detail.html index 08d5005f..b68122bd 100644 --- a/reports/templates/reports/report_detail.html +++ b/reports/templates/reports/report_detail.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% load bootstrap3 common %} +{% load bootstrap3 common django_tables2 %} {% block page_title %}Report - {{ report }} {% endblock %} @@ -12,17 +12,21 @@
@@ -48,71 +52,95 @@
-
-
- - {% for repo in report.repos.splitlines %} - - - - {% endfor %} -
{{ repo }}
+ {% if report.has_repos %} +
+
+ {% if repos_table %} + {% render_table repos_table %} + {% else %} + + {% for repo in report.repos.splitlines %} + + + + {% endfor %} +
{{ repo }}
+ {% endif %} +
-
+ {% endif %} - {% if report.modules %} + {% if report.has_modules %}
- - {% for module in report.modules.splitlines %} - - - - {% endfor %} -
{{ module }}
+ {% if modules_table %} + {% render_table modules_table %} + {% else %} + + {% for module in report.modules.splitlines %} + + + + {% endfor %} +
{{ module }}
+ {% endif %}
{% endif %} - {% if report.sec_updates %} + {% if report.has_sec_updates %}
- - {% for update in report.sec_updates.splitlines %} - - - - {% endfor %} -
{{ update }}
+ {% if sec_updates_table %} + {% render_table sec_updates_table %} + {% else %} + + {% for update in report.sec_updates.splitlines %} + + + + {% endfor %} +
{{ update }}
+ {% endif %}
{% endif %} - {% if report.bug_updates %} + {% if report.has_bug_updates %}
- - {% for update in report.bug_updates.splitlines %} - - - - {% endfor %} -
{{ update }}
+ {% if bug_updates_table %} + {% render_table bug_updates_table %} + {% else %} + + {% for update in report.bug_updates.splitlines %} + + + + {% endfor %} +
{{ update }}
+ {% endif %}
{% endif %} -
-
- - {% for package in report.packages.splitlines %} - - - - {% endfor %} -
{{ package }}
+ {% if report.has_packages %} +
+
+ {% if packages_table %} + {% render_table packages_table %} + {% else %} + + {% for package in report.packages.splitlines %} + + + + {% endfor %} +
{{ package }}
+ {% endif %} +
-
+ {% endif %}
{% endblock %} diff --git a/reports/tests.py b/reports/tests.py new file mode 100644 index 00000000..c75fe305 --- /dev/null +++ b/reports/tests.py @@ -0,0 +1,291 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APITestCase +from rest_framework_api_key.models import APIKey + +from reports.models import Report + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReportAPITests(APITestCase): + """Tests for the Protocol 2 JSON report API.""" + + def setUp(self): + self.url = '/api/report/' + + def test_upload_minimal_report(self): + """Test uploading a minimal valid report.""" + data = { + 'protocol': 2, + 'hostname': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0-91-generic', + 'os': 'Ubuntu 22.04.3 LTS', + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data['status'], 'accepted') + self.assertIn('report_id', response.data) + + # Verify report was created + report = Report.objects.get(id=response.data['report_id']) + self.assertEqual(report.host, 'testhost.example.com') + self.assertEqual(report.protocol, '2') + + def test_upload_full_report(self): + """Test uploading a report with all fields.""" + data = { + 'protocol': 2, + 'hostname': 'server1.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0-91-generic', + 'os': 'Ubuntu 22.04.3 LTS', + 'tags': ['web', 'production'], + 'reboot_required': True, + 'packages': [ + { + 'name': 'nginx', + 'epoch': '', + 'version': '1.18.0', + 'release': '6ubuntu14', + 'arch': 'amd64', + 'type': 'deb' + }, + { + 'name': 'curl', + 'epoch': '', + 'version': '7.81.0', + 'release': '1ubuntu1.15', + 'arch': 'amd64', + 'type': 'deb' + } + ], + 'repos': [ + { + 'type': 'deb', + 'name': 'Ubuntu 22.04 amd64 main', + 'id': 'ubuntu-main', + 'priority': 500, + 'urls': ['http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64'] + } + ], + 'modules': [], + 'sec_updates': [ + { + 'name': 'openssl', + 'version': '3.0.2-0ubuntu1.13', + 'arch': 'amd64', + 'repo': 'ubuntu-security' + } + ], + 'bug_updates': [] + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + report = Report.objects.get(id=response.data['report_id']) + self.assertEqual(report.host, 'server1.example.com') + self.assertEqual(report.tags, 'web,production') + self.assertEqual(report.reboot, 'True') + + def test_upload_missing_required_field(self): + """Test that missing required fields return 400.""" + data = { + 'protocol': 2, + 'hostname': 'testhost.example.com', + # missing arch, kernel, os + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data['status'], 'error') + self.assertIn('errors', response.data) + + def test_upload_wrong_protocol(self): + """Test that wrong protocol version returns 400.""" + data = { + 'protocol': 1, # Should be 2 + 'hostname': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_hostname_lowercased(self): + """Test that hostname is lowercased.""" + data = { + 'protocol': 2, + 'hostname': 'TESTHOST.EXAMPLE.COM', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + report = Report.objects.get(id=response.data['report_id']) + self.assertEqual(report.host, 'testhost.example.com') + + def test_domain_extracted_from_hostname(self): + """Test that domain is extracted from FQDN.""" + data = { + 'protocol': 2, + 'hostname': 'server1.prod.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + report = Report.objects.get(id=response.data['report_id']) + self.assertEqual(report.domain, 'prod.example.com') + + def test_invalid_package_type(self): + """Test that invalid package type returns 400.""" + data = { + 'protocol': 2, + 'hostname': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'packages': [ + { + 'name': 'nginx', + 'version': '1.18.0', + 'arch': 'amd64', + 'type': 'invalid' # Invalid type + } + ] + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + +class ReportSerializerTests(TestCase): + """Tests for report serializers.""" + + def test_package_serializer_valid(self): + from reports.serializers import PackageSerializer + data = { + 'name': 'nginx', + 'epoch': '', + 'version': '1.18.0', + 'release': '6ubuntu14', + 'arch': 'amd64', + 'type': 'deb' + } + serializer = PackageSerializer(data=data) + self.assertTrue(serializer.is_valid()) + + def test_package_serializer_minimal(self): + from reports.serializers import PackageSerializer + data = { + 'name': 'nginx', + 'version': '1.18.0', + 'arch': 'amd64', + 'type': 'deb' + } + serializer = PackageSerializer(data=data) + self.assertTrue(serializer.is_valid()) + + def test_repo_serializer_valid(self): + from reports.serializers import RepoSerializer + data = { + 'type': 'deb', + 'name': 'Ubuntu Main', + 'id': 'ubuntu-main', + 'priority': 500, + 'urls': ['http://archive.ubuntu.com/ubuntu'] + } + serializer = RepoSerializer(data=data) + self.assertTrue(serializer.is_valid()) + + def test_module_serializer_valid(self): + from reports.serializers import ModuleSerializer + data = { + 'name': 'nodejs', + 'stream': '18', + 'version': '8090020240101', + 'context': 'rhel9', + 'arch': 'x86_64', + 'repo': 'appstream', + 'packages': ['nodejs-18.19.0-1.module+el9'] + } + serializer = ModuleSerializer(data=data) + self.assertTrue(serializer.is_valid()) + + +@override_settings( + REQUIRE_API_KEY=True, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ApiKeyAuthTests(APITestCase): + """Tests for API key authentication using djangorestframework-api-key.""" + + def setUp(self): + self.url = '/api/report/' + self.api_key_obj, self.api_key = APIKey.objects.create_key(name='test-key') + self.valid_data = { + 'protocol': 2, + 'hostname': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + } + + def test_valid_api_key(self): + """Test that valid API key authenticates.""" + self.client.credentials(HTTP_AUTHORIZATION=f'Api-Key {self.api_key}') + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + def test_invalid_api_key(self): + """Test that invalid API key returns 403.""" + self.client.credentials(HTTP_AUTHORIZATION='Api-Key invalid_key') + response = self.client.post(self.url, self.valid_data, format='json') + # drf-api-key returns 403 for invalid keys when using HasAPIKey permission + self.assertIn(response.status_code, [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN]) + + def test_revoked_api_key(self): + """Test that revoked API key returns 403.""" + self.api_key_obj.revoked = True + self.api_key_obj.save() + self.client.credentials(HTTP_AUTHORIZATION=f'Api-Key {self.api_key}') + response = self.client.post(self.url, self.valid_data, format='json') + self.assertIn(response.status_code, [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN]) + + def test_missing_api_key_requires_auth(self): + """Test that missing API key returns 403 when REQUIRE_API_KEY is True.""" + # No credentials set + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + @override_settings(REQUIRE_API_KEY=False) + def test_no_auth_allowed_when_disabled(self): + """Test that no auth is allowed when REQUIRE_API_KEY is False.""" + # No credentials set + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) diff --git a/reports/utils.py b/reports/utils.py index 1a08cf3c..85cf6e3b 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -48,7 +48,7 @@ def process_repos(report, host): pbar_start.send(sender=None, ptext=f'{host} Repos', plen=len(repos)) for i, repo_str in enumerate(repos): debug_message(f'Processing report {report.id} repo: {repo_str}') - repo, priority = process_repo(repo_str, report.arch) + repo, priority = process_repo_text(repo_str, report.arch) if repo: repo_ids.append(repo.id) try: @@ -74,7 +74,7 @@ def process_modules(report, host): pbar_start.send(sender=None, ptext=f'{host} Modules', plen=len(modules)) for i, module_str in enumerate(modules): - module = process_module(module_str) + module = process_module_text(module_str) if module: module_ids.append(module.id) host.modules.add(module) @@ -95,7 +95,7 @@ def process_packages(report, host): pbar_start.send(sender=None, ptext=f'{host} Packages', plen=len(packages)) for i, pkg_str in enumerate(packages): debug_message(f'Processing report {report.id} package: {pkg_str}') - package = process_package(pkg_str, report.protocol) + package = process_package_text(pkg_str) if package: package_ids.append(package.id) host.packages.add(package) @@ -142,7 +142,7 @@ def add_updates(updates, host): if ulen > 0: pbar_start.send(sender=None, ptext=f'{host} Updates', plen=ulen) for i, (u, sec) in enumerate(updates.items()): - update = process_update(host, u, sec) + update = process_update_text(host, u, sec) if update: host.updates.add(update) pbar_update.send(sender=None, index=i + 1) @@ -161,7 +161,7 @@ def parse_updates(updates_string, security): return updates -def process_update(host, update_string, security): +def process_update_text(host, update_string, security): """ Processes a single sanitized update string and converts to an update object. Only works if the original package exists. Returns None otherwise """ @@ -173,12 +173,19 @@ def process_update(host, update_string, security): p_arch = parts[2] p_epoch, p_version, p_release = find_evr(update_str[1]) + + return process_update(host, p_name, p_epoch, p_version, p_release, p_arch, repo_id, security) + + +def process_update(host, name, epoch, version, release, arch, repo_id, security): + """ Core update processing logic shared by text and JSON handlers + """ package = get_or_create_package( - name=p_name, - epoch=p_epoch, - version=p_version, - release=p_release, - arch=p_arch, + name=name, + epoch=epoch, + version=version, + release=release, + arch=arch, p_type=Package.RPM ) try: @@ -194,6 +201,7 @@ def process_update(host, update_string, security): installed_package = installed_packages[0] update = get_or_create_package_update(oldpackage=installed_package, newpackage=package, security=security) return update + return None def parse_repos(repos_string): @@ -208,35 +216,30 @@ def parse_repos(repos_string): return repos -def process_repo(repo, arch): - """ Processes a single sanitized repo string and converts to a repo object +def _get_repo_type(type_str): + """ Convert repo type string to Repository constant """ - repository = r_id = None - - if repo[0] == 'deb': - r_type = Repository.DEB - r_priority = int(repo[2]) - elif repo[0] == 'rpm': - r_type = Repository.RPM - r_id = repo.pop(2) - r_priority = int(repo[2]) * -1 - elif repo[0] == 'arch': - r_type = Repository.ARCH - r_id = repo[2] - r_priority = 0 - elif repo[0] == 'gentoo': - r_type = Repository.GENTOO - r_id = repo.pop(2) - r_priority = repo[2] - arch = 'any' - - if repo[1]: - r_name = repo[1] - - r_arch, c = MachineArchitecture.objects.get_or_create(name=arch) + type_str = type_str.lower() + if type_str == 'deb': + return Repository.DEB + elif type_str == 'rpm': + return Repository.RPM + elif type_str == 'arch': + return Repository.ARCH + elif type_str == 'gentoo': + return Repository.GENTOO + return None + + +def process_repo(r_type, r_name, r_id, r_priority, urls, arch): + """ Core repo processing logic shared by text and JSON handlers + """ + r_arch, _ = MachineArchitecture.objects.get_or_create(name=arch) + repository = None unknown = [] - for r_url in repo[3:]: + + for r_url in urls: if r_type == Repository.GENTOO and r_url.startswith('rsync'): r_url = 'https://api.gentoo.org/mirrors/distfiles.xml' try: @@ -248,6 +251,7 @@ def process_repo(repo, arch): unknown.append(r_url) else: repository = mirror.repo + if not repository: repository = get_or_create_repo(r_name, r_arch, r_type) @@ -272,6 +276,36 @@ def process_repo(repo, arch): return repository, r_priority +def process_repo_text(repo, arch): + """ Processes a single sanitized repo string and converts to a repo object + """ + r_id = None + + if repo[0] == 'deb': + r_type = Repository.DEB + r_priority = int(repo[2]) + elif repo[0] == 'rpm': + r_type = Repository.RPM + r_id = repo.pop(2) + r_priority = int(repo[2]) * -1 + elif repo[0] == 'arch': + r_type = Repository.ARCH + r_id = repo[2] + r_priority = 0 + elif repo[0] == 'gentoo': + r_type = Repository.GENTOO + r_id = repo.pop(2) + r_priority = repo[2] + arch = 'any' + else: + return None, 0 + + r_name = repo[1] if repo[1] else '' + urls = repo[3:] + + return process_repo(r_type, r_name, r_id, r_priority, urls, arch) + + def parse_modules(modules_string): """ Parses modules string in a report and returns a sanitized version """ @@ -283,17 +317,10 @@ def parse_modules(modules_string): return modules -def process_module(module_str): - """ Processes a single sanitized module string and converts to a module +def process_module(m_name, m_stream, m_version, m_context, m_arch, repo_id, package_strings): + """ Core module processing logic shared by text and JSON handlers """ - m_name = module_str[0] - m_stream = module_str[1] - m_version = module_str[2] - m_context = module_str[3] - m_arch = module_str[4] - repo_id = module_str[5] - - arch, c = PackageArchitecture.objects.get_or_create(name=m_arch) + arch, _ = PackageArchitecture.objects.get_or_create(name=m_arch) try: repo = Repository.objects.get(repo_id=repo_id) @@ -301,7 +328,7 @@ def process_module(module_str): repo = None packages = set() - for pkg_str in module_str[6:]: + for pkg_str in package_strings: p_type = Package.RPM p_name, p_epoch, p_ver, p_rel, p_dist, p_arch = parse_package_string(pkg_str) package = get_or_create_package(p_name, p_epoch, p_ver, p_rel, p_arch, p_type) @@ -313,6 +340,20 @@ def process_module(module_str): return module +def process_module_text(module_str): + """ Processes a single sanitized module string and converts to a module + """ + m_name = module_str[0] + m_stream = module_str[1] + m_version = module_str[2] + m_context = module_str[3] + m_arch = module_str[4] + repo_id = module_str[5] + package_strings = module_str[6:] + + return process_module(m_name, m_stream, m_version, m_context, m_arch, repo_id, package_strings) + + def parse_packages(packages_string): """ Parses packages string in a report and returns a sanitized version """ @@ -322,42 +363,60 @@ def parse_packages(packages_string): return packages -def process_package(pkg, protocol): +def _get_package_type(type_str): + """ Convert package type string to Package constant + """ + type_str = type_str.lower() if type_str else '' + if type_str == 'deb': + return Package.DEB + elif type_str == 'rpm': + return Package.RPM + elif type_str == 'arch': + return Package.ARCH + elif type_str == 'gentoo': + return Package.GENTOO + return Package.UNKNOWN + + +def process_package(name, epoch, version, release, arch, p_type, category=None, repo=None): + """ Core package processing logic shared by text and JSON handlers + """ + package = get_or_create_package(name, epoch, version, release, arch, p_type) + if p_type == Package.GENTOO and category: + process_gentoo_package(package, name, category, repo) + return package + + +def process_package_text(pkg): """ Processes a single sanitized package string and converts to a package object """ - if protocol == '1': - epoch = ver = rel = '' - arch = 'unknown' - - name = pkg[0] - p_category = p_repo = None - if pkg[1]: - epoch = pkg[1] - if pkg[2]: - ver = pkg[2] - if pkg[3]: - rel = pkg[3] - if pkg[4]: - arch = pkg[4] - - if pkg[5] == 'deb': - p_type = Package.DEB - elif pkg[5] == 'rpm': - p_type = Package.RPM - elif pkg[5] == 'arch': - p_type = Package.ARCH - elif pkg[5] == 'gentoo': - p_type = Package.GENTOO - p_category = pkg[6] - p_repo = pkg[7] - else: - p_type = Package.UNKNOWN + name = pkg[0] + epoch = pkg[1] if pkg[1] else '' + ver = pkg[2] if pkg[2] else '' + rel = pkg[3] if pkg[3] else '' + arch = pkg[4] if pkg[4] else 'unknown' + + p_type = _get_package_type(pkg[5]) + p_category = pkg[6] if p_type == Package.GENTOO and len(pkg) > 6 else None + p_repo = pkg[7] if p_type == Package.GENTOO and len(pkg) > 7 else None - package = get_or_create_package(name, epoch, ver, rel, arch, p_type) - if p_type == Package.GENTOO: - process_gentoo_package(package, name, p_category, p_repo) - return package + return process_package(name, epoch, ver, rel, arch, p_type, p_category, p_repo) + + +def process_package_json(pkg): + """ Processes a single JSON package dict and converts to a package object + """ + name = pkg['name'] + epoch = pkg.get('epoch', '') + ver = pkg.get('version', '') + rel = pkg.get('release', '') + arch = pkg.get('arch', 'unknown') + p_type = _get_package_type(pkg.get('type', '')) + p_category = pkg.get('category') if p_type == Package.GENTOO else None + p_repo = pkg.get('repo') if p_type == Package.GENTOO else None + + return process_package(name, epoch, ver, rel, arch, p_type, p_category, p_repo) def process_gentoo_package(package, name, category, repo): @@ -368,6 +427,143 @@ def process_gentoo_package(package, name, category, repo): package.save() +def process_packages_json(packages_json, host): + """ Processes packages from JSON data (protocol 2) + """ + package_ids = [] + pbar_start.send(sender=None, ptext=f'{host} Packages', plen=len(packages_json)) + + for i, pkg in enumerate(packages_json): + debug_message(f'Processing JSON package: {pkg}') + package = process_package_json(pkg) + if package: + package_ids.append(package.id) + host.packages.add(package) + else: + if pkg.get('name', '').lower() != 'gpg-pubkey': + info_message(text=f'No package returned for {pkg}') + pbar_update.send(sender=None, index=i + 1) + + for package in host.packages.all(): + if package.id not in package_ids: + host.packages.remove(package) + + +def process_repo_json(repo, arch): + """ Processes a single JSON repo dict and converts to a repo object + """ + r_type = _get_repo_type(repo.get('type', '')) + if r_type is None: + return None, 0 + + if r_type == Repository.GENTOO: + arch = 'any' + + r_name = repo.get('name', '') + r_id = repo.get('id', '') + r_priority = repo.get('priority', 0) + urls = repo.get('urls', []) + + # Adjust priority for RPM repos (negative) + if r_type == Repository.RPM: + r_priority = r_priority * -1 + + return process_repo(r_type, r_name, r_id, r_priority, urls, arch) + + +def process_repos_json(repos_json, host, arch): + """ Processes repos from JSON data (protocol 2) + """ + repo_ids = [] + host_repos = HostRepo.objects.filter(host=host) + + pbar_start.send(sender=None, ptext=f'{host} Repos', plen=len(repos_json)) + for i, repo in enumerate(repos_json): + debug_message(f'Processing JSON repo: {repo}') + repository, priority = process_repo_json(repo, arch) + if repository: + repo_ids.append(repository.id) + try: + hostrepo, _ = host_repos.get_or_create(host=host, repo=repository) + except IntegrityError: + hostrepo = host_repos.get(host=host, repo=repository) + if hostrepo.priority != priority: + hostrepo.priority = priority + hostrepo.save() + pbar_update.send(sender=None, index=i + 1) + + for hostrepo in host_repos: + if hostrepo.repo_id not in repo_ids: + hostrepo.delete() + + +def process_module_json(module): + """ Processes a single JSON module dict and converts to a module object + """ + m_name = module.get('name') + m_stream = module.get('stream') + m_version = module.get('version') + m_context = module.get('context') + m_arch = module.get('arch') + repo_id = module.get('repo', '') + package_strings = module.get('packages', []) + + return process_module(m_name, m_stream, m_version, m_context, m_arch, repo_id, package_strings) + + +def process_modules_json(modules_json, host): + """ Processes modules from JSON data (protocol 2) + """ + module_ids = [] + + pbar_start.send(sender=None, ptext=f'{host} Modules', plen=len(modules_json)) + for i, module in enumerate(modules_json): + mod = process_module_json(module) + if mod: + module_ids.append(mod.id) + host.modules.add(mod) + pbar_update.send(sender=None, index=i + 1) + + for mod in host.modules.all(): + if mod.id not in module_ids: + host.modules.remove(mod) + + +def process_update_json(host, update, security): + """ Processes a single JSON update dict and converts to an update object + """ + name = update.get('name') + version = update.get('version') + arch = update.get('arch') + repo_id = update.get('repo', '') + + p_epoch, p_version, p_release = find_evr(version) + + return process_update(host, name, p_epoch, p_version, p_release, arch, repo_id, security) + + +def process_updates_json(sec_updates_json, bug_updates_json, host): + """ Processes updates from JSON data (protocol 2) + """ + # Clear existing updates + for host_update in host.updates.all(): + host.updates.remove(host_update) + + # Merge updates, preferring security over bugfix + sec_keys = {(u['name'], u['arch']) for u in sec_updates_json} + bug_updates_filtered = [u for u in bug_updates_json if (u['name'], u['arch']) not in sec_keys] + + all_updates = [(u, True) for u in sec_updates_json] + [(u, False) for u in bug_updates_filtered] + + if all_updates: + pbar_start.send(sender=None, ptext=f'{host} Updates', plen=len(all_updates)) + for i, (update, security) in enumerate(all_updates): + update_obj = process_update_json(host, update, security) + if update_obj: + host.updates.add(update_obj) + pbar_update.send(sender=None, index=i + 1) + + def get_arch(arch): """ Get or create MachineArchitecture from arch Returns the MachineArchitecture diff --git a/reports/views.py b/reports/views.py index c46b7a41..ee1489f1 100644 --- a/reports/views.py +++ b/reports/views.py @@ -1,5 +1,5 @@ # Copyright 2012 VPAC, http://www.vpac.org -# Copyright 2013-2021 Marcus Furlong +# Copyright 2013-2025 Marcus Furlong # # This file is part of Patchman. # @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +import json + from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db.models import Q @@ -24,11 +26,14 @@ from django.urls import reverse from django.views.decorators.csrf import csrf_exempt from django_tables2 import RequestConfig +from rest_framework import status, viewsets +from rest_framework.response import Response from tenacity import ( retry, retry_if_exception_type, stop_after_attempt, wait_exponential, ) from reports.models import Report +from reports.serializers import ReportUploadSerializer from reports.tables import ReportTable from util import sanitize_filter_params from util.filterspecs import Filter, FilterBar @@ -149,9 +154,28 @@ def report_detail(request, report_id): report = get_object_or_404(Report, id=report_id) + context = {'report': report} + + # Add tables for Protocol 2 reports + if report.protocol == '2': + from reports.tables import ( + ReportModuleTable, ReportPackageTable, ReportRepoTable, + ReportUpdateTable, + ) + if report.has_packages: + context['packages_table'] = ReportPackageTable(report.packages_parsed) + if report.has_repos: + context['repos_table'] = ReportRepoTable(report.repos_parsed) + if report.has_modules: + context['modules_table'] = ReportModuleTable(report.modules_parsed) + if report.has_sec_updates: + context['sec_updates_table'] = ReportUpdateTable(report.sec_updates_parsed) + if report.has_bug_updates: + context['bug_updates_table'] = ReportUpdateTable(report.bug_updates_parsed) + return render(request, 'reports/report_detail.html', - {'report': report}) + context) @login_required @@ -233,3 +257,88 @@ def report_bulk_action(request): if filter_params: return redirect(f"{reverse('reports:report_list')}?{filter_params}") return redirect('reports:report_list') + + +class ReportViewSet(viewsets.ViewSet): + """ + ViewSet for protocol 2 JSON report uploads. + + POST /api/report/ - Upload a new report in JSON format + + Authentication is optional by default. Set REQUIRE_API_KEY=True in settings + to require API key authentication for report uploads. + """ + + def get_permissions(self): + from django.conf import settings + from rest_framework.permissions import AllowAny + from rest_framework_api_key.permissions import HasAPIKey + if getattr(settings, 'REQUIRE_API_KEY', False): + return [HasAPIKey()] + return [AllowAny()] + + def create(self, request): + """Handle protocol 2 JSON report upload.""" + serializer = ReportUploadSerializer(data=request.data) + if not serializer.is_valid(): + return Response( + {'status': 'error', 'errors': serializer.errors}, + status=status.HTTP_400_BAD_REQUEST + ) + + data = serializer.validated_data + + # Extract client IP + x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + x_real_ip = request.META.get('HTTP_X_REAL_IP') + if x_forwarded_for: + report_ip = x_forwarded_for.split(',')[0] + elif x_real_ip: + report_ip = x_real_ip + else: + report_ip = request.META.get('REMOTE_ADDR') + + # Extract domain from hostname + hostname = data['hostname'].lower() + domain = None + fqdn = hostname.split('.', 1) + if len(fqdn) == 2: + domain = fqdn[1] + + # Convert tags list to comma-separated string + tags = ','.join(data.get('tags', [])) + + # Convert reboot_required to string for compatibility + reboot = 'True' if data.get('reboot_required') else 'False' + + # Store JSON data as strings in the report model + report = Report.objects.create( + host=hostname, + domain=domain, + tags=tags, + kernel=data['kernel'], + arch=data['arch'], + os=data['os'], + report_ip=report_ip, + protocol='2', + useragent=request.META.get('HTTP_USER_AGENT', ''), + packages=json.dumps(data.get('packages', [])), + repos=json.dumps(data.get('repos', [])), + modules=json.dumps(data.get('modules', [])), + sec_updates=json.dumps(data.get('sec_updates', [])), + bug_updates=json.dumps(data.get('bug_updates', [])), + reboot=reboot, + ) + + # Queue for async processing + from reports.tasks import process_report + process_report.delay(report.id) + + return Response( + { + 'status': 'accepted', + 'report_id': report.id, + 'message': 'Report queued for processing' + }, + status=status.HTTP_202_ACCEPTED + ) diff --git a/requirements.txt b/requirements.txt index 3101fac5..b85c0243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ PyYAML==6.0.2 requests==2.32.4 colorama==0.4.6 djangorestframework==3.15.2 +djangorestframework-api-key==3.0.0 django-filter==25.1 humanize==4.12.1 version-utils==0.3.2 diff --git a/util/management/__init__.py b/util/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/util/management/commands/__init__.py b/util/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/util/management/commands/create_api_key.py b/util/management/commands/create_api_key.py new file mode 100644 index 00000000..4cca1397 --- /dev/null +++ b/util/management/commands/create_api_key.py @@ -0,0 +1,43 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.core.management.base import BaseCommand +from rest_framework_api_key.models import APIKey + + +class Command(BaseCommand): + help = 'Create a new API key for protocol 2 report uploads' + + def add_arguments(self, parser): + parser.add_argument( + 'name', + type=str, + help='Descriptive name for this API key (e.g., "webserver-cluster")' + ) + + def handle(self, *args, **options): + name = options['name'] + + api_key, key = APIKey.objects.create_key(name=name) + + self.stdout.write(self.style.SUCCESS(f'Created API key: {name}')) + self.stdout.write('') + self.stdout.write(f' Key: {key}') + self.stdout.write('') + self.stdout.write('Add this to your patchman-client.conf:') + self.stdout.write(f' api_key={key}') + self.stdout.write('') + self.stdout.write(self.style.WARNING('Save this key - it cannot be retrieved later!')) diff --git a/util/management/commands/list_api_keys.py b/util/management/commands/list_api_keys.py new file mode 100644 index 00000000..fd77c364 --- /dev/null +++ b/util/management/commands/list_api_keys.py @@ -0,0 +1,52 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.core.management.base import BaseCommand +from rest_framework_api_key.models import APIKey + + +class Command(BaseCommand): + help = 'List all API keys' + + def add_arguments(self, parser): + parser.add_argument( + '--all', + action='store_true', + help='Show all keys including revoked ones' + ) + + def handle(self, *args, **options): + if options['all']: + api_keys = APIKey.objects.all() + else: + api_keys = APIKey.objects.filter(revoked=False) + + if not api_keys.exists(): + self.stdout.write('No API keys found.') + return + + self.stdout.write('') + self.stdout.write(f'{"Name":<30} {"Prefix":<12} {"Created":<20} {"Revoked":<8}') + self.stdout.write('-' * 72) + + for key in api_keys: + created = key.created.strftime('%Y-%m-%d %H:%M') + revoked = 'Yes' if key.revoked else 'No' + + self.stdout.write(f'{key.name:<30} {key.prefix:<12} {created:<20} {revoked:<8}') + + self.stdout.write('') + self.stdout.write(f'Total: {api_keys.count()} key(s)') diff --git a/util/management/commands/revoke_api_key.py b/util/management/commands/revoke_api_key.py new file mode 100644 index 00000000..d22955e7 --- /dev/null +++ b/util/management/commands/revoke_api_key.py @@ -0,0 +1,63 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.core.management.base import BaseCommand, CommandError +from rest_framework_api_key.models import APIKey + + +class Command(BaseCommand): + help = 'Revoke an API key' + + def add_arguments(self, parser): + parser.add_argument( + 'key_or_prefix', + type=str, + help='The API key prefix or name to revoke' + ) + parser.add_argument( + '--delete', + action='store_true', + help='Permanently delete the key instead of just revoking' + ) + + def handle(self, *args, **options): + key_input = options['key_or_prefix'] + delete = options['delete'] + + # Try to find by prefix first, then by name + api_keys = APIKey.objects.filter(prefix=key_input) + if not api_keys.exists(): + api_keys = APIKey.objects.filter(name=key_input) + + if api_keys.count() == 0: + raise CommandError(f'No API key found matching: {key_input}') + elif api_keys.count() > 1: + raise CommandError(f'Multiple keys match "{key_input}". Please be more specific.') + + api_key = api_keys.first() + + if delete: + name = api_key.name + api_key.delete() + self.stdout.write(self.style.SUCCESS(f'Permanently deleted API key: {name}')) + else: + if api_key.revoked: + self.stdout.write(self.style.WARNING(f'API key "{api_key.name}" is already revoked')) + return + + api_key.revoked = True + api_key.save() + self.stdout.write(self.style.SUCCESS(f'Revoked API key: {api_key.name}'))