diff --git a/.github/workflows/dev_workflow_func_app.yaml b/.github/workflows/dev_workflow_func_app.yaml new file mode 100644 index 0000000..b1beb19 --- /dev/null +++ b/.github/workflows/dev_workflow_func_app.yaml @@ -0,0 +1,105 @@ +--- +# Dev Workflow – build image and deploy to Azure Function App (Development environment). +# +# Required secrets (Settings → Environments → Development → Environment secrets): +# - REGISTRY_DOMAIN – Azure Container Registry login server (e.g. myregistry.azurecr.io) +# - REGISTRY_USERNAME – ACR username +# - REGISTRY_PASSWORD – ACR password +# - REGISTRY_REPO – Repository name in ACR for this app +# - TDEI_CORE_AZURE_CREDS – Azure service principal JSON (for az login) +# +# Required variables (Settings → Environments → Development → Environment variables): +# - FUNCTION_APP_NAME – Azure Function App name to deploy to +# - RESOURCE_GROUP – Azure resource group containing the Function App +# +# Optional variables (defaults used if not set): +# - RESTART_APP – Set to 'true' or 'false'; default 'true' +# - APP_SETTINGS_JSON – JSON object of extra app settings to apply; default '{}' +# +######### Dev Workflow ######## +on: + pull_request: + branches: [dev] + types: + - closed + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + Build: + environment: Development + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'main' && 'prod' || github.ref_name }}${{ github.ref_name != 'main' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + deploy: + environment: Development + runs-on: ubuntu-latest + needs: [Build] + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{ secrets.TDEI_CORE_AZURE_CREDS }} + + - name: Resolve deploy config from environment + id: deploy_config + run: | + echo "function_app_name=${{ vars.FUNCTION_APP_NAME }}" >> "$GITHUB_OUTPUT" + echo "resource_group=${{ vars.RESOURCE_GROUP }}" >> "$GITHUB_OUTPUT" + echo "aci_image=${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}" >> "$GITHUB_OUTPUT" + echo "restart_app=${{ vars.RESTART_APP || 'true' }}" >> "$GITHUB_OUTPUT" + + - name: Log target environment + shell: bash + run: | + echo "Deploying to:" + echo " Function App: ${{ steps.deploy_config.outputs.function_app_name }}" + echo " Resource Group: ${{ steps.deploy_config.outputs.resource_group }}" + echo " ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }}" + + - name: Update app settings (ACI_IMAGE + extras) + shell: bash + run: | + python - <<'PY' + import json + import os + app_settings = os.environ.get("APP_SETTINGS_JSON", "{}") + data = json.loads(app_settings) if app_settings else {} + data["ACI_IMAGE"] = os.environ["ACI_IMAGE"] + with open("/tmp/appsettings.txt", "w", encoding="utf-8") as handle: + for key, value in data.items(): + handle.write(f"{key}={value}\n") + PY + echo "Updating only provided settings (no clearing of others)." + az functionapp config appsettings set \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ + --settings $(cat /tmp/appsettings.txt | tr '\n' ' ') + env: + ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }} + APP_SETTINGS_JSON: ${{ vars.APP_SETTINGS_JSON || '{}' }} + + - name: Restart function app + if: ${{ steps.deploy_config.outputs.restart_app == 'true' }} + shell: bash + run: | + az functionapp restart \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ No newline at end of file diff --git a/.github/workflows/prod_workflow_func_app.yaml b/.github/workflows/prod_workflow_func_app.yaml new file mode 100644 index 0000000..923c11f --- /dev/null +++ b/.github/workflows/prod_workflow_func_app.yaml @@ -0,0 +1,105 @@ +--- +# Prod Workflow – build image and deploy to Azure Function App (Production environment). +# +# Required secrets (Settings → Environments → Production → Environment secrets): +# - REGISTRY_DOMAIN – Azure Container Registry login server (e.g. myregistry.azurecr.io) +# - REGISTRY_USERNAME – ACR username +# - REGISTRY_PASSWORD – ACR password +# - REGISTRY_REPO – Repository name in ACR for this app +# - TDEI_CORE_AZURE_CREDS – Azure service principal JSON (for az login) +# +# Required variables (Settings → Environments → Production → Environment variables): +# - FUNCTION_APP_NAME – Azure Function App name to deploy to +# - RESOURCE_GROUP – Azure resource group containing the Function App +# +# Optional variables (defaults used if not set): +# - RESTART_APP – Set to 'true' or 'false'; default 'true' +# - APP_SETTINGS_JSON – JSON object of extra app settings to apply; default '{}' +# +######### Prod Workflow ######## +on: + pull_request: + branches: [main] + types: + - closed + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + Build: + environment: Production + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'main' && 'prod' || github.ref_name }}${{ github.ref_name != 'main' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + deploy: + environment: Production + runs-on: ubuntu-latest + needs: [Build] + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{ secrets.TDEI_CORE_AZURE_CREDS }} + + - name: Resolve deploy config from environment + id: deploy_config + run: | + echo "function_app_name=${{ vars.FUNCTION_APP_NAME }}" >> "$GITHUB_OUTPUT" + echo "resource_group=${{ vars.RESOURCE_GROUP }}" >> "$GITHUB_OUTPUT" + echo "aci_image=${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}" >> "$GITHUB_OUTPUT" + echo "restart_app=${{ vars.RESTART_APP || 'true' }}" >> "$GITHUB_OUTPUT" + + - name: Log target environment + shell: bash + run: | + echo "Deploying to:" + echo " Function App: ${{ steps.deploy_config.outputs.function_app_name }}" + echo " Resource Group: ${{ steps.deploy_config.outputs.resource_group }}" + echo " ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }}" + + - name: Update app settings (ACI_IMAGE + extras) + shell: bash + run: | + python - <<'PY' + import json + import os + app_settings = os.environ.get("APP_SETTINGS_JSON", "{}") + data = json.loads(app_settings) if app_settings else {} + data["ACI_IMAGE"] = os.environ["ACI_IMAGE"] + with open("/tmp/appsettings.txt", "w", encoding="utf-8") as handle: + for key, value in data.items(): + handle.write(f"{key}={value}\n") + PY + echo "Updating only provided settings (no clearing of others)." + az functionapp config appsettings set \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ + --settings $(cat /tmp/appsettings.txt | tr '\n' ' ') + env: + ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }} + APP_SETTINGS_JSON: ${{ vars.APP_SETTINGS_JSON || '{}' }} + + - name: Restart function app + if: ${{ steps.deploy_config.outputs.restart_app == 'true' }} + shell: bash + run: | + az functionapp restart \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ No newline at end of file diff --git a/.github/workflows/stage_workflow_func_app.yaml b/.github/workflows/stage_workflow_func_app.yaml new file mode 100644 index 0000000..07b0e76 --- /dev/null +++ b/.github/workflows/stage_workflow_func_app.yaml @@ -0,0 +1,105 @@ +--- +# Stage Workflow – build image and deploy to Azure Function App (Stage environment). +# +# Required secrets (Settings → Environments → Stage → Environment secrets): +# - REGISTRY_DOMAIN – Azure Container Registry login server (e.g. myregistry.azurecr.io) +# - REGISTRY_USERNAME – ACR username +# - REGISTRY_PASSWORD – ACR password +# - REGISTRY_REPO – Repository name in ACR for this app +# - TDEI_CORE_AZURE_CREDS – Azure service principal JSON (for az login) +# +# Required variables (Settings → Environments → Stage → Environment variables): +# - FUNCTION_APP_NAME – Azure Function App name to deploy to +# - RESOURCE_GROUP – Azure resource group containing the Function App +# +# Optional variables (defaults used if not set): +# - RESTART_APP – Set to 'true' or 'false'; default 'true' +# - APP_SETTINGS_JSON – JSON object of extra app settings to apply; default '{}' +# +######### Stage Workflow ######## +on: + pull_request: + branches: [stage] + types: + - closed + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + Build: + environment: Stage + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + steps: + - uses: actions/checkout@v2 + - uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.REGISTRY_DOMAIN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Publish image to Azure Registry + run: | + docker build -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }} -t ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.ref_name == 'main' && 'prod' || github.ref_name }}${{ github.ref_name != 'main' && '-latest' || 'latest' }} . + docker push ${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }} --all-tags + deploy: + environment: Stage + runs-on: ubuntu-latest + needs: [Build] + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{ secrets.TDEI_CORE_AZURE_CREDS }} + + - name: Resolve deploy config from environment + id: deploy_config + run: | + echo "function_app_name=${{ vars.FUNCTION_APP_NAME }}" >> "$GITHUB_OUTPUT" + echo "resource_group=${{ vars.RESOURCE_GROUP }}" >> "$GITHUB_OUTPUT" + echo "aci_image=${{ secrets.REGISTRY_DOMAIN }}/${{ secrets.REGISTRY_REPO }}:${{ github.sha }}" >> "$GITHUB_OUTPUT" + echo "restart_app=${{ vars.RESTART_APP || 'true' }}" >> "$GITHUB_OUTPUT" + + - name: Log target environment + shell: bash + run: | + echo "Deploying to:" + echo " Function App: ${{ steps.deploy_config.outputs.function_app_name }}" + echo " Resource Group: ${{ steps.deploy_config.outputs.resource_group }}" + echo " ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }}" + + - name: Update app settings (ACI_IMAGE + extras) + shell: bash + run: | + python - <<'PY' + import json + import os + app_settings = os.environ.get("APP_SETTINGS_JSON", "{}") + data = json.loads(app_settings) if app_settings else {} + data["ACI_IMAGE"] = os.environ["ACI_IMAGE"] + with open("/tmp/appsettings.txt", "w", encoding="utf-8") as handle: + for key, value in data.items(): + handle.write(f"{key}={value}\n") + PY + echo "Updating only provided settings (no clearing of others)." + az functionapp config appsettings set \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ + --settings $(cat /tmp/appsettings.txt | tr '\n' ' ') + env: + ACI_IMAGE: ${{ steps.deploy_config.outputs.aci_image }} + APP_SETTINGS_JSON: ${{ vars.APP_SETTINGS_JSON || '{}' }} + + - name: Restart function app + if: ${{ steps.deploy_config.outputs.restart_app == 'true' }} + shell: bash + run: | + az functionapp restart \ + --name "${{ steps.deploy_config.outputs.function_app_name }}" \ + --resource-group "${{ steps.deploy_config.outputs.resource_group }}" \ No newline at end of file diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 8edbc9c..9a1b821 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -47,16 +47,28 @@ jobs: exit 1 fi - - name: Run tests with coverage + - name: Run tests with coverage (show failures in logs) + shell: bash run: | - timestamp=$(date '+%Y-%m-%d_%H-%M-%S') + set -o pipefail + timestamp="$(date '+%Y-%m-%d_%H-%M-%S')" mkdir -p test_results log_file="test_results/${timestamp}_report.log" - echo -e "\nTest Cases Report Report\n" >> $log_file - # Run the tests and append output to the log file - python -m coverage run --source=src -m unittest discover -s tests >> $log_file 2>&1 - echo -e "\nCoverage Report\n" >> $log_file - coverage report >> $log_file + + { + echo + echo "Test Cases Report" + echo + } | tee -a "$log_file" + + # Run unittest in verbose mode; mirror output to console and file + python -m coverage run --source=src -m unittest discover -s tests -v 2>&1 | tee -a "$log_file" + test_status=${PIPESTATUS[0]} + + echo -e "\nCoverage Report\n" | tee -a "$log_file" + coverage report 2>&1 | tee -a "$log_file" + + exit $test_status - name: Check coverage run: | diff --git a/README.md b/README.md index 0f1bff7..d4e8fd8 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,40 @@ # TDEI-python-osw-incline -Service that accepts requests to add incline to the osw dataset - -## Introduction -Service to adds the inclination to the existing edges.geojson file, the service does the following: -- Listens to the topic which is mentioned in `.env` file for any new message, example `REQUEST_TOPIC=test_request` -- Consumes the message and perform following checks - - - Download the zip file locally - - unzip it and read the edges.geojson file - - Calculate the edge geometry boundary box - - Download the DEM file from NED 1/3 arc-second - - Process the DEM file and calculate the inclination (This action is done by `osw-incline` package) - - Add the inclination to the edges.geojson file -- Publishes the result to the topic mentioned in `.env` file, example `RESPONSE_TOPIC=test_response` +Service that accepts queue requests and adds incline values to an OSW dataset. + +## What It Does +- Listens to `REQUEST_TOPIC` / `REQUEST_SUBSCRIPTION`. +- Downloads the input OSW zip from `data.dataset_url`. +- Extracts files and reads `edges` + `nodes` GeoJSON. +- Resolves DEM tiles (NED 1/3 arc-second), computes incline via `osw-incline`. +- Updates edges with incline values. +- Creates a result zip and uploads to storage. +- Publishes success/failure status to `RESPONSE_TOPIC`. + +## Actual Use Cases +- Enriching street/path networks with incline for accessibility routing. +- Batch processing city-scale OSW datasets through queue-driven jobs. +- Running one-message-per-container jobs in CI/data pipelines, then auto-stopping the service. +- Processing very large GeoJSON inputs where zip validation is required before upload. + +## Processing Lifecycle +1. Request received from queue. +2. Request validated (`dataset_url` must exist). +3. Incline processing executed. +4. Output uploaded to storage. +5. Response message published (with `success`, `message`, `file_upload_path`, package versions). +6. Temporary files cleaned up. +7. If `MAX_RECEIVABLE_MESSAGES > 0`, service triggers graceful shutdown after processing available messages. ## Getting Started -The project is built on Python with FastAPI framework. All the regular nuances for a Python project are valid for this. +The project is built with Python + FastAPI. ### System requirements | Software | Version | |------------|---------| -| Python | 3.10.x | +| Python | 3.10+ | -### Connectivity to cloud -- Connecting this to cloud will need the following in the `.env` file - +### Required `.env` configuration ```bash PROVIDER=Azure QUEUECONNECTION=xxx @@ -33,80 +43,78 @@ REQUEST_TOPIC=xxx REQUEST_SUBSCRIPTION=xxx RESPONSE_TOPIC=xxx CONTAINER_NAME=xxx -MAX_CONCURRENT_MESSAGES=xxx # Optional if not provided defaults to 2 +MAX_CONCURRENT_MESSAGES=1 # Optional, default: 1 +MAX_RECEIVABLE_MESSAGES=-1 # Optional, default: -1 (no receive limit) ``` -The application connect with the `STORAGECONNECTION` string provided in `.env` file and validates downloaded zipfile using `python-osw-validation` package. -`QUEUECONNECTION` is used to send out the messages and listen to messages. - -`MAX_CONCURRENT_MESSAGES` is the maximum number of concurrent messages that the service can handle. If not provided, defaults to 1 +`QUEUECONNECTION` is used for queue consume/publish. +`STORAGECONNECTION` is used to download input and upload processed zip. ### How to Set up and Build -Follow the steps to install the python packages required for both building and running the application - -1. Setup virtual environment - ``` - python3.10 -m venv .venv - source .venv/bin/activate - ``` - -2. Install the dependencies. Run the following command in terminal on the same directory as `requirements.txt` - ``` - # Installing requirements - pip install -r requirements.txt - ``` -### How to Run the Server/APIs - -1. The http server by default starts with `8000` port -2. Run server - ``` - uvicorn src.main:app --reload - ``` -3. By default `get` call on `localhost:8000/health` gives a sample response -4. Other routes include a `ping` with get and post. Make `get` or `post` request to `http://localhost:8000/health/ping` -5. Once the server starts, it will start to listening the subscriber(`REQUEST_SUBSCRIPTION` should be in env file) +1. Setup virtual environment: +```bash +python3.10 -m venv .venv +source .venv/bin/activate +``` +2. Install dependencies: +```bash +pip install -r requirements.txt +``` + +### How to Run +```bash +uvicorn src.main:app --reload +``` + +Health endpoints: +- `GET /` +- `GET /health` +- `GET|POST /ping` +- `GET|POST /health/ping` #### Request Format ```json - { - "messageId": "tdei_record_id", - "messageType": "workflow_identifier", - "data": { - "file_url": "file_upload_path" - } +{ + "messageId": "tdei_record_id", + "messageType": "workflow_identifier", + "data": { + "dataset_url": "https://.../input_osw.zip", + "user_id": "optional", + "jobId": "optional-job-id" } +} ``` #### Response Format ```json - { - "messageId": "tdei_record_id", - "messageType": "workflow_identifier", - "data": { - "file_url": "file_upload_path", - "updated_file_url": "file_upload_path", // updated file url whicjh contains the inclination added - "success": true/false, - "status": "Success/Failed" - }, - "publishedDate": "published date" +{ + "messageId": "tdei_record_id", + "messageType": "workflow_identifier", + "data": { + "message": "Successfully added inclination to the dataset.", + "success": true, + "file_upload_path": "https://.../jobs//.zip", + "package": { + "python-ms-core": "x.y.z", + "osw-incline": "x.y.z" + } } +} ``` +On failure, `data.success=false` and `data.message` contains the error reason. ### How to Set up and run the Tests +`.env` is not required for unit tests. -Make sure you have set up the project properly before running the tests, see above for `How to Setup and Build`. - - -#### How to run unit test cases -1. `.env` file is not required for Unit test cases. -2To run the coverage - 1. `python -m coverage run --source=src -m unittest discover -s tests` - 2. Above command will run all the unit test cases. - 3. To generate the coverage report in console - 1. `coverage report` - 2. Above command will generate the code coverage report in terminal. - 4. To generate the coverage report in html. - 1. `coverage html` - 2. Above command will generate the html report, and generated html would be in `htmlcov` directory at the root level. - 5. _NOTE :_ To run the `html` or `report` coverage, 3.i) command is mandatory +Run tests: +```bash +python -m unittest discover -s tests +``` + +Run coverage: +```bash +python -m coverage run --source=src -m unittest discover -s tests +coverage report +coverage html +``` diff --git a/requirements.txt b/requirements.txt index 9c6dff6..880dc6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ python-ms-core==0.0.25 fastapi uvicorn pydantic-settings -osw-incline~=0.0.4 \ No newline at end of file +osw-incline~=0.0.4 +psutil diff --git a/src/config.py b/src/config.py index 83cb5a4..b34e7ce 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,11 @@ import os from typing import ClassVar from dotenv import load_dotenv -from pydantic_settings import BaseSettings +try: + from pydantic_settings import BaseSettings +except ModuleNotFoundError: + class BaseSettings: # pragma: no cover - compatibility fallback for test environments + pass load_dotenv() @@ -17,7 +21,8 @@ class EventBusSettings: class Settings(BaseSettings): app_name: str = 'python-osw-inclination' event_bus: ClassVar[EventBusSettings] = EventBusSettings() # Annotate event_bus as a ClassVar - max_concurrent_messages: int = int(os.environ.get('MAX_CONCURRENT_MESSAGES', 2)) # Convert to int + max_concurrent_messages: int = int(os.environ.get('MAX_CONCURRENT_MESSAGES', 1)) # Convert to int + max_receivable_messages: int = int(os.environ.get('MAX_RECEIVABLE_MESSAGES', 1)) # -1 means no limit def get_root_directory(self) -> str: return os.path.dirname(os.path.abspath(__file__)) @@ -25,4 +30,4 @@ def get_root_directory(self) -> str: def get_download_directory(self) -> str: root_dir = self.get_root_directory() parent_dir = os.path.dirname(root_dir) - return os.path.join(parent_dir, 'downloads') \ No newline at end of file + return os.path.join(parent_dir, 'downloads') diff --git a/src/inclination_helper/inclination.py b/src/inclination_helper/inclination.py index 0abd34a..d088d8d 100644 --- a/src/inclination_helper/inclination.py +++ b/src/inclination_helper/inclination.py @@ -1,12 +1,14 @@ import os -import gc import json from pathlib import Path from src.logger import Logger from src.config import Settings from python_ms_core import Core from urllib.parse import urlparse -from osw_incline import OSWIncline +try: + from osw_incline import OSWIncline +except ModuleNotFoundError: + OSWIncline = None from shapely.geometry import shape from src.inclination_helper.dem_downloader import DEMDownloader from src.inclination_helper.utils import get_unique_id, unzip, create_zip @@ -63,6 +65,8 @@ def calculate(self): tile_sets = dem_downloader.list_ned13s_full_paths() Logger.info(f'No of NED13 files: {len(tile_sets)} to be processed') + if OSWIncline is None: + raise ModuleNotFoundError('osw_incline is not installed') dem_processor = OSWIncline( dem_files=tile_sets, nodes_file=str(graph_nodes_path), @@ -76,8 +80,8 @@ def calculate(self): files=all_files, zip_file_path=os.path.join(self.download_dir, f'{self.prefix}/{self.updated_file_name}') ) - - gc.collect() + if os.path.getsize(zip_file_path) == 0: + raise ValueError(f'Created zip file is empty: {zip_file_path}') return zip_file_path diff --git a/src/inclination_helper/utils.py b/src/inclination_helper/utils.py index aa5d287..f1e11c9 100644 --- a/src/inclination_helper/utils.py +++ b/src/inclination_helper/utils.py @@ -39,10 +39,24 @@ def clean_up(path, download_dir=None): def create_zip(files, zip_file_path): - with zipfile.ZipFile(zip_file_path, 'w') as zip_file: + added_files = 0 + with zipfile.ZipFile( + zip_file_path, + 'w', + compression=zipfile.ZIP_DEFLATED, + allowZip64=True + ) as zip_file: for file in files: - if not os.path.isdir(file): + if os.path.isfile(file): # Add each file to the zip file zip_file.write(file, os.path.basename(file)) + added_files += 1 + + if added_files == 0: + raise ValueError(f'No files were added to zip: {zip_file_path}') + + with zipfile.ZipFile(zip_file_path, 'r') as zip_file: + if len(zip_file.namelist()) == 0: + raise ValueError(f'Created zip is empty: {zip_file_path}') return zip_file_path diff --git a/src/main.py b/src/main.py index bb7f1d0..26644b5 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,4 @@ import os -import psutil from src.config import Settings from functools import lru_cache from fastapi import FastAPI, APIRouter, Depends, status @@ -38,11 +37,15 @@ async def startup_event(settings: Settings = Depends(get_settings)) -> None: print('\x1b[31m RESPONSE_TOPIC=xxxx \x1b[0m') print('\x1b[31m QUEUECONNECTION=xxxx \x1b[0m') print('\x1b[31m STORAGECONNECTION=xxxx \x1b[0m \n\n') - parent_pid = os.getpid() - parent = psutil.Process(parent_pid) - for child in parent.children(recursive=True): - child.kill() - parent.kill() + try: + import psutil + parent_pid = os.getpid() + parent = psutil.Process(parent_pid) + for child in parent.children(recursive=True): + child.kill() + parent.kill() + except ModuleNotFoundError: + os._exit(1) @app.on_event('shutdown') @@ -66,4 +69,4 @@ def ping(): return "I'm healthy !!" -app.include_router(prefix_router) \ No newline at end of file +app.include_router(prefix_router) diff --git a/src/services/inclination_service.py b/src/services/inclination_service.py index 6e5fdc1..0655547 100644 --- a/src/services/inclination_service.py +++ b/src/services/inclination_service.py @@ -1,7 +1,13 @@ import os -import gc +import time +import signal import threading -import osw_incline +import urllib.parse +try: + import osw_incline + _OSW_INCLINE_VERSION = osw_incline.__version__ +except ModuleNotFoundError: + _OSW_INCLINE_VERSION = 'unknown' from src.logger import Logger from python_ms_core import Core from src.config import Settings @@ -15,6 +21,10 @@ class InclinationService: _config = Settings() def __init__(self): + # Keep lifecycle same as osw-validation; only force callback worker mode to + # thread by default to avoid subprocess worker exits in this runtime. + os.environ.setdefault('TOPIC_CALLBACK_EXECUTION_MODE', 'thread') + self.core = Core() self._subscription_name = self._config.event_bus.request_subscription self.request_topic = self.core.get_topic( @@ -23,7 +33,8 @@ def __init__(self): ) self.storage_client = self.core.get_storage_client() self.container_name = self._config.event_bus.container_name - self.listening_thread = threading.Thread(target=self.subscribe) + self._shutdown_triggered = threading.Event() + self.listening_thread = threading.Thread(target=self.subscribe, daemon=True) self.listening_thread.start() def subscribe(self) -> None: @@ -36,53 +47,90 @@ def process(message) -> None: else: Logger.info(' No Message') - self.request_topic.subscribe(subscription=self._subscription_name, callback=process) + self.request_topic.subscribe( + subscription=self._subscription_name, + callback=process, + max_receivable_messages=self._config.max_receivable_messages + ) + if self._config.max_receivable_messages > 0: + Logger.info('Listener finished processing available messages; stopping server/container.') + self._stop_server_and_container(delay_seconds=2) def process_message(self, request_msg: RequestMessage) -> None: prefix = request_msg.data.jobId if request_msg.data.jobId else get_unique_id() file_path = request_msg.data.dataset_url - inclination = None + status_sent = False try: - Logger.info(f' Message ID: {request_msg.messageId}') - is_valid = True + tdei_record_id = request_msg.messageId + Logger.info( + f'Received message for: {tdei_record_id}. ' + f'Message received for OSW Incline! Core version: {Core.__version__}' + ) if file_path is None: - Logger.warning(' No file path found in the request!') - is_valid = False - else: + error_msg = 'Request does not have valid file path specified.' + Logger.error(f'{tdei_record_id}, {error_msg} !') + raise Exception(error_msg) + + file_upload_path = urllib.parse.unquote(request_msg.data.dataset_url) + if file_upload_path: inclination = Inclination( - file_path=file_path, + file_path=file_upload_path, storage_client=self.storage_client, prefix=prefix ) file_path = inclination.calculate() Logger.info(f' Calculated inclination for file: {file_path}') if file_path: - file_path = self.upload_to_azure( + uploaded_file_path = self.upload_to_azure( file_path=file_path, job_id=prefix ) + if uploaded_file_path: + file_path = uploaded_file_path + else: + raise Exception('Failed to upload processed file to storage') else: - is_valid = False - file_path = request_msg.data.dataset_url + raise Exception('Failed to generate processed file') + else: + raise Exception('File entity not found') - self.send_status(valid=is_valid, request_message=request_msg, file_path=file_path) + self.send_status(valid=True, request_message=request_msg, file_path=file_path) + status_sent = True except Exception as e: Logger.error(f' Error: {e}') - self.send_status(valid=False, request_message=request_msg, file_path=file_path) + self.send_status( + valid=False, + request_message=request_msg, + file_path=file_path, + upload_message=f'Error: {e}' + ) + status_sent = True finally: + if status_sent: + Logger.info(f'Validation status sent for {prefix}.') + else: + Logger.warning(f'Validation status was not sent for {prefix}.') Logger.info(f' Cleaning up files with prefix: {prefix}') clean_up(path=f'{self._config.get_download_directory()}/{prefix}') - del inclination - gc.collect() - def send_status(self, valid: bool, request_message: RequestMessage, file_path: str) -> None: + def send_status( + self, + valid: bool, + request_message: RequestMessage, + file_path: str, + upload_message: str = None + ) -> None: response_message = { - 'message': 'Success' if valid else 'Failed', + 'message': ( + upload_message + if upload_message + else 'Successfully added inclination to the dataset.' + ), 'success': valid, 'file_upload_path': file_path, 'package': { 'python-ms-core': Core.__version__, - 'osw-incline': osw_incline.__version__ + 'osw-incline': _OSW_INCLINE_VERSION } } Logger.info( @@ -97,12 +145,17 @@ def send_status(self, valid: bool, request_message: RequestMessage, file_path: s return def stop_listening(self): - self.listening_thread.join(timeout=0) - return + self._stop_server_and_container() + if hasattr(self, 'listening_thread'): + self.listening_thread.join(timeout=0) def upload_to_azure(self, job_id: str, file_path=None): Logger.info(f' Uploading file to Azure: {file_path}') try: + file_size = os.path.getsize(file_path) + if file_size == 0: + raise ValueError(f'File is empty, cannot upload: {file_path}') + target_directory = f'jobs/{job_id}' target_file_remote_path = f'{target_directory}/{os.path.basename(file_path)}' @@ -111,6 +164,7 @@ def upload_to_azure(self, job_id: str, file_path=None): ) file = container.create_file(name=target_file_remote_path) with open(file_path, 'rb') as data: + data.seek(0) file.upload(data) uploaded_path = file.get_remote_url() Logger.info(f' File uploaded to Azure: {uploaded_path}') @@ -118,3 +172,28 @@ def upload_to_azure(self, job_id: str, file_path=None): except Exception as e: Logger.error(f' Error: {e}') return None + + def _stop_server_and_container(self, delay_seconds: float = 0.0): + """ + Attempt to gracefully stop the current process (stopping FastAPI/uvicorn and the Docker container). + """ + Logger.info('Gracefully stopping FastAPI/uvicorn and Docker container') + if self._shutdown_triggered.is_set(): + Logger.info('Server stop already in progress; skipping duplicate trigger.') + return + self._shutdown_triggered.set() + Logger.info('Server stop triggered; scheduling shutdown.') + + def _terminate(): + if delay_seconds: + time.sleep(delay_seconds) + try: + Logger.info('Sending SIGTERM to stop server/container.') + os.kill(os.getpid(), signal.SIGTERM) + except Exception as err: + Logger.warning(f'Error occurred while sending SIGTERM: {err}') + finally: + Logger.info('Forcing process exit to stop server/container.') + os._exit(0) + + threading.Thread(target=_terminate, daemon=True).start() diff --git a/tests/inclination_helper/test_inclination.py b/tests/inclination_helper/test_inclination.py index 97e6d94..429ddb0 100644 --- a/tests/inclination_helper/test_inclination.py +++ b/tests/inclination_helper/test_inclination.py @@ -34,6 +34,7 @@ def test_inclination_init(self, mock_core, mock_makedirs, mock_exists, mock_urlp mock_core.return_value.get_storage_client.assert_called_once() @patch('src.inclination_helper.inclination.open', new_callable=mock_open) + @patch('src.inclination_helper.inclination.os.path.getsize', return_value=10) @patch('src.inclination_helper.inclination.create_zip') @patch('src.inclination_helper.inclination.OSWIncline') @patch('src.inclination_helper.inclination.DEMDownloader') @@ -41,7 +42,7 @@ def test_inclination_init(self, mock_core, mock_makedirs, mock_exists, mock_urlp @patch('src.inclination_helper.inclination.unzip') @patch('src.inclination_helper.inclination.Core') def test_calculate_inclination(self, mock_core, mock_unzip, mock_path, mock_dem_downloader, mock_osw_incline, - mock_create_zip, mock_open): + mock_create_zip, mock_getsize, mock_open): # Arrange # Mock for the 'ned_13_index.json' file ned_13_index_content = json.dumps({"tiles": ["tile1", "tile2"]}) diff --git a/tests/inclination_helper/test_utils.py b/tests/inclination_helper/test_utils.py index a46f482..4761117 100644 --- a/tests/inclination_helper/test_utils.py +++ b/tests/inclination_helper/test_utils.py @@ -1,4 +1,5 @@ import os +import zipfile import unittest from unittest.mock import patch, call from src.inclination_helper.utils import get_unique_id, unzip, clean_up, create_zip @@ -72,17 +73,22 @@ def test_create_zip(self, mock_zipfile): # Arrange files = ['file1.txt', 'file2.txt'] zip_file_path = 'output.zip' - mock_zip = mock_zipfile.return_value.__enter__.return_value + mock_zip_writer = mock_zipfile.return_value.__enter__.return_value + mock_zip_reader = mock_zipfile.return_value.__enter__.return_value + mock_zip_reader.namelist.return_value = ['file1.txt', 'file2.txt'] # Act - result = create_zip(files, zip_file_path) + with patch('src.inclination_helper.utils.os.path.isfile', return_value=True): + result = create_zip(files, zip_file_path) # Assert self.assertEqual(result, zip_file_path) - mock_zipfile.assert_called_once_with(zip_file_path, 'w') + self.assertEqual(mock_zipfile.call_count, 2) + mock_zipfile.assert_any_call(zip_file_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) + mock_zipfile.assert_any_call(zip_file_path, 'r') # Check that write was called with the correct arguments - mock_zip.write.assert_has_calls([ + mock_zip_writer.write.assert_has_calls([ call('file1.txt', 'file1.txt'), call('file2.txt', 'file2.txt') ], any_order=True) diff --git a/tests/services/test_inclination_service.py b/tests/services/test_inclination_service.py index 2eec345..9bf5195 100644 --- a/tests/services/test_inclination_service.py +++ b/tests/services/test_inclination_service.py @@ -1,5 +1,6 @@ import unittest from unittest.mock import patch, MagicMock, mock_open +import signal from src.services.inclination_service import InclinationService @@ -13,6 +14,7 @@ def setUp(self, mock_core, mock_settings): mock_settings.return_value.event_bus.request_topic = 'test_request_topic' mock_settings.return_value.event_bus.response_topic = 'test_response_topic' mock_settings.return_value.max_concurrent_messages = 10 + mock_settings.return_value.max_receivable_messages = -1 mock_settings.return_value.get_download_directory.return_value = '/tmp' mock_settings.return_value.event_bus.container_name = 'test_container' @@ -42,6 +44,29 @@ def test_subscribe_with_valid_message(self, mock_request_message, mock_queue_mes # Assert self.service.process_message.assert_called_once_with(mock_request_message) + @patch('src.services.inclination_service.QueueMessage') + @patch('src.services.inclination_service.RequestMessage') + @patch('src.services.inclination_service.Logger') + def test_subscribe_with_none_message(self, mock_logger, mock_request_message, mock_queue_message): + self.service.process_message = MagicMock() + self.service.subscribe() + callback = self.service.request_topic.subscribe.call_args[1]['callback'] + callback(None) + self.service.process_message.assert_not_called() + mock_logger.info.assert_called_with(' No Message') + + def test_subscribe_triggers_shutdown_when_max_receivable_positive(self): + self.service._config.max_receivable_messages = 1 + self.service._stop_server_and_container = MagicMock() + + self.service.subscribe() + + last_call_kwargs = self.service.request_topic.subscribe.call_args_list[-1][1] + self.assertIn('subscription', last_call_kwargs) + self.assertIn('callback', last_call_kwargs) + self.assertEqual(last_call_kwargs['max_receivable_messages'], 1) + self.service._stop_server_and_container.assert_called_once_with(delay_seconds=2) + @patch('src.services.inclination_service.get_unique_id', return_value='unique_id') @patch('src.services.inclination_service.Inclination') def test_process_message_with_valid_file_path(self, mock_inclination, mock_get_unique_id): @@ -62,6 +87,28 @@ def test_process_message_with_valid_file_path(self, mock_inclination, mock_get_u self.service.send_status.assert_called_once_with(valid=True, request_message=mock_request_message, file_path='uploaded_file_path') + @patch('src.services.inclination_service.get_unique_id', return_value='unique_id') + @patch('src.services.inclination_service.Inclination') + def test_process_message_marks_invalid_if_upload_fails(self, mock_inclination, mock_get_unique_id): + # Arrange + mock_request_message = MagicMock() + mock_request_message.data.jobId = None + mock_request_message.data.dataset_url = 'test_dataset_url' + mock_inclination.return_value.calculate.return_value = 'calculated_file_path' + self.service.upload_to_azure = MagicMock(return_value=None) + self.service.send_status = MagicMock() + + # Act + self.service.process_message(mock_request_message) + + # Assert + self.service.send_status.assert_called_once_with( + valid=False, + request_message=mock_request_message, + file_path='calculated_file_path', + upload_message='Error: Failed to upload processed file to storage' + ) + @patch('src.services.inclination_service.Logger') @patch('src.services.inclination_service.get_unique_id', return_value='unique_id') @patch('src.services.inclination_service.Inclination') @@ -77,7 +124,7 @@ def test_process_message_with_invalid_file_path(self, mock_inclination, mock_get # Assert self.service.send_status.assert_called_once_with(valid=False, request_message=mock_request_message, - file_path=None) + file_path=None, upload_message='Error: Request does not have valid file path specified.') @patch('src.services.inclination_service.Logger') @patch('src.services.inclination_service.Inclination') @@ -94,7 +141,7 @@ def test_process_message_when_file_path_is_none_and_valid_is_false(self, mock_in # Assert self.service.send_status.assert_called_once_with(valid=False, request_message=mock_request_message, - file_path='dataset_url') + file_path=None, upload_message='Error: Failed to generate processed file') @patch('src.services.inclination_service.Logger') @patch('src.services.inclination_service.Inclination') @@ -113,7 +160,7 @@ def test_process_message_when_exception_is_raised(self, mock_inclination, mock_l # Assert self.service.send_status.assert_called_once_with(valid=False, request_message=mock_request_message, - file_path='dataset_url') + file_path='dataset_url', upload_message='Error: Some error occurred') @patch('src.services.inclination_service.QueueMessage') def test_send_status_success(self, mock_queue_message): @@ -130,8 +177,25 @@ def test_send_status_success(self, mock_queue_message): mock_queue_message.data_from.assert_called_once() mock_response_topic.publish.assert_called_once_with(data=mock_data) + @patch('src.services.inclination_service.QueueMessage') + def test_send_status_with_error_message(self, mock_queue_message): + mock_request_message = MagicMock() + mock_data = {'messageId': '1234', 'messageType': 'test', 'data': {'success': False}} + mock_queue_message.data_from.return_value = mock_data + + self.service.send_status( + valid=False, + request_message=mock_request_message, + file_path='file_path', + upload_message='Error: failed' + ) + + payload = mock_queue_message.data_from.call_args[0][0] + self.assertEqual(payload['data']['message'], 'Error: failed') + + @patch('src.services.inclination_service.os.path.getsize', return_value=10) @patch('builtins.open', new_callable=mock_open) # Mock open to simulate file handling - def test_upload_to_azure_exception(self, mock_open): + def test_upload_to_azure_exception(self, mock_open, mock_getsize): # Arrange mock_open.side_effect = Exception('File open error') @@ -141,8 +205,9 @@ def test_upload_to_azure_exception(self, mock_open): # Assert self.assertIsNone(result) + @patch('src.services.inclination_service.os.path.getsize', return_value=10) @patch('builtins.open', new_callable=mock_open) - def test_upload_to_azure_container_error(self, mock_open): + def test_upload_to_azure_container_error(self, mock_open, mock_getsize): # Arrange self.service.storage_client.get_container.side_effect = Exception('Container error') @@ -159,6 +224,8 @@ def test_stop_listening(self, mock_thread): mock_thread.return_value = mock_thread_instance self.service.listening_thread = mock_thread_instance + self.service._shutdown_triggered = MagicMock() + self.service._shutdown_triggered.is_set.return_value = True # Act result = self.service.stop_listening() @@ -167,8 +234,67 @@ def test_stop_listening(self, mock_thread): mock_thread_instance.join.assert_called_once_with(timeout=0) self.assertIsNone(result) + @patch('src.services.inclination_service.Logger') + @patch('src.services.inclination_service.threading.Thread') + def test_stop_server_and_container_skips_when_already_triggered(self, mock_thread, mock_logger): + self.service._shutdown_triggered = MagicMock() + self.service._shutdown_triggered.is_set.return_value = True + + self.service._stop_server_and_container() + + mock_logger.info.assert_any_call('Server stop already in progress; skipping duplicate trigger.') + mock_thread.assert_not_called() + + @patch('src.services.inclination_service.time.sleep') + @patch('src.services.inclination_service.os._exit') + @patch('src.services.inclination_service.os.kill') + @patch('src.services.inclination_service.os.getpid', return_value=999) + @patch('src.services.inclination_service.threading.Thread') + def test_stop_server_and_container_terminates_process( + self, mock_thread, mock_getpid, mock_kill, mock_exit, mock_sleep + ): + self.service._shutdown_triggered = MagicMock() + self.service._shutdown_triggered.is_set.return_value = False + + def make_thread(target=None, daemon=None): + t = MagicMock() + t.start.side_effect = lambda: target() + return t + + mock_thread.side_effect = make_thread + + self.service._stop_server_and_container(delay_seconds=1) + + mock_sleep.assert_called_once_with(1) + mock_kill.assert_called_once_with(999, signal.SIGTERM) + mock_exit.assert_called_once_with(0) + + @patch('src.services.inclination_service.Logger') + @patch('src.services.inclination_service.os._exit') + @patch('src.services.inclination_service.os.kill', side_effect=Exception('kill failed')) + @patch('src.services.inclination_service.os.getpid', return_value=999) + @patch('src.services.inclination_service.threading.Thread') + def test_stop_server_and_container_handles_kill_error( + self, mock_thread, mock_getpid, mock_kill, mock_exit, mock_logger + ): + self.service._shutdown_triggered = MagicMock() + self.service._shutdown_triggered.is_set.return_value = False + + def make_thread(target=None, daemon=None): + t = MagicMock() + t.start.side_effect = lambda: target() + return t + + mock_thread.side_effect = make_thread + + self.service._stop_server_and_container(delay_seconds=0) + + mock_logger.warning.assert_called() + mock_exit.assert_called_once_with(0) + + @patch('src.services.inclination_service.os.path.getsize', return_value=10) @patch('builtins.open', new_callable=mock_open, read_data='file data') - def test_upload_to_azure_success(self, mock_file): + def test_upload_to_azure_success(self, mock_file, mock_getsize): job_id = 'test_job_id' file_path = '/path/to/test_file.geojson' @@ -187,6 +313,11 @@ def test_upload_to_azure_success(self, mock_file): mock_file_obj.upload.assert_called_once_with(mock_file()) self.assertEqual(result, 'https://azure.example.com/test-container/jobs/test_job_id/test_file.geojson') + @patch('src.services.inclination_service.os.path.getsize', return_value=0) + def test_upload_to_azure_empty_file(self, mock_getsize): + result = self.service.upload_to_azure(job_id='1234', file_path='/tmp/file_path') + self.assertIsNone(result) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_main.py b/tests/test_main.py index f6b388f..fac721b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,13 +1,19 @@ import unittest +import asyncio from fastapi import status from src.main import app, get_settings +from unittest.mock import patch, MagicMock from fastapi.testclient import TestClient +from src.main import startup_event, shutdown_event class TestApp(unittest.TestCase): def setUp(self): self.client = TestClient(app) + def tearDown(self): + app.incline_service = None + def test_root(self): response = self.client.get("/") self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -22,6 +28,65 @@ def test_get_settings(self): settings = get_settings() self.assertIsNotNone(settings) + @patch('src.main.InclinationService') + @patch('src.main.os.makedirs') + @patch('src.main.os.path.exists', return_value=False) + @patch('src.main.Settings') + def test_startup_event_success(self, mock_settings, mock_exists, mock_makedirs, mock_inclination_service): + mock_settings_obj = MagicMock() + mock_settings_obj.get_download_directory.return_value = '/tmp/downloads' + mock_settings.return_value = mock_settings_obj + mock_service = MagicMock() + mock_inclination_service.return_value = mock_service + + asyncio.run(startup_event()) + + mock_makedirs.assert_called_once_with('/tmp/downloads') + self.assertEqual(app.incline_service, mock_service) + + @patch('src.main.os._exit') + @patch('src.main.Settings', side_effect=Exception('invalid settings')) + def test_startup_event_failure_without_psutil(self, mock_settings, mock_exit): + import builtins + original_import = builtins.__import__ + + def import_side_effect(name, *args, **kwargs): + if name == 'psutil': + raise ModuleNotFoundError('No module named psutil') + return original_import(name, *args, **kwargs) + + with patch('builtins.__import__', side_effect=import_side_effect): + asyncio.run(startup_event()) + + mock_exit.assert_called_once_with(1) + + @patch('src.main.Settings', side_effect=Exception('invalid settings')) + @patch('src.main.os.getpid', return_value=123) + def test_startup_event_failure_with_psutil(self, mock_getpid, mock_settings): + fake_child = MagicMock() + fake_parent = MagicMock() + fake_parent.children.return_value = [fake_child] + + fake_psutil_module = MagicMock() + fake_psutil_module.Process.return_value = fake_parent + + with patch.dict('sys.modules', {'psutil': fake_psutil_module}): + asyncio.run(startup_event()) + + fake_parent.children.assert_called_once_with(recursive=True) + fake_child.kill.assert_called_once() + fake_parent.kill.assert_called_once() + + def test_shutdown_event_with_service(self): + mock_service = MagicMock() + app.incline_service = mock_service + asyncio.run(shutdown_event()) + mock_service.stop_listening.assert_called_once() + + def test_shutdown_event_without_service(self): + app.incline_service = None + asyncio.run(shutdown_event()) + if __name__ == '__main__': unittest.main()