SRE-371 Refactor application and add tests

* Refactors the application to enable better testing
* Implements tests

Author: justinfranco

.github/workflows/publish.yaml .github/workflows/ci-cd.yaml.github/workflows/publish.yaml renamed to .github/workflows/ci-cd.yaml

@@ -1,24 +1,28 @@
-name: publish
+name: CI/CD
 on:
   push:
-    tags:
-       - 'v*'
+  pull_request:
+    branches:
+      - 'main'
+
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  release:
+  test:
     runs-on: ubuntu-latest
+
     steps:
-      - name: Create GitHub release
-        uses: Roang-zero1/github-create-release-action@master
-        with:
-          version_regex: ^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Run unit tests and linter
+        run: docker build --target test .
 
   build-and-push-image:
+    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'pull_request'
+    needs: test
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -48,3 +52,14 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+  release:
+    runs-on: ubuntu-latest
+    needs: build-and-push-image
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - name: Create GitHub release
+        uses: Roang-zero1/github-create-release-action@master
+        with:
+          version_regex: ^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -1,3 +1,5 @@
+default_language_version:
+    python: python3.11
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0

Dockerfile

@@ -1,16 +1,27 @@
-FROM python:3.9.10 AS build
+FROM python:3.11-slim AS base
+
+WORKDIR /opt/brpc
 COPY requirements.txt .
-RUN pip install -r ./requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+
+FROM base AS test
+
+COPY requirements-dev.txt .
+RUN pip install --no-cache-dir -r requirements-dev.txt
+
+COPY src/*.py ./
+COPY src/tests tests
+
+RUN coverage run --branch -m pytest
+RUN coverage report --fail-under 90
+RUN pylint *.py
 
-FROM gcr.io/distroless/python3:nonroot
+FROM base AS prod
 
-COPY --from=build /usr/local/lib/python3.9/site-packages/ \
- /usr/lib/python3.9/.
+COPY src/*.py ./
 
-ENV LC_ALL C.UTF-8
-WORKDIR /usr/src/app
-COPY src/*.py .
-COPY src/collectors/* collectors/
-# https://github.com/GoogleContainerTools/distroless/blob/main/experimental/python3/BUILD#L77
+RUN useradd -r -s /sbin/nologin nonroot
 USER nonroot
-CMD ["exporter.py"]
+EXPOSE 8080
+CMD [ "python", "exporter.py" ]

README.md

@@ -17,21 +17,22 @@ Please note that this tool is in the early development stage and should not be u
 The project in its current form suits our short-term needs and will receive limited support. We encourage you to fork the project and extend it with additional functionality you might need.
 
 ## Development
-You should install [pre-commit](https://pre-commit.com/) so that automated linting and formatting checks are performed before each commit. 
+You should install [pre-commit](https://pre-commit.com/) so that automated linting and formatting checks are performed before each commit.
 
 Run:
 ```bash
 pip install pre-commit
 pre-commit install
 ```
 ### Running locally
-1. Make sure you have python3 installed (>3.9.10)
+1. Make sure you have python3 installed (>3.11)
 2. Set up your python environment
 ```bash
 pip3 install virtualenv
 virtualenv venv
 source venv/bin/activate
 pip install -r requirements.txt
+pip install -r requirements-dev.txt
 ```
 1. Generate valid exporter config and validation file. For example see [config example](config/exporter_example/config.yml) and [validation example](config/exporter_example/validation.yml).
 2. Export paths of generated configuration files relative to `src/exporter.py`:
@@ -53,8 +54,35 @@ export CONFIG_FILE_PATH="src/config.yml"  # For example if we saved config file
 3. Execute
 ```bash
 docker-compose build
-docker-compose-up
+docker-compose up
 curl localhost:8000/metrics # Exporter
 curl localhost:3000         # Grafana
 curl localhost:9090         # Prometheus
 ```
+
+### Testing
+Testing is performed using [pytest](https://docs.pytest.org/) run by [coverage.py](https://coverage.readthedocs.io/) to generate test coverage reporting.
+[pylint](https://pylint.readthedocs.io/) is used to lint the pyhton code.
+These dependencies can be found in the [requirements-dev.txt](requirements-dev.txt) file. Unit testing and linting is performed on every commit push to the repository. 90% test coverage and no linter errors/warnings are a requirement for the tests to pass.
+
+#### Testing Locally (venv)
+Tests can be run locally in the virtual environment.
+1. Run the unit tests with coverage.py from within the `src` directory.
+```bash
+coverage run --branch -m pytest
+```
+2. Generate the coverage report. To view the report open the generated `index.html` file in a browser.
+```bash
+coverage html
+```
+3. Run the linter to find any errors/warnings.
+```bash
+pylint src/*py
+```
+
+#### Testing Locally (docker)
+The tests and linter can be run using docker by building the `test` docker stage.
+1. Build the `test` stage in the `Dockerfile`.
+```bash
+docker build --target test .
+```

docker-compose.yml

@@ -18,7 +18,7 @@ services:
       - monitoring
     volumes:
       - type: bind
-        source: ${PWD}/${CONFIGURATION_FILE_PATH}
+        source: ${PWD}/${CONFIG_FILE_PATH}
         target: /config/config.yml
       - type: bind
         source: ${PWD}/${VALIDATION_FILE_PATH}

requirements-dev.txt

@@ -0,0 +1,5 @@
+pytest==7.2.1
+pylint==2.16.2
+coverage==7.1.0
+requests==2.28.2
+requests_mock==1.10.0

requirements.txt

@@ -2,6 +2,7 @@
 prometheus-client==0.13.1
 pyyaml==6.0
 schema==0.7.5
-websockets==10.3
+websockets==10.4
 structlog==22.1.0
-requests==2.28.1
+requests==2.28.1
+jsonrpcclient==4.0.2

src/cache.py

@@ -0,0 +1,33 @@
+"""Module for providing in-memory cache"""
+
+
+class Cache():
+    """A rudimentary in-memory cache implementation."""
+
+    def __init__(self):
+        self._cache = {}
+
+    def is_cached(self, key: str):
+        """Check if key is cached."""
+        if key in self._cache:
+            return True
+        return False
+
+    def store_key_value(self, key: str, value):
+        """Stores key-value in cache dict."""
+        self._cache[key] = value
+
+    def retrieve_key_value(self, key: str):
+        """Retrieves key value from cache."""
+        if self.is_cached(key):
+            return self._cache.get(key)
+        return None
+
+    def remove_key_from_cache(self, key: str):
+        """Deletes a key from cache."""
+        if self.is_cached(key):
+            del self._cache[key]
+
+    def clear_cache(self):
+        """Clears the entire cache"""
+        self._cache.clear()