Merge pull request #72 from openplans/test-modernization

Modernize Test Suite and Improve Tests

Author: mjumbewu

.gitignore

@@ -14,6 +14,7 @@ build/
 coverage
 coverage.data
 .coverage
+htmlcov/
 
 # junk
 .DS_Store

Containerfile

@@ -23,6 +23,7 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
 # Copy the application code to the container
 COPY src /app
+COPY pytest.ini /app/pytest.ini
 WORKDIR /app
 
 # Run collectstatic to gather static files

Makefile

@@ -1,4 +1,4 @@
-.PHONY: test-env test test-clean build gcp-push gcp-restart gcp-deploy
+.PHONY: test-env test test-coverage test-coverage-html test-clean build gcp-push gcp-restart gcp-deploy
 
 # Build the container image
 build:
@@ -32,7 +32,15 @@ test-env:
 
 # Run tests in a clean container environment
 test: test-env test-clean
-	podman-compose run --rm test
+	podman-compose run --rm test pytest
+
+# Run tests with coverage report
+test-coverage: test-env test-clean
+	podman-compose run --rm test pytest --cov=sa_api_v2 --cov-report=term-missing
+
+# Run tests with HTML coverage report (outputs to htmlcov/)
+test-coverage-html: test-env test-clean
+	podman-compose run --rm test pytest --cov=sa_api_v2 --cov-report=html
 
 # Just clean up containers
 test-clean:

compose.yml

@@ -13,8 +13,8 @@ services:
     environment:
       - REDIS_URL=redis://redis:6379/0
     depends_on:
-      db: {"condition": "service_healthy"}
-      redis: {"condition": "service_healthy"}
+      db: { "condition": "service_healthy" }
+      redis: { "condition": "service_healthy" }
     restart: "no"
 
   web:
@@ -27,9 +27,9 @@ services:
     ports:
       - "8000:8000"
     depends_on:
-      db: {"condition": "service_healthy"}
-      redis: {"condition": "service_healthy"}
-      init: {"condition": "service_completed_successfully"}
+      db: { "condition": "service_healthy" }
+      redis: { "condition": "service_healthy" }
+      init: { "condition": "service_completed_successfully" }
 
   worker:
     build:
@@ -41,22 +41,24 @@ services:
       C_FORCE_ROOT: "true"
       REDIS_URL: "redis://redis:6379/0"
     depends_on:
-      db: {"condition": "service_healthy"}
-      redis: {"condition": "service_healthy"}
-      init: {"condition": "service_completed_successfully"}
+      db: { "condition": "service_healthy" }
+      redis: { "condition": "service_healthy" }
+      init: { "condition": "service_completed_successfully" }
 
   test:
-    build:
-      context: .
-      dockerfile: Containerfile
-    command: python3 manage.py test .
+    image: shareabouts-api
+    command: pytest
     env_file: .env
     environment:
       - REDIS_URL=redis://redis:6379/0
+    volumes:
+      - ./htmlcov:/app/htmlcov
+      - ./src:/app
+      - ./pytest.ini:/app/pytest.ini
     depends_on:
-      db: {"condition": "service_healthy"}
-      redis: {"condition": "service_healthy"}
-      init: {"condition": "service_completed_successfully"}
+      db: { "condition": "service_healthy" }
+      redis: { "condition": "service_healthy" }
+      init: { "condition": "service_completed_successfully" }
 
   db:
     image: postgis/postgis:15-3.3
@@ -67,7 +69,7 @@ services:
     ports:
       - "15432:5432"
     healthcheck:
-      test: [ "CMD-SHELL","psql -U postgres -d shareabouts -c \"SELECT ST_Buffer( ST_SetSRID('POINT(0 0)'::GEOMETRY, 4326), 1) AS geom ;\""]
+      test: [ "CMD-SHELL", "psql -U postgres -d shareabouts -c \"SELECT ST_Buffer( ST_SetSRID('POINT(0 0)'::GEOMETRY, 4326), 1) AS geom ;\"" ]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -80,4 +82,4 @@ services:
       test: [ "CMD", "redis-cli", "ping" ]
       interval: 10s
       timeout: 5s
-      retries: 5
+      retries: 5

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+DJANGO_SETTINGS_MODULE = project.settings
+python_files = tests.py test_*.py *_tests.py
+addopts = --reuse-db --nomigrations

requirements.txt

@@ -45,6 +45,10 @@ prompt-toolkit==3.0.43
 psycopg==3.1.17
 psycopg-binary==3.1.17
 pycparser==2.21
+pytest==7.4.4
+pytest-cov==4.1.0
+pytest-django==4.7.0
+pytest-xdist==3.5.0
 PyJWT==2.8.0
 python-dateutil==2.8.2
 python-social-auth==0.2.21

src/project/settings.py

@@ -66,7 +66,6 @@
 
 LANGUAGE_CODE = 'en-us'
 USE_I18N = True
-USE_L10N = True
 
 ###############################################################################
 #
@@ -106,7 +105,8 @@
     },
 ]
 
-ATTACHMENT_STORAGE = 'django.core.files.storage.FileSystemStorage'
+# Use the new div-based form rendering to avoid RemovedInDjango50Warning
+FORM_RENDERER = 'django.forms.renderers.DjangoDivFormRenderer'
 
 ###############################################################################
 #
@@ -433,16 +433,27 @@ def custom_show_toolbar(request):
 # We support S3, GCS, and local filesystem.
 # Precedence: GCS > S3 > Local
 
-DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
-STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'
-ATTACHMENT_STORAGE = DEFAULT_FILE_STORAGE
+STORAGES = {
+    'default': {
+        'BACKEND': 'django.core.files.storage.FileSystemStorage',
+    },
+    # Static files (CSS, JavaScript, Images)
+    # https://docs.djangoproject.com/en/1.6/howto/static-files/
+    'staticfiles': {
+        'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage',
+    },
+    'attachments': {
+        'BACKEND': 'django.core.files.storage.FileSystemStorage',
+    },
+}
+ATTACHMENT_STORAGE = STORAGES['attachments']['BACKEND']
 
 if 'GS_BUCKET_NAME' in environ:
     # Google Cloud Storage
     GS_BUCKET_NAME = environ['GS_BUCKET_NAME']
     GS_PROJECT_ID = environ.get('GS_PROJECT_ID')
 
-    DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
+    STORAGES['default']['BACKEND'] = "storages.backends.gcloud.GoogleCloudStorage"
 
     GS_DEFAULT_ACL = "publicRead"
 
@@ -456,7 +467,7 @@ def custom_show_toolbar(request):
     MEDIA_URL = f"https://storage.googleapis.com/{GS_BUCKET_NAME}/media/"
 
     # Attachments
-    ATTACHMENT_STORAGE = DEFAULT_FILE_STORAGE
+    ATTACHMENT_STORAGE = STORAGES['attachments']['BACKEND'] = STORAGES['default']['BACKEND']
 
 elif all([key in environ for key in ('SHAREABOUTS_AWS_KEY',
                                      'SHAREABOUTS_AWS_SECRET',
@@ -468,8 +479,8 @@ def custom_show_toolbar(request):
     AWS_QUERYSTRING_AUTH = False
     AWS_PRELOAD_METADATA = True
 
-    DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
-    ATTACHMENT_STORAGE = DEFAULT_FILE_STORAGE
+    STORAGES['default']['BACKEND'] = 'storages.backends.s3boto3.S3Boto3Storage'
+    ATTACHMENT_STORAGE = STORAGES['attachments']['BACKEND'] = STORAGES['default']['BACKEND']
 
 
 if 'SHAREABOUTS_TWITTER_KEY' in environ and 'SHAREABOUTS_TWITTER_SECRET' in environ:

src/pytest.ini

@@ -82,7 +82,7 @@ def render(self, name, value, attrs=None, renderer=None):
         return super(PrettyAceWidget, self).render(name, value, attrs=attrs, renderer=renderer)
 
 
-BaseGeoAdmin = admin.OSMGeoAdmin if settings.USE_GEODB else admin.ModelAdmin
+BaseGeoAdmin = admin.GISModelAdmin if settings.USE_GEODB else admin.ModelAdmin
 
 
 class SubmittedThingAdmin(BaseGeoAdmin):

src/sa_api_v2/admin.py

@@ -7,7 +7,7 @@
 else:
     from django.db import models
 
-from django.core.files.storage import get_storage_class
+from django.core.files.storage import storages
 from django.utils.timezone import now
 from .. import cache
 from .. import utils
@@ -18,6 +18,10 @@
 from PIL import Image, UnidentifiedImageError
 
 
+# Create a simple AttachmentStorage factory
+AttachmentStorage = lambda *a, **kw : storages.create_storage(storages.backends['attachments'], *a, **kw)
+
+
 class TimeStampedModel (models.Model):
     created_datetime = models.DateTimeField(default=now, blank=True, db_index=True)
     updated_datetime = models.DateTimeField(auto_now=True, db_index=True)
@@ -294,9 +298,6 @@ def timestamp_filename(attachment, filename):
     return ''.join(['attachments/', utils.base62_time(), '-', filename])
 
 
-AttachmentStorage = get_storage_class(settings.ATTACHMENT_STORAGE)
-
-
 class Attachment (CacheClearingModel, TimeStampedModel):
     """
     A file attached to a submitted thing.