Django to 4.2.0 (#1959)

* django_to_3.2.25

* django_to_4.0.0

* django to 4.20 - close but still failing tests

* pdb in test_competitions

* STATICFILES_STORAGE is old pattern

* some migrations for changes to default field behaviors

* flake adjustments

* remove a pdb for test

* channels to 4.2.0

* merged develop into django4 branch

* Added Daphne middleware + whitenoise for static files; fixed csrf errors

* CSRF fixed

* CSRF fixes

* Fixed potential botocore problem with use_ssl state

* rebase and modifed migration

* remove daphne and selenium

* flake8 fixes

* makemigrations results after rebase

* Cleanup

* Update packages

---------

Co-authored-by: Obada Haddad <obada.haddad@lisn.fr>
Co-authored-by: didayolo <adrien.pavao@gmail.com>

Author: 3 people

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.20
+FROM python:3.10.0
 
 RUN apt-get update && apt-get install -y gcc build-essential && rm -rf /var/lib/apt/lists/*
 

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   #-----------------------------------------------
   #   Web Services

poetry.lock

@@ -7,47 +7,45 @@ readme = "README.md"
 package-mode = false
 
 [tool.poetry.dependencies]
-python = "3.9.20"
-django = "3.2.0"
-django-oauth-toolkit = "1.2.0"
+python = ">=3.10,<3.11"
+django = "^4.2.0"
+django-oauth-toolkit = "1.6.3"
 django-cors-middleware = "1.5.0"
-social-auth-core = "2.0.0"
-social-auth-app-django = "3.1.0"
+social-auth-core = "^4.1.0"
+social-auth-app-django = "^5.0.0"
 six = "1.16.0"
-django-extensions = "2.2.6"
-channels = "3.0.0"
+django-extensions = "^3.2"
+channels = "4.2.0"
 channels-redis = "4.0.0"
-django-extra-fields = "0.9"
 pillow = "10.3.0"
 celery = "4.4.7"
 gunicorn = "22.0.0"
-urllib3 = ">=1.21.1,<1.25"
-uvicorn = {version = "0.13.3", extras = ["standard"]}
+urllib3 = ">=1.25.4,<1.27"
+uvicorn = "^0.22.0"
 pyyaml = "5.3.1"
 watchdog = "2.1.1"
 argh = "0.26.2"
 python-dateutil = "2.7.3"
 bpython = "^0.21.0"
-websockets = "8.1"
+websockets = "^10.4.0"
 aiofiles = "0.4.0"
 oyaml = "0.7"
 factory-boy = "2.11.1"
-bleach = "3.1.4"
+bleach = ">=5.0.0"
 django-debug-toolbar = "3.2"
 django-querycount = "0.7.0"
 blessings = "1.7"
-django-su = "0.9.0"
+django-su = "^1.0.0"
 django-ajax-selects = "2.0.0"
 dj-database-url = "0.4.2"
-psycopg2-binary = "2.8.6"
+psycopg2-binary = "^2.9.9"
 django-redis = "4.12.1"
-django-storages = "1.9.1"
-azure-storage-blob = "2.1.0"
+django-storages = {extras = ["azure"], version = "^1.14.6"}
+azure-storage-blob = "^12"
 azure-storage-common = "2.1.0"
-boto3 = "1.9.68"
+boto3 = "1.26.76"
 whitenoise = "5.2.0"
-coreapi = "2.3.3"
-djangorestframework = "3.12.0"
+djangorestframework = ">=3.13.0"
 djangorestframework-csv = "3.0.1"
 drf-extensions = "0.4.0"
 markdown = "2.6.11"
@@ -60,16 +58,21 @@ django-enforce-host = "1.0.1"
 twisted = "24.7.0"
 ipdb = "0.13"
 flake8 = "3.8.4"
-pytest = "6.2.1"
-pytest-django = "4.1.0"
-pytest-pythonpath = "0.7.3"
+pytest = "^7.0.0"
+pytest-django = "^4.5.0"
 jinja2 = "3.1.4"
 requests = "2.32.2"
-drf-extra-fields = "3.0.2"
-drf-yasg2 = "^1.19.4"
-swagger-spec-validator = "^3.0.4"
+drf-extra-fields = ">=3.5.0"
+botocore = "1.29.76"
+s3transfer = "0.6.0"
+drf-spectacular = "^0.28.0"
+coreapi = "^2.3.3"
 loguru = "^0.7.3"
-
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = "settings.develop"  # Just "settings" since pytest will be running from src/
+pythonpath = [".", "src", "src/apps"]
+testpaths = ["src"]  # Tell pytest to look for tests in src/
+addopts = "-v --tb=short --reuse-db"
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

pyproject.toml

@@ -0,0 +1,41 @@
+# Generated by Django 4.2.23 on 2025-09-08 12:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analytics", "0002_auto_20250218_1143"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="adminstoragedatapoint",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="competitionstoragedatapoint",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="storageusagehistory",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userstoragedatapoint",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+    ]

src/apps/analytics/migrations/0003_alter_adminstoragedatapoint_id_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.23 on 2025-09-08 12:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("announcements", "0003_auto_20230616_1326"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="announcement",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="newspost",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+    ]

src/apps/announcements/migrations/0004_alter_announcement_id_alter_newspost_id.py

@@ -109,7 +109,6 @@ def test_manual_migration_checks_permissions_must_be_collaborator_to_migrate(sel
 
     def test_manual_migration_makes_submissions_from_one_phase_in_another(self):
         self.client.login(username='creator', password='creator')
-
         # make 5 submissions in phase 1
         for _ in range(5):
             SubmissionFactory(owner=self.creator, phase=self.phase_1, status=Submission.FINISHED, leaderboard=self.leaderboard)
@@ -123,6 +122,7 @@ def test_manual_migration_makes_submissions_from_one_phase_in_another(self):
             assert resp.status_code == 200
             assert run_submission_mock.call_count == 5
 
+        self.phase_2.refresh_from_db()
         # check phase 2 has the 5 submissions
         assert self.phase_1.submissions.count() == 5
         assert self.phase_2.submissions.count() == 5

src/apps/api/tests/test_competitions.py

@@ -1,10 +1,12 @@
 from django.conf.urls import include
-from django.urls import path, re_path
-from drf_yasg2 import openapi
-from drf_yasg2.views import get_schema_view
+from django.urls import path
+
+
+from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView, SpectacularRedocView
+
+
 from rest_framework.authtoken.views import obtain_auth_token
 from rest_framework.routers import SimpleRouter
-from rest_framework.permissions import AllowAny
 from rest_framework.urlpatterns import format_suffix_patterns
 
 from .views import (
@@ -33,15 +35,6 @@
 router.register('users', profiles.UserViewSet, 'users')
 router.register('organizations', profiles.OrganizationViewSet, 'organizations')
 
-schema_view = get_schema_view(
-    openapi.Info(
-        title="Codabench API",
-        default_version='v1',
-    ),
-    validators=['flex', 'ssv'],
-    public=True,
-    permission_classes=(AllowAny,),
-)
 
 urlpatterns = [
     path('my_profile/', profiles.GetMyProfile.as_view()),
@@ -74,9 +67,9 @@
     path('analytics/check_orphans_deletion_status/', analytics.check_orphans_deletion_status, name="check_orphans_deletion_status"),
 
     # API Docs
-    re_path(r'docs(?P<format>\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0), name='schema-json'),
-    path('docs/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
-    path('redoc/', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'),
+    path('api/schema/', SpectacularAPIView.as_view(), name='schema'),
+    path('docs/', SpectacularSwaggerView.as_view(url_name='schema'), name='schema-swagger-ui'),
+    path('redoc/', SpectacularRedocView.as_view(url_name='schema'), name='schema-redoc'),
 
     # Include this at the end so our URLs above run first, like /datasets/completed/<pk>/ before /datasets/<pk>/
     path('', include(format_suffix_patterns(router.urls, allowed=['html', 'json', 'csv', 'zip']))),

src/apps/api/urls.py

@@ -8,7 +8,7 @@
 from django.db import IntegrityError
 from django.db.models import Subquery, OuterRef, Q
 from django_filters.rest_framework import DjangoFilterBackend
-from drf_yasg2.utils import swagger_auto_schema, no_body
+from drf_spectacular.utils import extend_schema
 from rest_framework import status
 from rest_framework.decorators import action
 from rest_framework.exceptions import PermissionDenied, ValidationError
@@ -505,7 +505,8 @@ def results(self, request, pk, format='json'):
                 dict_writer.writerow(row)
             return response
 
-    @swagger_auto_schema(responses={200: CompetitionCreationTaskStatusSerializer()})
+    # @swagger_auto_schema(responses={200: CompetitionCreationTaskStatusSerializer()})
+    @extend_schema(responses={200: CompetitionCreationTaskStatusSerializer})
     @action(detail=True, methods=('GET',))
     def creation_status(self, request, pk):
         """This endpoint gets the creation status for a competition during upload"""
@@ -518,7 +519,8 @@ def creation_status(self, request, pk):
 
         return Response(serializer.data)
 
-    @swagger_auto_schema(responses={200: FrontPageCompetitionsSerializer()})
+    # @swagger_auto_schema(responses={200: FrontPageCompetitionsSerializer()})
+    @extend_schema(responses={200: FrontPageCompetitionsSerializer})
     @action(detail=False, methods=('GET',), permission_classes=(AllowAny,))
     def front_page(self, request):
         popular_comps = get_popular_competitions()
@@ -530,7 +532,8 @@ def front_page(self, request):
             "recent_comps": recent_comps_serializer.data
         })
 
-    @swagger_auto_schema(request_body=no_body, responses={201: CompetitionCreationTaskStatusSerializer()})
+    # @swagger_auto_schema(request_body=no_body, responses={201: CompetitionCreationTaskStatusSerializer()})
+    @extend_schema(request=None, responses={201: CompetitionCreationTaskStatusSerializer})
     @action(detail=True, methods=('POST',), serializer_class=CompetitionCreationTaskStatusSerializer)
     def create_dump(self, request, pk=None):
         competition = self.get_object()
@@ -708,6 +711,7 @@ def manually_migrate(self, request, pk):
                 {"detail": "You do not have administrative permissions for this competition"},
                 status=status.HTTP_403_FORBIDDEN
             )
+        # import pdb; pdb.set_trace()
         manual_migration.apply_async((pk,))
         return Response({}, status=status.HTTP_200_OK)
 
@@ -764,7 +768,8 @@ def rerun_submissions(self, request, pk):
         else:
             raise PermissionDenied(error_message)
 
-    @swagger_auto_schema(responses={200: PhaseResultsSerializer})
+    # @swagger_auto_schema(responses={200: PhaseResultsSerializer})
+    @extend_schema(responses={200: PhaseResultsSerializer})
     @action(detail=True, methods=['GET'], permission_classes=[AllowAny])
     def get_leaderboard(self, request, pk):
         phase = self.get_object()

src/apps/api/views/competitions.py

@@ -3,10 +3,16 @@
 from django.db import migrations, models
 import django.db.models.deletion
 import django.utils.timezone
-import storages.backends.s3boto3
 import utils.data
 import uuid
 
+# New
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+class _MigrationPrivateStorage(S3Boto3Storage):
+    bucket_name = 'private'
+
 
 class Migration(migrations.Migration):
 
@@ -109,8 +115,8 @@ class Migration(migrations.Migration):
                 ('status', models.CharField(choices=[('None', 'None'), ('Submitting', 'Submitting'), ('Submitted', 'Submitted'), ('Preparing', 'Preparing'), ('Running', 'Running'), ('Scoring', 'Scoring'), ('Cancelled', 'Cancelled'), ('Finished', 'Finished'), ('Failed', 'Failed')], default='Submitting', max_length=128)),
                 ('status_details', models.TextField(blank=True, null=True)),
                 ('appear_on_leaderboards', models.BooleanField(default=False)),
-                ('prediction_result', models.FileField(blank=True, null=True, storage=storages.backends.s3boto3.S3Boto3Storage(bucket='private'), upload_to=utils.data.PathWrapper('prediction_result'))),
-                ('scoring_result', models.FileField(blank=True, null=True, storage=storages.backends.s3boto3.S3Boto3Storage(bucket='private'), upload_to=utils.data.PathWrapper('scoring_result'))),
+                ('prediction_result', models.FileField(blank=True, null=True, storage=_MigrationPrivateStorage(), upload_to=utils.data.PathWrapper('prediction_result'))),
+                ('scoring_result', models.FileField(blank=True, null=True, storage=_MigrationPrivateStorage(), upload_to=utils.data.PathWrapper('scoring_result'))),
                 ('secret', models.UUIDField(default=uuid.uuid4)),
                 ('task_id', models.UUIDField(blank=True, null=True)),
                 ('name', models.CharField(blank=True, default='', max_length=120, null=True)),
@@ -127,7 +133,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.CharField(max_length=50)),
-                ('data_file', models.FileField(storage=storages.backends.s3boto3.S3Boto3Storage(bucket='private'), upload_to=utils.data.PathWrapper('submission_details'))),
+                ('data_file', models.FileField(storage=_MigrationPrivateStorage(), upload_to=utils.data.PathWrapper('submission_details'))),
                 ('is_scoring', models.BooleanField(default=False)),
                 ('submission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='details', to='competitions.Submission')),
             ],