deprecate parliament corpus settings

Author: lukavdplas

backend/corpora/parliament/canada.py

@@ -3,22 +3,30 @@
 import logging
 import re
 
-from django.conf import settings
-
 from corpora.parliament.parliament import Parliament
 from corpora.utils.constants import document_context
 from ianalyzer_readers.extract import Constant, CSV
-from addcorpus.python_corpora.corpus import CSVCorpusDefinition
+from addcorpus.python_corpora.corpus import CSVCorpusDefinition, get_deprecated_setting
 import corpora.parliament.utils.field_defaults as field_defaults
 from corpora.parliament.uk import format_house
 
 class ParliamentCanada(Parliament, CSVCorpusDefinition):
     title = 'People & Parliament (Canada)'
     description = "Speeches from House of Commons"
     min_date = datetime(year=1901, month=1, day=1)
-    data_directory = settings.PP_CANADA_DATA
-    es_index = getattr(settings, 'PP_CANADA_INDEX', 'parliament-canada')
-    word_model_path = getattr(settings, 'PP_CANADA_WM', None)
+
+    @property
+    def data_directory(self):
+        return get_deprecated_setting('PP_CANADA_DATA') or super().data_directory
+
+    @property
+    def es_index(self):
+        return get_deprecated_setting('PP_CANADA_INDEX') or 'parliament-canada'
+
+    @property
+    def word_model_path(self):
+        return get_deprecated_setting('PP_CANADA_WM') or super().word_model_path
+
     image = 'canada.jpeg'
     languages = ['en']
     description_page = 'canada.md'

backend/corpora/parliament/conftest.py

@@ -31,24 +31,60 @@ def parliament_corpora_settings(settings):
         'parliament-europe': 'corpora.parliament.euparl.ParliamentEurope',
     }
 
-    settings.PP_CANADA_DATA = os.path.join(here, 'tests', 'data', 'canada')
-    settings.PP_DENMARK_DATA = os.path.join(here, 'tests', 'data', 'denmark')
-    settings.PP_DENMARK_NEW_DATA = os.path.join(here, 'tests', 'data', 'denmark-new')
-    settings.PP_EUPARL_DATA = os.path.join(here, 'tests', 'data', 'euparl', 'rds')
-    settings.PP_FINLAND_DATA = os.path.join(here, 'tests', 'data', 'finland')
-    settings.PP_FINLAND_OLD_DATA = os.path.join(here, 'tests', 'data', 'finland-old')
-    settings.PP_FR_DATA = os.path.join(here, 'tests', 'data', 'france')
-    settings.PP_GERMANY_NEW_DATA = os.path.join(here, 'tests', 'data', 'germany-new')
-    settings.PP_GERMANY_OLD_DATA = os.path.join(here, 'tests', 'data', 'germany-old')
-    settings.PP_IRELAND_DATA = os.path.join(here, 'tests', 'data', 'ireland')
-    settings.PP_NL_DATA = os.path.join(here, 'tests', 'data', 'netherlands')
-    settings.PP_NL_RECENT_DATA = os.path.join(here, 'tests', 'data', 'netherlands-recent')
-    settings.PP_NORWAY_DATA = os.path.join(here, 'tests', 'data', 'norway')
-    settings.PP_NORWAY_NEW_DATA = os.path.join(here, 'tests', 'data', 'norway-new')
-    settings.PP_SWEDEN_DATA = os.path.join(here, 'tests', 'data', 'sweden')
-    settings.PP_SWEDEN_OLD_DATA = os.path.join(here, 'tests', 'data', 'sweden-old')
-    settings.PP_SWEDEN_SWERIK_DATA = os.path.join(here, 'tests', 'data', 'sweden-swerik')
-    settings.PP_UK_DATA = os.path.join(here, 'tests', 'data', 'uk')
+    settings.CORPUS_SETTINGS = {
+        'parliament-canada': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'canada'),
+        },
+        'parliament-denmark': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'denmark'),
+        },
+        'parliament-denmark-new': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'denmark-new'),
+        },
+        'parliament-europe': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'euparl', 'rds'),
+        },
+        'parliament-finland': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'finland'),
+        },
+        'parliament-finland-old': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'finland-old'),
+        },
+        'parliament-france': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'france'),
+        },
+        'parliament-germany-new': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'germany-new'),
+        },
+        'parliament-germany-old': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'germany-old'),
+        },
+        'parliament-ireland': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'ireland'),
+        },
+        'parliament-netherlands': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'netherlands-recent'),
+            'data_directory_old': os.path.join(here, 'tests', 'data', 'netherlands'),
+        },
+        'parliament-norway': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'norway'),
+        },
+        'parliament-norway-new': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'norway-new'),
+        },
+        'parliament-sweden': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'sweden'),
+        },
+        'parliament-sweden-old': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'sweden-old'),
+        },
+        'parliament-sweden-swerik': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'sweden-swerik'),
+        },
+        'parliament-uk': {
+            'data_directory': os.path.join(here, 'tests', 'data', 'uk')
+        }
+    }
 
 
 CORPUS_TEST_DATA = [

backend/corpora/parliament/denmark-new.py

@@ -1,12 +1,12 @@
 from datetime import datetime
 from glob import glob
-from django.conf import settings
+
 import os
 import re
 
 from corpora.parliament.parliament import Parliament
 from ianalyzer_readers.extract import Constant, CSV, Metadata, Combined
-from addcorpus.python_corpora.corpus import CSVCorpusDefinition
+from addcorpus.python_corpora.corpus import CSVCorpusDefinition, get_deprecated_setting
 import corpora.parliament.utils.field_defaults as field_defaults
 import corpora.utils.formatting as formatting
 import corpora.utils.constants as constants
@@ -37,9 +37,20 @@ class ParliamentDenmarkNew(Parliament, CSVCorpusDefinition):
     description = "Speeches from the Folketing"
     min_date = datetime(year=2009, month=1, day=1)
     max_date = datetime(year=2016, month=12, day=31)
-    data_directory = settings.PP_DENMARK_NEW_DATA
-    es_index = getattr(settings, 'PP_DENMARK_NEW_INDEX', 'parliament-denmark-new')
-    word_model_path = getattr(settings, 'PP_DENMARK_WM', None)
+
+    @property
+    def data_directory(self):
+        return get_deprecated_setting('PP_DENMARK_NEW_DATA') or super().data_directory
+
+    @property
+    def es_index(self):
+        return get_deprecated_setting('PP_DENMARK_NEW_INDEX') or 'parliament-denmark-new'
+
+    @property
+    def word_model_path(self):
+        return get_deprecated_setting('PP_DENMARK_WM') or super().word_model_path
+
+
     image = 'denmark.jpg'
     description_page = 'denmark-new.md'
     languages = ['da']

backend/corpora/parliament/denmark.py

@@ -1,11 +1,10 @@
 from datetime import datetime
 from glob import glob
 import logging
-from django.conf import settings
 
 from corpora.parliament.parliament import Parliament
 from ianalyzer_readers.extract import Constant, CSV
-from addcorpus.python_corpora.corpus import CSVCorpusDefinition
+from addcorpus.python_corpora.corpus import CSVCorpusDefinition, get_deprecated_setting
 import corpora.parliament.utils.field_defaults as field_defaults
 import corpora.utils.formatting as formatting
 
@@ -28,8 +27,15 @@ class ParliamentDenmark(Parliament, CSVCorpusDefinition):
     description = "Speeches from the Folketing and Landsting"
     min_date = datetime(year=1848, month=1, day=1)
     max_date = datetime(year=2008, month=12, day=31)
-    data_directory = settings.PP_DENMARK_DATA
-    es_index = getattr(settings, 'PP_DENMARK_INDEX', 'parliament-denmark')
+
+    @property
+    def data_directory(self):
+        return get_deprecated_setting('PP_DENMARK_DATA') or super().data_directory
+
+    @property
+    def es_index(self):
+        return get_deprecated_setting('PP_DENMARK_INDEX') or 'parliament-denmark'
+
     image = 'denmark.jpg'
     description_page = 'denmark.md'
 

backend/corpora/parliament/euparl.py

@@ -8,15 +8,14 @@
 import numbers
 
 from bs4 import BeautifulSoup
-from django.conf import settings
 from langcodes import standardize_tag, Language
 import requests
-from ianalyzer_readers.extract import Combined, JSON, Metadata, Pass, CSV, Constant
+from ianalyzer_readers.extract import Combined, JSON, Pass, CSV, Constant
 from ianalyzer_readers.readers.core import Field
 from ianalyzer_readers.readers.json import JSONReader
 
 from addcorpus.es_mappings import keyword_mapping, main_content_mapping
-from addcorpus.python_corpora.corpus import FieldDefinition
+from addcorpus.python_corpora.corpus import FieldDefinition, get_deprecated_setting
 from addcorpus.python_corpora.filters import MultipleChoiceFilter
 from corpora.parliament.parliament import Parliament
 import corpora.parliament.utils.field_defaults as field_defaults
@@ -31,26 +30,48 @@ def language_name(lang_code: str) -> str:
 
 
 class ParliamentEurope(Parliament):
+    '''
+    Corpus definition for EU Parliament.
+
+    During data extraction, the corpus will harvest data from an API up until the present,
+    so `max_date` is `datetime.now()` by default.
+    If you are not doing periodic updates, configure `max_date` to the date of the harvest.
+    '''
     title = 'People & Parliament (European Parliament)'
     description = "Speeches from the European Parliament (EP)"
-    es_index = getattr(settings, 'PP_EUPARL_INDEX', 'parliament-euparl')
-    data_directory = settings.PP_EUPARL_DATA
+
+    @property
+    def es_index(self):
+        return get_deprecated_setting('PP_EUPARL_INDEX') or 'parliament-euparl'
+
+    @property
+    def data_directory(self):
+        return get_deprecated_setting('PP_EUPARL_DATA') or super().data_directory
+
     languages = ['en']
     category = "parliament"
     document_context = document_context()
     description_page = 'euparl.md'
     image = 'euparl.jpeg'
     min_date = datetime(year=1999, month=7, day=20)
-    max_date = getattr(settings, 'PP_EUPARL_MAX_DATE', datetime.now())
+
+    @property
+    def max_date(self):
+        return get_deprecated_setting('PP_EUPARL_MAX_DATE') or datetime.now()
+
     language_field = 'original_language_code'
-    word_model_path = getattr(settings, 'PP_EUPARL_WM', None)
+
+    @property
+    def word_model_path(self):
+        return get_deprecated_setting('PP_EUPARL_WM') or super().word_model_path
+
     wordmodels_page = 'euparl.md'
 
 
     @property
     def subcorpora(self):
         return [
-            EUPDCorpReader(),
+            EUPDCorpReader(self.data_directory),
             ParliamentEuropeFromAPI(),
         ]
 
@@ -75,7 +96,11 @@ def source2dicts(self, source, **kwargs):
 
     debate_id = field_defaults.debate_id()
     debate_title = field_defaults.debate_title()
-    date = field_defaults.date(min_date, max_date)
+
+    @property
+    def date(self):
+        return field_defaults.date(self.min_date, self.max_date)
+
     party = field_defaults.party()
     party_full = field_defaults.party_full()
     party_id = field_defaults.party_id()
@@ -605,7 +630,10 @@ class EUPDCorpReader(RDSReader):
     Reader for the EUPDCorp dataset. Contains debates from 20/7/1999 to 8/2/2024
     '''
 
-    data_directory = settings.PP_EUPARL_DATA
+    data_directory = None
+
+    def __init__(self, data_directory: str):
+        self.data_directory = data_directory
 
     def sources(self, **kwargs):
         for filename in os.listdir(self.data_directory):

backend/corpora/parliament/finland-old.py

@@ -1,23 +1,28 @@
 from datetime import datetime
 from glob import glob
 
-from addcorpus.python_corpora.corpus import CSVCorpusDefinition
+from addcorpus.python_corpora.corpus import CSVCorpusDefinition, get_deprecated_setting
 from ianalyzer_readers.extract import CSV, Combined, Constant
 from addcorpus.python_corpora.filters import MultipleChoiceFilter
 from corpora.parliament.parliament import Parliament
 import corpora.parliament.utils.field_defaults as field_defaults
 from corpora.utils.constants import document_context
 from corpora.utils import formatting
 
-from django.conf import settings
 
 class ParliamentFinlandOld(Parliament, CSVCorpusDefinition):
     title = 'People and Parliament (Finland, 1863-1905)'
     description = 'Speeches from the early Finnish estates'
     max_date = datetime(year=1906, month=12, day=31)
     min_date = datetime(year=1863, month=1, day=1)
-    data_directory = settings.PP_FINLAND_OLD_DATA
-    es_index = getattr(settings, 'PP_FINLAND_OLD_INDEX', 'parliament-finland-old')
+
+    @property
+    def data_directory(self):
+        return get_deprecated_setting('PP_FINLAND_OLD_DATA') or super().data_directory
+
+    @property
+    def es_index(self):
+        return get_deprecated_setting('PP_FINLAND_OLD_INDEX') or 'parliament-finland-old'
 
     def sources(self, start, end):
         for csv_file in glob('{}/**/*.csv'.format(self.data_directory), recursive=True):

backend/corpora/parliament/finland.py

@@ -2,15 +2,14 @@
 from glob import glob
 from ianalyzer_readers.xml_tag import Tag, FindParentTag, PreviousSiblingTag, ParentTag
 
-from addcorpus.python_corpora.corpus import XMLCorpusDefinition
+from addcorpus.python_corpora.corpus import XMLCorpusDefinition, get_deprecated_setting
 from ianalyzer_readers.extract import XML, Combined, Constant, Metadata
 from corpora.parliament.parliament import Parliament
 import corpora.parliament.utils.field_defaults as field_defaults
 from corpora.utils.constants import document_context
 from bs4 import BeautifulSoup
 from corpora.parliament.utils.parlamint import extract_all_party_data, extract_people_data, extract_role_data, party_attribute_extractor, person_attribute_extractor, clean_value
 
-from django.conf import settings
 
 def format_role(values):
     id, roles = values
@@ -23,9 +22,18 @@ class ParliamentFinland(Parliament, XMLCorpusDefinition):
     title = 'People and Parliament (Finland, 1907-)'
     description = 'Speeches from the eduskunta'
     min_date = datetime(year=1907, month=1, day=1)
-    data_directory = settings.PP_FINLAND_DATA
-    es_index = getattr(settings, 'PP_FINLAND_INDEX', 'parliament-finland')
-    word_model_path = getattr(settings, 'PP_FINLAND_WM', None)
+
+    @property
+    def data_directory(self):
+        return get_deprecated_setting('PP_FINLAND_DATA') or super().data_directory
+
+    @property
+    def es_index(self):
+        return get_deprecated_setting('PP_FINLAND_INDEX') or 'parliament-finland'
+
+    @property
+    def word_model_path(self):
+        return get_deprecated_setting('PP_FINLAND_WM') or super().word_model_path
 
     def sources(self, start, end):
         for xml_file in glob('{}/**/*.xml'.format(self.data_directory), recursive=True):

backend/corpora/parliament/france.py

@@ -2,11 +2,9 @@
 import logging
 from datetime import datetime
 
-from django.conf import settings
-
 from corpora.parliament.parliament import Parliament
 from ianalyzer_readers.extract import Constant, Combined, CSV
-from addcorpus.python_corpora.corpus import CSVCorpusDefinition
+from addcorpus.python_corpora.corpus import CSVCorpusDefinition, get_deprecated_setting
 import corpora.parliament.utils.field_defaults as field_defaults
 from corpora.utils.formatting import underscore_to_space
 from corpora.utils.constants import document_context
@@ -15,12 +13,22 @@ class ParliamentFrance(Parliament, CSVCorpusDefinition):
     title = "People & Parliament (France 1881-2022)"
     description = "Speeches from the 3rd, 4th and 5th republic of France"
     min_date = datetime(year=1881, month=1, day=1)
-    data_directory = settings.PP_FR_DATA
-    es_index = getattr(settings, 'PP_FR_INDEX', 'parliament-france')
+
+    @property
+    def data_directory(self):
+        return get_deprecated_setting('PP_FR_DATA') or super().data_directory
+
+    @property
+    def es_index(self):
+        return get_deprecated_setting('PP_FR_INDEX') or 'parliament-france'
+
     image = 'france.jpg'
     languages = ['fr']
     description_page = 'france.md'
-    word_model_path = getattr(settings, 'PP_FR_WM', None)
+
+    @property
+    def word_model_path(self):
+        return get_deprecated_setting('PP_FR_WM') or super().word_model_path
 
     field_entry = 'speech_id'