Merge pull request #154 from open-data/feature/ds-reference-tables

Reference Tables & Org Specific Fields

Author: JVickery-TBS

changes/154.a.changes

@@ -0,0 +1 @@
+Added output functionality to the Data Dictionary, Excel References, and JSON Schemas to display Organizaion based choices.

changes/154.a.feature

@@ -0,0 +1 @@
+Added a new subcommand `create-ref-tables` which executes pSQL scripts defined in the `recombinant.reference_definitions` config option. This allows for creation of advanced DataStore tables outside of the CKAN framework. These tables are not editable by CKAN users.

changes/154.b.changes

@@ -0,0 +1 @@
+Improved the `format_trigger_error` to support multiple splitting of the private code point `\uF8FF`. This allows for more dynamic data error messages, while still having the text go through the translation interfaces properly.

changes/154.b.feature

@@ -0,0 +1 @@
+Recombinant schemas now support `choices_reference_table` definition for fields, which can reference an existing table in the DataStore database. Choices are filterable with `choices_filter_query` which accepts a basic pSQL WHERE clause.

changes/154.c.feature

@@ -0,0 +1 @@
+Recombinant schemas now support `choices_fiscal_year` definition for fields. This is a YAML Mapping which can take `min_year (default: 2005)`, `max_year (default: current)`, and `month_start (default: 4/April)`

ckanext/recombinant/assets/js/code_block_buttons.js

@@ -1,49 +1,29 @@
 window.addEventListener('load', function(){
   $(document).ready(function() {
-
     // Code example switching
     let codeExampleControl = $('#api-access-example-code-control');
     let codeBlocks = $('#api-access-accordion').find('figure');
-
     if( codeExampleControl.length > 0 ){
-
       let controlButtons = $(codeExampleControl).find('input[name="api-access-example-code"]');
-
       if( controlButtons.length > 0 ){
-
         $(controlButtons).on('change', function(_event){
-
           let selectedCode = $(codeExampleControl).find('input[name="api-access-example-code"]:checked').val();
-
           $(codeBlocks).each(function(_index, _codeBlock){
-
             if( $(_codeBlock).hasClass(selectedCode) ){
-
               $(_codeBlock).show();
-
             }else{
-
               $(_codeBlock).hide();
-
             }
-
           });
-
         });
-
       }
-
     }
 
     // Activity tab link
     let activityTab = $('#activity-lnk');
-
     if( activityTab.length > 0 ){
-
       let link = $('#activity').find('a').first().attr('href');
-
       if( link && link.length > 0 ){
-
         $(activityTab).attr('href', link);
         $(activityTab).removeAttr('aria-controls');
         $(activityTab).attr('tabindex', 0);
@@ -59,6 +39,7 @@ window.addEventListener('load', function(){
           _event.preventDefault();
           _goto_activity();
         });
+
         $(activityTab).on('keyup.Link', function(_event){
           let keyCode = _event.keyCode ? _event.keyCode : _event.which;
           // space and enter keys required for a11y
@@ -67,10 +48,15 @@ window.addEventListener('load', function(){
             _goto_activity();
           }
         });
-
       }
-
     }
 
+    // Rebind fix for wet-boew tab redraw
+    let dictionaryTables = $('table.recombinant-data-dictionary[data-module="table-toggle-more"]');
+    if( dictionaryTables && dictionaryTables.length > 0 && typeof ckan != 'undefined' ){
+      $(dictionaryTables).each(function(_index, _table){
+        ckan.module.initialize(_table);
+      });
+    }
   });
 });

ckanext/recombinant/cli.py

@@ -5,19 +5,23 @@
 import json
 import re
 from openpyxl.formula import Tokenizer
+import sqlalchemy as sa
 
 from typing import Dict, List, Any, Optional, TextIO
 
 from ckan.logic import ValidationError
 from ckanapi import LocalCKAN, NotFound
+from ckanext.datastore.backend import DatastoreBackend
+from ckanext.datastore.backend.postgres import DatastorePostgresqlBackend
 
 from ckanext.recombinant.tables import (
     get_dataset_type_for_resource_name,
     get_dataset_types,
     get_chromo,
     get_geno,
     get_target_datasets,
-    get_resource_names
+    get_resource_names,
+    get_reference_tables_sql,
 )
 from ckanext.recombinant.read_csv import csv_data_batch
 from ckanext.recombinant.write_excel import excel_template
@@ -910,3 +914,17 @@ def _template(dataset_type: str,
     with open(output_file, 'w') as out:
         # type_ignore_reason: incomplete typing
         tmpl.save(out)  # type: ignore
+
+
+@recombinant.command(short_help="Run all the sql scripts "
+                                "from recombinant.reference_definitions")
+def create_ref_tables():
+    """
+    Run all the sql scripts from recombinant.reference_definitions
+    """
+    # type_ignore_reason: incomplete typing
+    backend: DatastorePostgresqlBackend = DatastoreBackend.\
+        get_active_backend()  # type: ignore
+    with backend._get_write_engine().begin() as connection:
+        connection.execute(sa.text(get_reference_tables_sql()))
+    click.echo('Done!')

ckanext/recombinant/errors.py

@@ -29,7 +29,5 @@ def format_trigger_error(error_values: List[str]) -> Generator[str, None, None]:
     replacements, allowing i18n support in the framework.
     """
     for e in error_values:
-        if '\uF8FF' in e:
-            yield _(e.split('\uF8FF')[0]).format(e.split('\uF8FF')[1])
-        else:
-            yield _(e)
+        err, *args = e.split('\uF8FF')
+        yield _(err).format(*args)

ckanext/recombinant/helpers.py

@@ -1,6 +1,8 @@
 import json
 import os.path
 from markupsafe import Markup
+import sqlalchemy as sa
+from datetime import datetime
 
 from typing import Dict, Any, Optional, List, Union
 
@@ -9,6 +11,13 @@
 import ckanapi
 from ckan.lib.helpers import lang
 
+from ckanext.datastore.backend import DatastoreBackend
+from ckanext.datastore.backend.postgres import (
+    DatastorePostgresqlBackend,
+    identifier,
+    literal_string
+)
+
 from ckanext.recombinant.tables import (
     get_chromo, get_geno, get_dataset_types,
     get_published_resource_resource_name
@@ -129,9 +138,24 @@ def recombinant_example(resource_name: str,
     return left[2:] + ('\n' + left[2:]).join(out.split('\n')[1:-1])
 
 
-def recombinant_choice_fields(resource_name: str,
-                              all_languages: bool = False,
-                              prefer_lang: Optional[str] = None) -> Dict[str, Any]:
+def get_choices_fiscal_year(min_year: int = 2005,
+                            max_year: Optional[int] = None,
+                            month_start: int = 4) -> List[str]:
+    """
+    Dynamically generate choices for fiscal years.
+    """
+    if not max_year:
+        max_year = datetime.now().year if \
+            datetime.now().month >= month_start else datetime.now().year - 1
+    return [f'{i}-{i + 1}'
+            for i in range(min_year, max_year + 1)]
+
+
+def recombinant_choice_fields(
+        resource_name: str,
+        all_languages: bool = False,
+        prefer_lang: Optional[str] = None,
+        org_name: Optional[str] = None) -> Dict[str, Any]:
     """
     Return a datastore_id: choices dict from the resource definition
     that contain lists of choices, with labels pre-translated
@@ -168,11 +192,42 @@ def key_fn(v: str) -> Any:
             if v not in exclude_choices
         ]
 
+    def build_choices_psql(f: Dict[str, Any]):
+        # type_ignore_reason: incomplete typing
+        backend: DatastorePostgresqlBackend = DatastoreBackend.\
+            get_active_backend()  # type: ignore
+        with backend._get_read_engine().begin() as connection:
+            filter_clause = f.get('choices_filter_query', '')
+            if filter_clause and "{org}" in filter_clause and not org_name:
+                filter_clause = ''  # if no org_name passed, cannot query it
+            filter_clause = filter_clause.format(
+                org=literal_string(org_name) if org_name else '')
+            results = connection.execute(sa.text("""
+                SELECT * FROM {ref_table} {filter_clause}
+                ORDER BY {ds_id} ASC;
+            """.format(
+                ref_table=identifier(f['choices_reference_table']),
+                filter_clause=filter_clause,
+                ds_id=identifier(f['datastore_id'])
+            ).replace(':', r'\:'))).mappings().fetchall()  # avoid bind params
+        out[f['datastore_id']] = [
+            (r[f['datastore_id']],
+             dict(en=r['label_en'],
+                  fr=r['label_fr'],
+                  valid_orgs=r.get('org_years') or {})
+             if all_languages else
+             r['label_%s' % (prefer_lang or lang())]) for r in results]
+
     for f in chromo['fields']:
         if 'choices' in f:
             build_choices(f, f['choices'])
         elif 'choices_file' in f and '_path' in chromo:
             build_choices(f, _read_choices_file(chromo, f))
+        elif 'choices_reference_table' in f:
+            build_choices_psql(f)
+        elif 'choices_fiscal_year' in f:
+            out[f['datastore_id']] = [
+                (v, v) for v in get_choices_fiscal_year(**f['choices_fiscal_year'])]
 
     return out
 

ckanext/recombinant/tables.py

@@ -4,10 +4,14 @@
 
 This module provides access to those definitions.
 """
+import importlib
+import os
+
 from typing import List, Dict, Optional, Any
 
 import ckan.plugins as p
 
+from ckanext.datastore.cli import _parse_db_config
 from ckanext.recombinant.errors import RecombinantException
 
 
@@ -97,3 +101,27 @@ def get_target_datasets() -> List[str]:
     """
     genos = _get_plugin()._genos
     return sorted((t['target_dataset'] for t in genos.values()))
+
+
+def get_reference_tables_sql():
+    """
+    Compiles all sql scripts from recombinant.reference_definitions
+    """
+    ref_tables_uris = p.toolkit.config.get('recombinant.reference_definitions',
+                                           "").split()
+    write_url = _parse_db_config('ckan.datastore.write_url')
+    read_url = _parse_db_config('ckan.datastore.read_url')
+    sql = ""
+    for uri in ref_tables_uris:
+        module, file_name = uri.split(':', 1)
+        try:
+            m = importlib.import_module(module)
+        except ImportError:
+            raise RecombinantException('Could not load module path %s' % uri)
+        _p = m.__path__[0]
+        _p = os.path.join(_p, file_name)
+        if not os.path.exists(_p):
+            raise RecombinantException('File path does not exist %s' % uri)
+        with open(_p) as f:
+            sql += f.read()
+    return sql.format(readuser=read_url['db_user'], writeuser=write_url['db_user'])