Org Based Triggers Capabilities

changes/147.feature

@@ -0,0 +1 @@
+Created new schema directive `per_org_triggers` which allows creation of Organization based choices for fields. Definable with `valid_orgs` in the choice objects.

ckanext/recombinant/cli.py

@@ -166,18 +166,25 @@ def show(dataset_type: Optional[str] = None,
     is_flag=True,
     help="All dataset types/resource names",
 )
+@click.option(
+    "-o",
+    "--org-name",
+    help="Organization name to create/update triggers for. E.g. tbs-sct",
+)
 @click.option('-v', '--verbose', is_flag=True,
               type=click.BOOL, help='Increase verbosity.')
 def create_triggers(dataset_type: Optional[List[str]] = None,
                     all_types: bool = False,
-                    verbose: bool = False):
+                    verbose: bool = False,
+                    org_name: Optional[str] = None):
     """
     Create and update triggers
 
     Full Usage:\n
         recombinant create-triggers (-a | DATASET_TYPE ...)
     """
-    _create_triggers(dataset_type, all_types, verbose=verbose)
+    _create_triggers(dataset_type, all_types,
+                     verbose=verbose, org_name=org_name)
 
 
 @recombinant.command(
@@ -526,14 +533,35 @@ def _expand_resource_names(resource_names: Optional[List[str]],
 
 def _create_triggers(dataset_types: Optional[List[str]],
                      all_types: bool = False,
-                     verbose: bool = False):
+                     verbose: bool = False,
+                     org_name: Optional[str] = None):
     """
     Create and update triggers
     """
     lc = LocalCKAN()
     for dtype in _expand_dataset_types(dataset_types, all_types):
+        if verbose:
+            click.echo('(re)Creating DataStore DB triggers for %s' % dtype)
         for chromo in get_geno(dtype)['resources']:
-            _update_triggers(lc, chromo)
+            if chromo.get('per_org_triggers'):
+                # need to loop all datasets of this type...
+                orgs = [org_name] if org_name else _get_orgs()
+                packages = _get_packages(dtype, orgs)
+                existing = dict((p['owner_org'], p) for p in packages)
+                for o in orgs:
+                    if o not in existing:
+                        continue
+                    if verbose:
+                        click.echo('(re)Creating %s DataStore '
+                                   'DB triggers for Organization %s' % (dtype, o))
+                    _update_triggers(lc, chromo, o)
+                    if verbose:
+                        click.echo('Successfully (re)created %s '
+                                   'triggers for Organization %s' % (dtype, o))
+            else:
+                _update_triggers(lc, chromo)
+                if verbose:
+                    click.echo('Successfully (re)created triggers for %s' % dtype)
 
 
 def _remove_empty(dataset_types: Optional[List[str]],

ckanext/recombinant/helpers.py

@@ -4,7 +4,7 @@
 
 from typing import Dict, Any, Optional, List, Union
 
-from ckan.plugins.toolkit import c, config
+from ckan.plugins.toolkit import c, config, get_action
 from ckan.plugins.toolkit import _ as gettext
 import ckanapi
 from ckan.lib.helpers import lang
@@ -129,9 +129,36 @@ def recombinant_example(resource_name: str,
     return left[2:] + ('\n' + left[2:]).join(out.split('\n')[1:-1])
 
 
-def recombinant_choice_fields(resource_name: str,
-                              all_languages: bool = False,
-                              prefer_lang: Optional[str] = None) -> Dict[str, Any]:
+def _filter_choices_for_org(choices: Dict[str, Any],
+                            org_name: str):
+    invalid_choices = []
+    for choice_key, choice_obj in choices.items():
+        if 'valid_orgs' not in choice_obj:
+            continue
+        _valid_orgs = choice_obj['valid_orgs']
+        if isinstance(_valid_orgs, dict):
+            _valid_orgs = _valid_orgs.keys()
+        if org_name in _valid_orgs:
+            continue
+        invalid_choices.append(choice_key)
+    for key in invalid_choices:
+        del choices[key]
+
+
+def recombinant_org_specific_fields(resource_name: str) -> Dict[str, Any]:
+    chromo = recombinant_get_chromo(resource_name)
+    if not chromo:
+        return {}
+    return set(
+        x for trigger in chromo.get('per_org_triggers', {}).values() for x in trigger)
+
+
+def recombinant_choice_fields(
+        resource_name: str,
+        all_languages: bool = False,
+        prefer_lang: Optional[str] = None,
+        org_name: Optional[str] = None,
+        for_published_resource: Optional[bool] = False) -> Dict[str, Any]:
     """
     Return a datastore_id: choices dict from the resource definition
     that contain lists of choices, with labels pre-translated
@@ -146,6 +173,9 @@ def recombinant_choice_fields(resource_name: str,
     if not chromo:
         return {}
 
+    org_specific_fields = set(
+        x for trigger in chromo.get('per_org_triggers', {}).values() for x in trigger)
+
     def build_choices(f: Dict[str, Any], choices: Dict[str, Any]):
         order_expr = f.get('choice_order_expression')
         if order_expr:
@@ -161,6 +191,8 @@ def key_fn(v: str) -> Any:
             key_fn = None  # type: ignore
 
         exclude_choices = f.get('exclude_choices', [])
+        if f['datastore_id'] in org_specific_fields and not for_published_resource:
+            _filter_choices_for_org(choices, org_name)
         out[f['datastore_id']] = [
             (v, choices[v] if all_languages
                 else recombinant_language_text(choices[v], prefer_lang))
@@ -177,6 +209,72 @@ def key_fn(v: str) -> Any:
     return out
 
 
+def recombinant_choice_field_valid_orgs(
+        resource_name: str, field_id: str,
+        prefer_lang: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Return Organization translated titles for a Recombinant choice
+    field which uses valid_orgs
+    """
+    chromo = recombinant_get_chromo(resource_name)
+    if not chromo:
+        return {}
+
+    org_specific_fields = set(
+        x for trigger in chromo.get('per_org_triggers', {}).values() for x in trigger)
+
+    if field_id not in org_specific_fields:
+        return {}
+
+    recombinant_field = None
+    for f in chromo['fields']:
+        if f['datastore_id'] == field_id:
+            recombinant_field = f
+            break
+
+    if not recombinant_field:
+        return {}
+
+    choices = None
+    if 'choices' in recombinant_field:
+        choices = recombinant_field['choices']
+    elif 'choices_file' in f and '_path' in chromo:
+        choices = _read_choices_file(chromo, f)
+
+    if not choices:
+        return {}
+
+    keyed_orgs = {}
+    orgs = get_action('organization_list')({'ignore_auth': True}, {
+        'include_dataset_count': False,
+        'all_fields': True,
+        'include_extras': True,
+    })
+    for o in orgs:
+        keyed_orgs[o['name']] = recombinant_language_text(
+            o['title_translated'], prefer_lang) if \
+                'title_translated' in o else o['title']
+
+    return_dict = {}
+    for choice_key, choice_obj in choices.items():
+        if 'valid_orgs' not in choice_obj:
+            continue
+        _valid_orgs = choice_obj['valid_orgs']
+        if isinstance(_valid_orgs, dict):
+            _valid_orgs = _valid_orgs.keys()
+        if choice_key not in return_dict:
+            return_dict[choice_key] = {}
+        for o in _valid_orgs:
+            if o not in keyed_orgs:
+                return_dict[choice_key] = choice_key
+                continue
+            return_dict[choice_key] = keyed_orgs[o]
+    # TODO: put org names and titles for the choices...
+    # FIXME: do we just do this in our yaml choices file so we do not have to always do it on the fly??
+
+    return return_dict
+
+
 def _read_choices_file(chromo: Dict[str, Any], f: Dict[str, Any]) -> Dict[str, Any]:
     with open(os.path.join(chromo['_path'], f['choices_file'])) as cf:
         return load.load(cf)

ckanext/recombinant/logic.py

@@ -1,7 +1,7 @@
 from six import string_types
 from ckan.plugins.toolkit import _, chained_action, h, side_effect_free
 
-from typing import Dict, Any, List, Tuple
+from typing import Dict, Any, List, Tuple, Optional
 from ckan.types import Context, DataDict, Action, ChainedAction
 
 from sqlalchemy import and_
@@ -18,7 +18,10 @@
     format_trigger_error
 )
 from ckanext.recombinant.datatypes import datastore_type
-from ckanext.recombinant.helpers import _read_choices_file
+from ckanext.recombinant.helpers import (
+    _read_choices_file,
+    _filter_choices_for_org
+)
 
 from ckanext.datastore.backend.postgres import literal_string
 
@@ -301,7 +304,8 @@ def _update_datastore(lc: LocalCKAN,
                     new_fields.append(f)
                 fields = new_fields
 
-        trigger_names = _update_triggers(lc, chromo)
+        trigger_names = _update_triggers(lc, chromo,
+                                         dataset['organization']['name'])
 
         chromo_foreign_keys = chromo.get('datastore_foreign_keys', None)
         foreign_keys = {}
@@ -327,32 +331,45 @@ def _update_datastore(lc: LocalCKAN,
             force=True)
 
 
-def _update_triggers(lc: LocalCKAN, chromo: Dict[str, Any]) -> List[str]:
+def _update_triggers(lc: LocalCKAN, chromo: Dict[str, Any],
+                     org_name: Optional[str] = None) -> List[str]:
     definitions = dict(chromo.get('trigger_strings', {}))
     trigger_names = []
 
+    org_specific_fields = set(
+        x for trigger in chromo.get('per_org_triggers', {}).values() for x in trigger)
+
     for f in chromo['fields']:
+        # TODO: pass in an orgs list instead, and try to have it not constantly yaml load
         if 'choices' in f:
             if f['datastore_id'] in definitions:
                 raise RecombinantConfigurationError(
                     "trigger_string {name} can't be used because that "
                     "name is required for the {name} field choices".format(
                         name=f['datastore_id']))
-            definitions[f['datastore_id']] = sorted(f['choices'])
+            choice_definitions = f['choices']
+            if f['datastore_id'] in org_specific_fields:
+                _filter_choices_for_org(choice_definitions, org_name)
+            definitions[f['datastore_id']] = sorted(choice_definitions)
         elif 'choices_file' in f and '_path' in chromo:
             if f['datastore_id'] in definitions:
                 raise RecombinantConfigurationError(
                     "trigger_string {name} can't be used because that "
                     "name is required for the {name} field choices".format(
                         name=f['datastore_id']))
-            definitions[f['datastore_id']] = sorted(_read_choices_file(chromo, f))
+            choice_definitions = _read_choices_file(chromo, f)
+            if f['datastore_id'] in org_specific_fields:
+                _filter_choices_for_org(choice_definitions, org_name)
+            definitions[f['datastore_id']] = sorted(choice_definitions)
 
     for tr in chromo.get('triggers', []):
         if isinstance(tr, dict):
             if len(tr) != 1:
                 raise RecombinantConfigurationError(
                     "inline trigger may have only one key: " + repr(tr.keys()))
             ((trname, trcode),) = tr.items()
+            if org_name and trname in chromo.get('per_org_triggers', {}):
+                trname += '_{}'.format(org_name.replace('-', '_'))
             trigger_names.append(trname)
             try:
                 lc.action.datastore_function_create(
@@ -365,6 +382,8 @@ def _update_triggers(lc: LocalCKAN, chromo: Dict[str, Any]) -> List[str]:
             except NotAuthorized:
                 pass  # normal users won't be able to reset triggers
         else:
+            if org_name and tr in chromo.get('per_org_triggers', {}):
+                tr += '_{}'.format(org_name.replace('-', '_'))
             trigger_names.append(tr)
     return trigger_names
 

ckanext/recombinant/plugins.py

@@ -118,6 +118,10 @@ def get_helpers(self) -> Dict[str, Callable[..., Any]]:
             'recombinant_get_types': helpers.recombinant_get_types,
             'recombinant_example': helpers.recombinant_example,
             'recombinant_choice_fields': helpers.recombinant_choice_fields,
+            'recombinant_org_specific_fields':
+                helpers.recombinant_org_specific_fields,
+            'recombinant_choice_field_valid_orgs':
+                helpers.recombinant_choice_field_valid_orgs,
             'recombinant_show_package': helpers.recombinant_show_package,
             'recombinant_get_field': helpers.recombinant_get_field,
             'recombinant_published_resource_chromo':

ckanext/recombinant/templates/datastore/snippets/dictionary_view.html

@@ -3,12 +3,12 @@
 {% block dictionary_view %}
   {% set chromo = h.recombinant_published_resource_chromo(res.id) %}
   {% if chromo %}
+    {# TODO: make org specific!!! this is the snippet for the published resource. so we need to display what options are valid for what orgs. need org names too??? #}
     {% set choice_fields=h.recombinant_choice_fields(chromo.resource_name) %}
     <div class="module-content">
       {% snippet 'recombinant/snippets/data_dictionary.html',
         chromo=chromo, all_expanded=false, is_published_resource=true %}
     </div>
-
   {% else %}
     {{ super() }}
   {% endif %}

ckanext/recombinant/templates/recombinant/resource_edit.html

@@ -114,7 +114,9 @@ <h3>{{ _('DataStore Dump') }}{% snippet 'snippets/sysadmin_only.html' %}</h3>
                 {{ _("Reference") }}
               </summary>
               {% snippet "recombinant/snippets/data_dictionary.html",
-                chromo=h.recombinant_get_chromo(resource.name), all_expanded=true, is_published_resource=false %}
+                chromo=h.recombinant_get_chromo(resource.name), all_expanded=true,
+                is_published_resource=false, organization=organization,
+                organization_display_name=organization_display_name %}
             </details>
             <details id="api">
               <summary><span class="glyphicon glyphicon-wrench"></span>

ckanext/recombinant/templates/recombinant/snippets/data_dictionary.html

@@ -1,21 +1,27 @@
-{% set choice_fields=h.recombinant_choice_fields(chromo.resource_name) %}
+{% set choice_fields=h.recombinant_choice_fields(chromo.resource_name, org_name=organization.name if organization else none, for_published_resource=is_published_resource) %}
+{% set org_specifc_fields=h.recombinant_org_specific_fields(chromo.resource_name) %}
+{% set ns = namespace(choice_fields_valid_orgs={}) %}
+{% for org_specifc_field in org_specifc_fields %}
+  {{ h.recombinant_choice_field_valid_orgs(chromo.resource_name, org_specifc_field) }}
+{% endfor %}
 
 <h3>{{ _("Data Dictionary") }}</h3>
 
 <div class="form-group control-medium row">
-  {% set excel_dictionary = 'recombinant.published_data_dictionary' if is_published_resource else 'recombinant.data_dictionary' %}
-  {% set json_dictionary = 'recombinant.published_schema_json' if is_published_resource else 'recombinant.schema_json' %}
+  {%- set excel_dictionary_link = h.url_for('recombinant.data_dictionary', dataset_type=chromo.dataset_type) -%}
+  {%- set json_dictionary_link = h.url_for('recombinant.schema_json', dataset_type=chromo.dataset_type) -%}
+  {%- if is_published_resource -%}
+    {%- set excel_dictionary_link = h.url_for('recombinant.published_data_dictionary', dataset_type=chromo.dataset_type) -%}
+    {%- set json_dictionary_link = h.url_for('recombinant.published_schema_json', dataset_type=chromo.dataset_type) -%}
+  {%- elif org_specifc_fields -%}
+    {%- set excel_dictionary_link = h.url_for('recombinant.org_based_data_dictionary', dataset_type=chromo.dataset_type, org_name=organization.name if organization else none) -%}
+    {%- set json_dictionary_link = h.url_for('recombinant.org_based_schema_json', dataset_type=chromo.dataset_type, org_name=organization.name if organization else none) -%}
+  {%- endif -%}
   <div class="col-md-6">
-    <a class="button" href="{{ h.url_for(
-      excel_dictionary,
-      dataset_type=chromo.dataset_type) }}"><span class="glyphicon glyphicon-download-alt"></span>
-      {{ _('Download data dictionary') }} XLSX</a>
+    <a class="button" href="{{- excel_dictionary_link -}}"><span class="glyphicon glyphicon-download-alt"></span>&nbsp;{{- _('Download data dictionary') }} XLSX</a>
   </div>
   <div class="col-md-6">
-    <a class="button" href="{{ h.url_for(
-      json_dictionary,
-      dataset_type=chromo.dataset_type) }}"><span class="glyphicon glyphicon-wrench"></span>
-      {{ _('Schema as JSON') }}</a>
+    <a class="button" href="{{- json_dictionary_link -}}"><span class="glyphicon glyphicon-wrench"></span>&nbsp;{{- _('Schema as JSON') -}}</a>
   </div>
 </div>
 
@@ -56,7 +62,7 @@ <h3>{{ _("Data Dictionary") }}</h3>
         {% endcall %}
         {% set choices = choice_fields.get(field.datastore_id) %}
         {% if choices %}
-          {% call dictionary_field(_('Choices')) %}
+          {% call dictionary_field(_('Choices') if is_published_resource or field.datastore_id not in org_specifc_fields or not organization else _('Choices for ') ~ organization_display_name(organization, 120)) %}
             <table class="table table-striped table-bordered table-condensed" data-module-show-label="{{ _('Show more') }}" data-module-hide-label="{{ _('Hide') }}" data-module="table-toggle-more">
               <thead>
                 <tr>