Merge pull request #1503 from ImagingDataCommons/release-49-sp

-> Tooltip fixes/improvements

Author: s-paquette

etl/etl.py

@@ -61,7 +61,7 @@
 FIELD_MAP = {x: i for i, x in enumerate([
     "collection_id", "collection_uuid", "name", "collections", "image_types", "supporting_data", "subject_count", "doi",
     "source_url", "cancer_type", "species", "location", "analysis_artifacts", "description", "collection_type",
-    "program", "access", "date_updated", "tcia_wiki_collection_id", "license_short_name", "active", "total_size"
+    "program", "access", "date_updated", "tcia_wiki_collection_id", "license_short_name", "active", "total_size", "total_size_with_ar"
 ])}
 
 TOKENIZED_FIELDS = ["PatientID", "SeriesInstanceUID", "StudyInstanceUID"]
@@ -146,8 +146,8 @@
 }
 
 SOLR_SINGLE_VAL = {
-    "StudyInstanceUID": ["PatientID", "StudyInstanceUID", "crdc_study_uuid"],
-    "SeriesInstanceUID": ["PatientID", "StudyInstanceUID", "SeriesInstanceUID", "crdc_study_uuid", "crdc_series_uuid"]
+    "StudyInstanceUID": ["PatientID", "StudyInstanceUID", "crdc_study_uuid","instance_size"],
+    "SeriesInstanceUID": ["PatientID", "StudyInstanceUID", "SeriesInstanceUID", "crdc_study_uuid", "crdc_series_uuid", "instance_size"]
 }
 
 ETL_CONFIG = {}
@@ -356,7 +356,7 @@ def load_collections(filename, data_version="8.0"):
         exact_collection_fields = [
             "collection_id", "collection_uuid", "name", "collections", "image_types", "supporting_data", "subject_count", "doi",
             "source_url", "cancer_type", "species", "location", "analysis_artifacts", "description", "collection_type",
-            "access", "date_updated", "active", "total_size"]
+            "access", "date_updated", "active", "total_size", "total_size_with_ar"]
         field_map = FIELD_MAP
         for line in csv_reader(collection_file):
             if COLLECTION_HEADER_CHK in line:

idc/urls.py

@@ -51,7 +51,6 @@
     re_path(r'^explore/manifest/$', views.explorer_manifest, name='get_explore_manifest'),
     re_path(r'^explore/manifest/series/$', views.explorer_manifest, name='get_series_ids_manifest'),
     re_path(r'^tables/', views.populate_tables, name='populate_tables'),
-    re_path(r'^studymp/', views.studymp, name='studymp'),
     re_path(r'^warning/', views.warn_page, name='warn'),
     re_path(r'^about/$', views.about_page, name='about_page'),
     re_path(r'^dashboard/', views.dashboard_page, name='dashboard'),

idc/views.py

@@ -51,6 +51,7 @@
 from solr_helpers import build_solr_query, query_solr_and_format_result
 
 from idc.settings import MAX_FILE_LIST_REQUEST
+from idc_collections.collex_metadata_utils import convert_disk_size
 
 debug = settings.DEBUG
 logger = logging.getLogger(__name__)
@@ -247,86 +248,6 @@ def save_ui_hist(request):
     return JsonResponse({}, status=status)
 
 
-# returns various metadata mappings for selected projects used in calculating cart selection
-# counts 'on the fly' client side
-def studymp(request):
-    response = {}
-    status = 200
-    sources = ImagingDataCommonsVersion.objects.get(active=True).get_data_sources(
-            active=True, source_type=DataSource.SOLR,
-            aggregate_level="StudyInstanceUID"
-        )
-    data_types = [DataSetType.IMAGE_DATA, DataSetType.ANCILLARY_DATA, DataSetType.DERIVED_DATA]
-    data_sets = DataSetType.objects.filter(data_type__in=data_types)
-    aggregate_level='StudyInstanceUID'
-    versions=[]
-    versions = ImagingDataCommonsVersion.objects.filter(
-        version_number__in=versions
-    ).get_data_versions(active=True) if len(versions) else ImagingDataCommonsVersion.objects.filter(
-        active=True
-    ).get_data_versions(active=True)
-    aux_sources = data_sets.get_data_sources().filter(
-        source_type=DataSource.SOLR,
-        aggregate_level__in=["case_barcode", "sample_barcode", aggregate_level],
-        id__in=versions.get_data_sources().filter(source_type=DataSource.SOLR).values_list("id", flat=True)
-    ).distinct()
-
-    try:
-       req = request.GET if request.method == 'GET' else request.POST
-       filters = json.loads(req.get('filters', '{}'))
-
-       mxSeries = int(req.get('mxseries'))
-       mxStudies= int(req.get('mxstudies'))
-       limit = int(req.get('limit', mxStudies))
-       offset = int(req.get('offset',0))
-
-       casestudymp = dict()
-       studymp = dict()
-       projstudymp = dict()
-
-       idsEx = get_collex_metadata(
-            filters, ['collection_id', 'PatientID','StudyInstanceUID', 'SeriesInstanceUID'], record_limit=limit,
-            sources=sources, offset=offset, records_only=True, custom_facets={}, aux_sources=aux_sources,
-            collapse_on='StudyInstanceUID', counts_only=False, filtered_needed=False,
-            raw_format=True, default_facets=False, sort=None
-        )
-
-       logger.debug("[STATUS] records pulled: {}".format(len(idsEx['docs'])))
-
-       for doc in idsEx['docs']:
-          proj=doc['collection_id'][0]
-          patientid=doc['PatientID']
-          studyid= doc['StudyInstanceUID']
-          cnt = len(doc['SeriesInstanceUID'])
-
-          if not patientid in casestudymp:
-              casestudymp[patientid]={}
-          if not proj in projstudymp:
-              projstudymp[proj] = {}
-          studymp[studyid]={}
-          studymp[studyid]['cnt'] = cnt
-          studymp[studyid]['proj'] = proj
-          studymp[studyid]['PatientID'] = patientid
-          studymp[studyid]['val'] = []
-          projstudymp[proj][studyid] = cnt
-          casestudymp[patientid][studyid] = cnt
-
-       response["studymp"] = studymp
-       response['casestudymp'] = casestudymp
-       response['projstudymp'] = projstudymp
-
-    except Exception as e:
-        logger.error("[ERROR] While attempting to get studymp:")
-        logger.exception(e)
-        messages.error(
-           request,
-           "Encountered an error when attempting to get the studymp - please contact the administrator."
-        )
-        status = 400
-
-    return JsonResponse(response, status=status)
-
-
 def populate_tables(request):
     response = {}
     status = 200
@@ -648,6 +569,7 @@ def cart_data(request):
         aggregate_level = req.get('aggregate_level', 'StudyInstanceUID')
         results_level = req.get('results_level', 'StudyInstanceUID')
         dois_only = bool(req.get('dois_only', 'false').lower() == 'true')
+        size_only = bool(req.get('size_only', 'false').lower() == 'true')
 
         partitions = json.loads(req.get('partitions', '{}'))
 
@@ -656,22 +578,29 @@ def cart_data(request):
         length = int(req.get('length', 100))
         mxseries = int(req.get('mxseries',1000))
 
+        doi_or_size_only = (dois_only or size_only)
+
         if len(partitions) <= 0 or aggregate_level not in ['SeriesInstanceUID', 'StudyInstanceUID']:
             response['numFound'] = 0
             response['docs'] = []
         else:
             if aggregate_level == 'StudyInstanceUID':
                 response = get_cart_data_studylvl(
-                    filtergrp_list, partitions, limit, offset, length, mxseries, with_records=(not dois_only),
-                    results_lvl=results_level, dois_only=dois_only
+                    filtergrp_list, partitions, limit, offset, length, mxseries, with_records=(not doi_or_size_only),
+                    results_lvl=results_level, dois_only=dois_only, size_only=size_only
                 )
             elif aggregate_level == 'SeriesInstanceUID':
                 response = get_cart_data_serieslvl(
-                    filtergrp_list, partitions, field_list if not dois_only else None, limit, offset,
-                    with_records=(not dois_only), dois_only=dois_only
+                    filtergrp_list, partitions, field_list if (not doi_or_size_only) else None, limit, offset,
+                    with_records=(not doi_or_size_only), dois_only=dois_only, size_only=size_only
                 )
         if dois_only:
             response = {'dois': response['dois']}
+        if size_only:
+            response = {
+                "total_size": response['total_size'],
+                "display_size": convert_disk_size(response['total_size'])
+            }
     except Exception as e:
         logger.error("[ERROR] While loading cart:")
         logger.exception(e)
@@ -704,7 +633,7 @@ def get_series(request, collection_id=None, patient_id=None, study_uid=None):
                 filter_query = build_solr_query(
                     { w: v for x in filtergrp_list for w, v in list(x.items())},
                     with_tags_for_ex=False,
-                    search_child_records_by=None, solr_default_op='AND'
+                    search_child_records_by={w: "StudyInstanceUID" for x in filtergrp_list for w in x}, solr_default_op='AND'
                 )
                 result = query_solr_and_format_result(
                     {

static/css/style.css

@@ -4524,8 +4524,11 @@ d-topics-list iframe {
 }
 
 .download-all-disabled {
-  cursor: not-allowed;
-  color: gray;
+  cursor: not-allowed !important;
+}
+
+.fa.download-all-disabled {
+    color: gray;
 }
 
 #floating-message {

static/css/vendor.css

@@ -2832,7 +2832,7 @@ select[multiple].input-lg,
     box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125); }
   .btn.disabled, .btn[disabled],
   fieldset[disabled] .btn {
-    cursor: not-allowed;
+    cursor: not-allowed !important;
     opacity: 0.65;
     filter: alpha(opacity=65);
     -webkit-box-shadow: none;

static/js/base.js

@@ -276,6 +276,7 @@ define(['jquery', 'utils'], function($, utils) {
             utils.removeCookie(name, path);
         },
         blockResubmit: utils.blockResubmit,
-        checkManifestReady: utils.checkManifestReady
+        checkManifestReady: utils.checkManifestReady,
+        updateDownloadBtns: utils.updateDownloadBtns
     };
 });

static/js/cartutils.js

@@ -122,10 +122,11 @@ define(['filterutils','jquery', 'tippy', 'base' ], function(filterutils, $,  tip
         return deferred.promise();
     };
 
-
-    const updateCartCounts =function(){
+    const updateCartCounts = async function(){
         var buttonContents = '<button class="btn filter-type clear-cart" role="button" title="Empty your cart."><i class="fa fa-rotate-left"></i></button>';
-        if (Object.keys(window.proj_in_cart).length>0){
+        let cart_has_contents = Boolean(Object.keys(window.proj_in_cart).length>0);
+        let cart_controls = $('.cart-activated-controls');
+        if (cart_has_contents){
             var nmprojs = 0;
             var nmcases=0;
             var nmstudies=0;
@@ -136,19 +137,46 @@ define(['filterutils','jquery', 'tippy', 'base' ], function(filterutils, $,  tip
                 nmstudies=nmstudies+window.proj_in_cart[projid]['studies'];
                 nmseries=nmseries+window.proj_in_cart[projid]['series'];
             }
-
             let content = buttonContents+'<span id ="#cart_stats">Cart contents: ' + nmseries.toString()+' series from '+nmprojs.toString()+
-                ' collections / '+nmcases.toString()+' cases / '+nmstudies.toString()+' studies</span>';
+                ' collections / '+nmcases.toString()+' cases / '+nmstudies.toString()+' studies</span> <span class="cart_disk_size">(Calculating size...)</span>';
             localStorage.setItem('manifestSeriesCount',nmseries);
 
             $('#cart_stats_holder').html(content) ;
             $('#cart_stats').removeClass('empty-cart');
-            $('.cart-activated-controls').removeAttr('disabled');
+            cart_controls.each(function(){
+                $(this).removeAttr('disabled');
+                !$(this).hasClass('tip-titled') && $(this).attr("title",$(this).attr("data-default-title"));
+            });
         } else {
             $('#cart_stats_holder').html('<span id="#cart_stats">Your cart is currently empty</span>');
             $('#cart_stats').addClass('empty-cart');
-            $('.cart-activated-controls').attr('disabled', 'disabled');
+            cart_controls.each(function(){
+                $(this).attr('disabled', 'disabled');
+                !$(this).hasClass('tip-titled') && $(this).attr("title","Add items to the cart to enable this feature.");
+            });
+        }
+        let cart_disk_size = 0;
+        let cart_disk_display_size = "(Calculating...)";
+        let cart_disk_resp = await fetch(`${BASE_URL}/cart_data/`, {
+                method: "POST",
+                body: new URLSearchParams({
+                    'filtergrp_list': JSON.stringify(window.filtergrp_list ? window.filtergrp_list : [{}]),
+                    'partitions': JSON.stringify(window.partitions),
+                    'size_only': 'true'
+                }),
+                headers: {"X-CSRFToken": $.getCookie('csrftoken'), "content-type": 'application/x-www-form-urlencoded'}
+        });
+        if(!cart_disk_resp.ok) {
+            console.error("Unable to fetch cart size!");
+            cart_disk_size = 4;
+            cart_disk_display_size = "(Unable to retrieve size.)";
+        } else {
+            let cart_disk_res = await cart_disk_resp.json();
+            cart_disk_size = cart_disk_res['total_size']/Math.pow(1000,4);
+            cart_disk_display_size = cart_disk_res['display_size'];
         }
+        $('.cart_disk_size').html(cart_disk_display_size);
+        base.updateDownloadBtns('cart', cart_has_contents, cart_disk_size);
 
     }
 
@@ -298,7 +326,6 @@ define(['filterutils','jquery', 'tippy', 'base' ], function(filterutils, $,  tip
 
      window.updatePartitionsFromScratch = function(){
         window.partitions = new Array();
-
         for (var i=0;i<window.cartHist.length;i++){
            var cartHist=window.cartHist[i];
            updateGlobalPartitions(cartHist.partitions);
@@ -308,10 +335,6 @@ define(['filterutils','jquery', 'tippy', 'base' ], function(filterutils, $,  tip
            refilterGlobalPartitions(cartHist,i);
         }
         fixpartitions();
-        var filtStrings = createFiltStrings();
-        var solrStr = createSolrString(filtStrings);
-        window.solrStr = solrStr;
-        var ii=1;
     };
 
     //looking across the history of cart selections, create one set of exclusive partitions of the imaging data
@@ -490,29 +513,6 @@ define(['filterutils','jquery', 'tippy', 'base' ], function(filterutils, $,  tip
         return(filtStrings);
     }
 
-    // not really needed, but used to creating the solr string in on the client side
-    const createSolrString = function(filtStringA){
-        var solrStr=''
-        var solrA=[]
-        for (var i=0;i< window.partitions.length;i++){
-            var curPart = window.partitions[i];
-            if (!curPart['null']) {
-                var curPartAttStrA = parsePartitionAttStrings(filtStringA, curPart);
-                var curPartStr = parsePartitionStrings(curPart);
-                for (var j = 0; j < curPartAttStrA.length; j++) {
-                    if (curPartAttStrA[j].length > 0) {
-                        solrA.push('(' + curPartStr + ')(' + curPartAttStrA[j] + ')')
-                    } else{
-                        solrA.push(curPartStr);
-                    }
-                }
-            }
-        }
-        solrA = solrA.map(x => '('+x+')');
-        var solrStr = solrA.join(' OR ')
-        return solrStr
-    }
-
     const parsePartitionAttStrings = function(filtStringA, partition){
         var attStrA =[];
         var filt2D = partition['filt'];