Merge pull request #1500 from ImagingDataCommons/release-49-sp

cart and filter downloads

Author: s-paquette

idc/views.py

@@ -39,7 +39,9 @@
 from cohorts.models import Cohort, Cohort_Perms
 
 from idc_collections.models import Program, DataSource, Collection, ImagingDataCommonsVersion, Attribute, Attribute_Tooltips, DataSetType, Citation
-from idc_collections.collex_metadata_utils import build_explorer_context, get_collex_metadata, create_file_manifest, get_cart_data_serieslvl, get_cart_data_studylvl, get_table_data_with_cart_data
+from idc_collections.collex_metadata_utils import (build_explorer_context, get_collex_metadata, create_file_manifest,
+                                                   get_cart_data_serieslvl, get_cart_data_studylvl,
+                                                   get_table_data_with_cart_data, cart_manifest)
 from allauth.socialaccount.models import SocialAccount
 from django.core.exceptions import ObjectDoesNotExist
 from django.http import HttpResponse, JsonResponse
@@ -48,7 +50,7 @@
 from idc.models import User_Data
 from solr_helpers import build_solr_query, query_solr_and_format_result
 
-
+from idc.settings import MAX_FILE_LIST_REQUEST
 
 debug = settings.DEBUG
 logger = logging.getLogger(__name__)
@@ -678,35 +680,48 @@ def cart_data(request):
     return JsonResponse(response, status=status)
 
 
-def get_series(request, collection_id, patient_id=None, study_uid=None):
+def get_series(request, collection_id=None, patient_id=None, study_uid=None):
+    status = 200
+    response = {"result": []}
     try:
-        status = 200
-        response = { "result": [] }
-        source = ImagingDataCommonsVersion.objects.get(active=True).get_data_sources(
-            active=True, source_type=DataSource.SOLR,
-            aggregate_level="SeriesInstanceUID"
-        ).first()
-        filters = {
-            "collection_id": [collection_id]
-        }
-        if patient_id:
-            filters['PatientID'] = [patient_id]
-        if study_uid:
-            filters["StudyInstanceUID"] = [study_uid]
-        filter_query = build_solr_query(
-            filters,
-            with_tags_for_ex=False,
-            search_child_records_by=None, solr_default_op='AND'
-        )
-        result = query_solr_and_format_result(
-            {
-                "collection": source.name,
-                "fields": ["collection_id", "PatientID", "StudyInstanceUID", "Modality", "crdc_series_uuid", "SeriesInstanceUID", "aws_bucket", "instance_size"],
-                "query_string": None,
-                "fqs": [filter_query['full_query_str']],
-                "facets": None, "sort": None, "counts_only": False, "limit": 2000
+        fields = ["collection_id", "PatientID", "StudyInstanceUID", "Modality", "crdc_series_uuid", "SeriesInstanceUID", "aws_bucket", "instance_size"]
+        result = {}
+        if not collection_id:
+            # This is a request for filters and/or a cart
+            body_unicode = request.body.decode('utf-8')
+            body = json.loads(body_unicode)
+            partitions = body.get("partitions", {})
+            filtergrp_list = body.get("filtergrp_list", {})
+            if not(len(partitions) or len(filtergrp_list)):
+                raise Exception("You can only request series IDs based on a filter, collection ID, Patient ID, or study ID!")
+            result = cart_manifest(filtergrp_list, partitions, 0, fields, MAX_FILE_LIST_REQUEST)
+        else:
+            source = ImagingDataCommonsVersion.objects.get(active=True).get_data_sources(
+                active=True, source_type=DataSource.SOLR,
+                aggregate_level="SeriesInstanceUID"
+            ).first()
+            filters = {
+                "collection_id": [collection_id]
             }
-        )
+            if patient_id:
+                filters['PatientID'] = [patient_id]
+            if study_uid:
+                filters["StudyInstanceUID"] = [study_uid]
+            filter_query = build_solr_query(
+                filters,
+                with_tags_for_ex=False,
+                search_child_records_by=None, solr_default_op='AND'
+            )
+            result = query_solr_and_format_result(
+                {
+                    "collection": source.name,
+                    "fields": fields,
+                    "query_string": None,
+                    "fqs": [filter_query['full_query_str']],
+                    "facets": None, "sort": None, "counts_only": False, "limit": 2000
+                }
+            )
+
         for doc in result['docs']:
             response['result'].append({
                 "series_id": doc['SeriesInstanceUID'],
@@ -716,7 +731,7 @@ def get_series(request, collection_id, patient_id=None, study_uid=None):
                 "modality": doc['Modality'][0],
                 "study_id": doc['StudyInstanceUID'],
                 "patient_id": doc["PatientID"],
-                "collection_id": doc['collection_id']
+                "collection_id": doc['collection_id'][0]
             })
 
     except Exception as e:

static/js/cohorts/export-manifest.js

@@ -75,29 +75,28 @@ require([
         if (button.hasClass('series-export') || button.hasClass('study-export')){
             update_export_modal_for_mini(button);
         } else if (button.hasClass('cart-export')){
-            updatePartitionsFromScratch();
              window.updatePartitionsFromScratch();
-             var ret =cartutils.formcartdata();
+             let ret = cartutils.formcartdata();
              window.partitions = ret[0];
              window.filtergrp_lst = ret[1];
 
-             var projS = new Set();
+             let projS = new Set();
              for (var i=0;i<partitions.length;i++){
                 projS.add(partitions[i].id[0]);
              }
-            var mxstudies=0;
-            var mxseries=0;
-            var projl = [...projS]
-            for (var i=0;i<projl.length;i++){
-               var proj = projl[i]
+            let mxstudies=0;
+            let mxseries=0;
+            let projl = [...projS]
+            for (let i=0;i<projl.length;i++){
+               let proj = projl[i]
                mxseries+= window.selProjects[proj].mxseries;
                mxstudies+= window.selProjects[proj].mxstudies;
             }
-            var filterSets = new Array();
+            let filterSets = [];
             for(let i=0; i< window.cartHist.length;i++) {
                filterSets.push(window.cartHist[i]['filter'])
             }
-            update_export_modal_for_cart(partitions, filterSets);
+            update_export_modal_for_cart(window.partitions, filterSets);
         } else if (button.hasClass('cart-export-from-cp')){
             update_export_modal_for_cart(window.partitions, window.filtergrp_list);
         } else if(button.hasClass('export-cohort-manifest')) {

static/js/downloader.js

@@ -21,13 +21,19 @@ require.config({
     baseUrl: STATIC_FILES_URL + 'js/',
     paths: {
         jquery: 'libs/jquery-3.7.1.min',
-        base: 'base'
+        base: 'base',
+        cartutils: 'cartutils',
+        filterutils: 'filterutils'
+    },
+    shim: {
+        'cartutils': ['jquery'],
+        'filterutils': ['jquery']
     }
 });
 
 require([
-    'base', 'jquery'
-], function (base, $) {
+    'base', 'jquery', 'cartutils', 'filterutils',
+], function (base, $, cartutils, filterutils) {
 
     const byte_level = Object.freeze({
         1: "KB",
@@ -423,7 +429,6 @@ require([
                     response = await fetch(s3_url, {
                         signal: abort_controller.signal
                     });
-                    console.error("Checking response.");
                     if (!response.ok) {
                         console.error("[Worker] Saw !ok response of ",response.status);
                         if(pending_abort) {
@@ -749,8 +754,34 @@ require([
 
             } else {
                 let response = null;
-                if(download_type in ["filter", "cart"]) {
-                    response = await fetch(`${BASE_URL}${SERIES_IDS_FILTER_URL}`);
+                if(["cohort", "cart"].includes(download_type)) {
+                    let filter_and_cart = {};
+                    if (download_type === "cohort") {
+                        let filtergrp_list = [];
+                        for(const [attr, vals] of Object.entries(filterutils.parseFilterObj())) {
+                            filtergrp_list.push({[attr]: vals});
+                        }
+                        filter_and_cart["filtergrp_list"] = filtergrp_list;
+                    } else {
+                         window.updatePartitionsFromScratch();
+                         let ret = cartutils.formcartdata();
+                         window.partitions = ret[0];
+                         window.filtergrp_lst = ret[1];
+                        let filterSets = [];
+                        for(let i=0; i< window.cartHist.length;i++) {
+                           filterSets.push(window.cartHist[i]['filter']);
+                        }
+                        filter_and_cart['partitions'] = window.partitions;
+                        filter_and_cart['filtergrp_list'] = filterSets;
+                    }
+                    response = await fetch(`${BASE_URL}${SERIES_IDS_FILTER_URL}`, {
+                            method: "POST",
+                            body: JSON.stringify(filter_and_cart),
+                            headers: {
+                                "X-CSRFToken": csrftoken,
+                                "content-type": "application/json"
+                            }
+                        });
                 } else {
                     let study_uri = (study_id !== undefined && study_id !== null) ? `${study_id}/` : "";
                     let patient_uri = (patient_id !== undefined && patient_id !== null) ? `${patient_id}/` : "";

static/js/explore.js

@@ -483,8 +483,8 @@ require([
 
     const download_tooltip = {
         dynamicTip: function(ref){
-            let table_type = $(ref).parents('table').attr('data-table-type');
-            return `Download all of the image instances in this ${table_type}.`;
+            let download_type = $(ref).attr('data-download-type');
+            return `Download all of the image instances in this ${download_type}.`;
         },
         content: 'Download all images.', // placeholder text
         theme: 'dark',

static/js/image_search.js

@@ -141,7 +141,7 @@ require([
             success: function (data) {
                 try {
                     let curInd = window.cartHist.length-1;
-                    let tmp=filterutils.parseFilterObj()
+                    let tmp=filterutils.parseFilterObj();
                     let filtObj=JSON.parse(JSON.stringify(tmp));
                     if (cartHist[curInd].selections.length>0){
                         let cartSel = new Object();

templates/idc/explore_data_core.html

@@ -630,7 +630,13 @@ <h4>Cohort Filters</h4>
                                 data-default-title="Export this cohort's series as a manifest for download."
                                 data-target="#export-manifest-modal"
                                 title="Select a filter to enable this feature."
-                            > Manifest <i class="fa fa-solid fa-download"></i>
+                            > Manifest <i class="fa fa-solid fa-list"></i>
+                            </button>
+                            <button id="download-cohort-images" class="btn btn-primary btn-small filter-activated-controls download-all-instances" disabled
+                                data-default-title="Download the instance image files for this cohort."
+                                title="Select a filter to enable this feature."
+                                data-download-type="cohort"
+                            > Download <i class="fa fa-solid fa-download"></i>
                             </button>
                             <button class="btn btn-small btn-special citations-button filter-activated-controls" disabled data-toggle="modal" data-target="#citations-modal" role="button"
                                 data-default-title="Display this cohort's citation list in Vancouver format."
@@ -675,7 +681,13 @@ <h4>Cart</h4>
                             <button id="export-manifest-cart" class="btn btn-primary cart-export btn-small cart-activated-controls" disabled data-toggle="modal"
                                 data-target="#export-manifest-modal"
                                 title="Add items to the cart to enable this feature."
-                            > Manifest <i class="fa fa-solid fa-download"></i>
+                            > Manifest <i class="fa fa-solid fa-list"></i>
+                            </button>
+                            <button id="download-cart-images" class="btn btn-primary btn-small cart-activated-controls download-all-instances" disabled
+                                data-default-title="Download the instance image files for this cart."
+                                title="Select a filter to enable this feature."
+                                data-download-type="cart"
+                            > Download <i class="fa fa-solid fa-download"></i>
                             </button>
                             <button class="btn btn-small btn-special citations-button cart-activated-controls for-cart" disabled data-toggle="modal" data-target="#citations-modal" role="button"
                                 data-default-title="Display this cart's citation list in Vancouver format."