Add support for filtering ACLs using subqueries

Rename EE2CAclQueryUtils to AclDenormalizedTableQueryUtils since it's
also being used for other denormalized tables.

Make the mask column and requested authority configurable when querying
denormalized tables.

Author: arteymix

gemma-core/src/main/java/ubic/gemma/core/loader/util/mapper/EntityMapper.java

@@ -10,6 +10,16 @@
 
 /**
  * Map external identifiers to Gemma entities.
+ * <p>
+ * Mapping works by defining a set of candidates {@link Identifiable} and a set of external identifiers (e.g. gene
+ * symbols, accession numbers, etc.) to map against. For efficiency, the mapping process creates a "state" for the
+ * candidates that enables fast lookups that one can obtain with {@link #forCandidates(Collection)}.
+ * <p>
+ * It is then possible to tell if an identifier has a match among the candidates, to obtain a single match, to obtain
+ * all matches or to obtain mapping statistics. Mapping statistics are useful for debugging, but also comparing
+ * different mapping strategies. For example, one could want to compare whether mapping by official gene symbols or
+ * Ensembl identifiers yields better results.
+ *
  * @param <T> the type of entity being mapped
  * @author poirigui
  */

gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/BibliographicReferenceDaoImpl.java

@@ -75,7 +75,7 @@ public long countDistinctWithRelatedExperiments() {
                 .createQuery( "select count(" + ( AclQueryUtils.requiresCountDistinct() ? "distinct " : "" ) + " b)" + " "
                         + "from ExpressionExperiment e join e.primaryPublication b "
                         + AclQueryUtils.formAclRestrictionClause( "e.id" ) );
-        AclQueryUtils.addAclParameters( q, ExpressionExperiment.class );
+        AclQueryUtils.setAclParameters( q, ExpressionExperiment.class );
         return ( Long ) q.uniqueResult();
     }
 
@@ -87,7 +87,7 @@ public long countWithRelatedExperiments() {
                 .createQuery( "select count(" + ( AclQueryUtils.requiresCountDistinct() ? "distinct " : "" ) + " e)" + " "
                         + "from ExpressionExperiment e join e.primaryPublication b"
                         + AclQueryUtils.formAclRestrictionClause( "e.id" ) );
-        AclQueryUtils.addAclParameters( q, ExpressionExperiment.class );
+        AclQueryUtils.setAclParameters( q, ExpressionExperiment.class );
         return ( Long ) q.uniqueResult();
     }
 
@@ -99,7 +99,7 @@ public LinkedHashMap<BibliographicReference, Set<ExpressionExperimentIdAndShortN
                         + ( AclQueryUtils.requiresGroupBy() ? "group by b, e " : "" )
                         + "order by b.authorList nulls last, b.title nulls last"
                 );
-        AclQueryUtils.addAclParameters( q, ExpressionExperiment.class );
+        AclQueryUtils.setAclParameters( q, ExpressionExperiment.class );
         //noinspection unchecked
         List<Object[]> os = q
                 .setFirstResult( offset )
@@ -126,7 +126,7 @@ public LinkedHashMap<BibliographicReference, Collection<ExpressionExperiment>> g
                         + "and b in (:recs) "
                         + ( AclQueryUtils.requiresGroupBy() ? "group by b, e " : "" )
                         + "order by b.authorList nulls last, b.title nulls last" );
-        AclQueryUtils.addAclParameters( query, ExpressionExperiment.class );
+        AclQueryUtils.setAclParameters( query, ExpressionExperiment.class );
         List<Object[]> os = QueryUtils.listByIdentifiableBatch( query, "recs", records, eeBatchSize );
         LinkedHashMap<BibliographicReference, Collection<ExpressionExperiment>> result = new LinkedHashMap<>();
         for ( Object[] o : os ) {

gemma-core/src/main/java/ubic/gemma/persistence/service/common/description/CharacteristicDaoImpl.java

@@ -29,6 +29,7 @@
 import org.hibernate.engine.spi.SessionFactoryImplementor;
 import org.hibernate.type.StandardBasicTypes;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.acls.domain.BasePermission;
 import org.springframework.stereotype.Repository;
 import org.springframework.util.Assert;
 import ubic.gemma.core.ontology.OntologyUtils;
@@ -45,7 +46,7 @@
 import ubic.gemma.model.genome.Taxon;
 import ubic.gemma.model.genome.gene.GeneSet;
 import ubic.gemma.persistence.service.AbstractNoopFilteringVoEnabledDao;
-import ubic.gemma.persistence.util.EE2CAclQueryUtils;
+import ubic.gemma.persistence.util.AclDenormalizedTableQueryUtils;
 import ubic.gemma.persistence.util.IdentifiableUtils;
 import ubic.gemma.persistence.util.QueryUtils;
 
@@ -56,6 +57,7 @@
 
 import static org.hibernate.criterion.Restrictions.like;
 import static org.hibernate.criterion.Restrictions.not;
+import static ubic.gemma.persistence.service.maintenance.TableMaintenanceUtil.EE2C_IS_AUTHENTICATED_ANONYMOUSLY_MASK_COLUMN;
 import static ubic.gemma.persistence.service.maintenance.TableMaintenanceUtil.EE2C_QUERY_SPACE;
 import static ubic.gemma.persistence.util.IdentifiableUtils.toIdentifiableSet;
 import static ubic.gemma.persistence.util.QueryUtils.*;
@@ -242,11 +244,13 @@ public Map<Class<? extends Identifiable>, Map<String, Set<ExpressionExperiment>>
     private Map<Class<? extends Identifiable>, Map<String, Set<Long>>> findExperimentsByUrisInternal( Collection<String> uris, boolean includeSubjects, boolean includePredicates, boolean includeObjects, @Nullable Taxon taxon, boolean rankByLevel, int limit ) {
         String qs = "select T.`LEVEL`, T.VALUE_URI, T.PREDICATE_URI, T.OBJECT_URI, T.SECOND_PREDICATE_URI, T.SECOND_OBJECT_URI, T.EXPRESSION_EXPERIMENT_FK from EXPRESSION_EXPERIMENT2CHARACTERISTIC T"
                 + ( taxon != null ? " join INVESTIGATION I on T.EXPRESSION_EXPERIMENT_FK = I.ID " : "" )
-                + EE2CAclQueryUtils.formNativeAclJoinClause( "T.EXPRESSION_EXPERIMENT_FK" ) + " "
+                + AclDenormalizedTableQueryUtils.formNativeAclJoinClause( "T.EXPRESSION_EXPERIMENT_FK", "IS_AUTHENTICATED_ANONYMOUSLY" ) + " "
                 + "where "
                 + createPredicates( "T", "uris", includeSubjects, includePredicates, includeObjects )
                 + ( taxon != null ? " and I.TAXON_FK = :taxonId" : "" )
-                + EE2CAclQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory(), "T.ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK" )
+                + AclDenormalizedTableQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory(),
+                "IS_AUTHENTICATED_ANONYMOUSLY", "T." + EE2C_IS_AUTHENTICATED_ANONYMOUSLY_MASK_COLUMN,
+                BasePermission.READ )
                 + ( rankByLevel ? " order by FIELD(T.LEVEL, :eeClass, :edClass, :bmClass)" : "" );
 
         Query query = getSessionFactory().getCurrentSession().createSQLQuery( qs )
@@ -274,7 +278,7 @@ private Map<Class<? extends Identifiable>, Map<String, Set<Long>>> findExperimen
             query.setParameter( "taxonId", taxon.getId() );
         }
 
-        EE2CAclQueryUtils.addAclParameters( query, ExpressionExperiment.class );
+        AclDenormalizedTableQueryUtils.setAclParameters( query, "IS_AUTHENTICATED_ANONYMOUSLY", ExpressionExperiment.class );
 
         query.setCacheable( true );
 

gemma-core/src/main/java/ubic/gemma/persistence/service/expression/arrayDesign/ArrayDesignDaoImpl.java

@@ -25,6 +25,7 @@
 import org.hibernate.engine.spi.SessionFactoryImplementor;
 import org.hibernate.type.StandardBasicTypes;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.acls.domain.BasePermission;
 import org.springframework.stereotype.Repository;
 import org.springframework.util.Assert;
 import ubic.gemma.model.common.auditAndSecurity.AuditEvent;
@@ -43,6 +44,7 @@
 import ubic.gemma.model.genome.sequenceAnalysis.BlatAssociation;
 import ubic.gemma.model.genome.sequenceAnalysis.BlatResult;
 import ubic.gemma.persistence.service.common.auditAndSecurity.curation.AbstractCuratableDao;
+import ubic.gemma.persistence.service.maintenance.TableMaintenanceUtil;
 import ubic.gemma.persistence.util.*;
 import ubic.gemma.persistence.util.Filter;
 
@@ -492,7 +494,7 @@ public long countExpressionExperiments( ArrayDesign arrayDesign ) {
                         + AclQueryUtils.formAclRestrictionClause( "ee.id" ) + " "
                         + "and ad = :ad" )
                 .setParameter( "ad", arrayDesign );
-        AclQueryUtils.addAclParameters( query, ExpressionExperiment.class );
+        AclQueryUtils.setAclParameters( query, ExpressionExperiment.class );
         return ( Long ) query.uniqueResult();
     }
 
@@ -503,7 +505,7 @@ public Map<Taxon, Long> getPerTaxonCount() {
                         + "join ad.primaryTaxon t "
                         + AclQueryUtils.formAclRestrictionClause( "ad.id" ) + " "
                         + "group by t" );
-        AclQueryUtils.addAclParameters( query, ArrayDesign.class );
+        AclQueryUtils.setAclParameters( query, ArrayDesign.class );
         //noinspection unchecked
         List<Object[]> csList = query.setCacheable( true ).list();
         return csList.stream().collect( Collectors.toMap( row -> ( Taxon ) row[0], row -> ( Long ) row[1] ) );
@@ -536,7 +538,7 @@ public long countSwitchedExpressionExperiments( ArrayDesign arrayDesign ) {
                         + AclQueryUtils.formAclRestrictionClause( "e.id" ) + " "
                         + "and b.originalPlatform = :arrayDesign" )
                 .setParameter( "arrayDesign", arrayDesign );
-        AclQueryUtils.addAclParameters( query, ExpressionExperiment.class );
+        AclQueryUtils.setAclParameters( query, ExpressionExperiment.class );
         return ( Long ) query
                 .setCacheable( true )
                 .uniqueResult();
@@ -1059,7 +1061,7 @@ private Query finishFilteringQuery( String queryString, @Nullable Filters filter
 
         Query query = this.getSessionFactory().getCurrentSession().createQuery( queryString );
 
-        AclQueryUtils.addAclParameters( query, ArrayDesign.class );
+        AclQueryUtils.setAclParameters( query, ArrayDesign.class );
         FilterQueryUtils.addRestrictionParameters( query, filters );
 
         return query;
@@ -1136,10 +1138,12 @@ private void populateBlacklisted( Collection<ArrayDesignValueObject> vos ) {
     private void populateExpressionExperimentCount( Collection<ArrayDesignValueObject> entities ) {
         Query query = this.getSessionFactory().getCurrentSession()
                 .createSQLQuery( "select ee2ad.ARRAY_DESIGN_FK as ID, count(distinct ee2ad.EXPRESSION_EXPERIMENT_FK) as EE_COUNT from EXPRESSION_EXPERIMENT2ARRAY_DESIGN ee2ad "
-                        + EE2CAclQueryUtils.formNativeAclJoinClause( "ee2ad.EXPRESSION_EXPERIMENT_FK" ) + " "
+                        + AclDenormalizedTableQueryUtils.formNativeAclJoinClause( "ee2ad.EXPRESSION_EXPERIMENT_FK", "IS_AUTHENTICATED_ANONYMOUSLY" ) + " "
                         + "where ee2ad.ARRAY_DESIGN_FK in :ids "
                         + "and not ee2ad.IS_ORIGINAL_PLATFORM"
-                        + EE2CAclQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory(), "ee2ad.ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK" )
+                        + AclDenormalizedTableQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory(),
+                        "IS_AUTHENTICATED_ANONYMOUSLY", "ee2ad." + TableMaintenanceUtil.EE2AD_IS_AUTHENTICATED_ANONYMOUSLY_MASK_COLUMN,
+                        BasePermission.READ )
                         + formNativeNonTroubledClause( "ee2ad.EXPRESSION_EXPERIMENT_FK", ExpressionExperiment.class )
                         + " group by ee2ad.ARRAY_DESIGN_FK" )
                 .addScalar( "ID", StandardBasicTypes.LONG )
@@ -1150,7 +1154,7 @@ private void populateExpressionExperimentCount( Collection<ArrayDesignValueObjec
                 .addSynchronizedEntityClass( ExpressionExperiment.class )
                 .addSynchronizedEntityClass( ArrayDesign.class )
                 .setCacheable( true );
-        EE2CAclQueryUtils.addAclParameters( query, ExpressionExperiment.class );
+        AclDenormalizedTableQueryUtils.setAclParameters( query, "IS_AUTHENTICATED_ANONYMOUSLY", ExpressionExperiment.class );
         Map<Long, Long> countById = QueryUtils.<Long, Object[]>streamByBatch( query, "ids", IdentifiableUtils.getIds( entities ), 2048 )
                 .collect( Collectors.toMap( o -> ( Long ) o[0], o -> ( Long ) o[1] ) );
         for ( ArrayDesignValueObject vo : entities ) {
@@ -1162,12 +1166,14 @@ private void populateExpressionExperimentCount( Collection<ArrayDesignValueObjec
     private void populateSwitchedExpressionExperimentCount( Collection<ArrayDesignValueObject> entities ) {
         Query query = this.getSessionFactory().getCurrentSession()
                 .createSQLQuery( "select ee2ad.ARRAY_DESIGN_FK as ID, count(distinct ee2ad.EXPRESSION_EXPERIMENT_FK) as EE_COUNT from EXPRESSION_EXPERIMENT2ARRAY_DESIGN ee2ad "
-                        + EE2CAclQueryUtils.formNativeAclJoinClause( "ee2ad.EXPRESSION_EXPERIMENT_FK" ) + " "
+                        + AclDenormalizedTableQueryUtils.formNativeAclJoinClause( "ee2ad.EXPRESSION_EXPERIMENT_FK", "IS_AUTHENTICATED_ANONYMOUSLY" ) + " "
                         + "where ee2ad.ARRAY_DESIGN_FK in :ids "
                         + "and ee2ad.IS_ORIGINAL_PLATFORM "
                         // ignore noop switches
                         + "and ee2ad.ARRAY_DESIGN_FK not in (select ARRAY_DESIGN_FK from EXPRESSION_EXPERIMENT2ARRAY_DESIGN where EXPRESSION_EXPERIMENT_FK = ee2ad.EXPRESSION_EXPERIMENT_FK and ARRAY_DESIGN_FK = ee2ad.ARRAY_DESIGN_FK and not IS_ORIGINAL_PLATFORM)"
-                        + EE2CAclQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory(), "ee2ad.ACL_IS_AUTHENTICATED_ANONYMOUSLY_MASK" )
+                        + AclDenormalizedTableQueryUtils.formNativeAclRestrictionClause( ( SessionFactoryImplementor ) getSessionFactory(),
+                        "IS_AUTHENTICATED_ANONYMOUSLY", "ee2ad." + TableMaintenanceUtil.EE2AD_IS_AUTHENTICATED_ANONYMOUSLY_MASK_COLUMN,
+                        BasePermission.READ )
                         + formNativeNonTroubledClause( "ee2ad.EXPRESSION_EXPERIMENT_FK", ExpressionExperiment.class )
                         + " group by ee2ad.ARRAY_DESIGN_FK" )
                 .addScalar( "ID", StandardBasicTypes.LONG )
@@ -1178,7 +1184,7 @@ private void populateSwitchedExpressionExperimentCount( Collection<ArrayDesignVa
                 .addSynchronizedEntityClass( ExpressionExperiment.class )
                 .addSynchronizedEntityClass( ArrayDesign.class )
                 .setCacheable( true );
-        EE2CAclQueryUtils.addAclParameters( query, ExpressionExperiment.class );
+        AclDenormalizedTableQueryUtils.setAclParameters( query, "IS_AUTHENTICATED_ANONYMOUSLY", ExpressionExperiment.class );
         Map<Long, Long> switchedCountById = QueryUtils.<Long, Object[]>streamByBatch( query, "ids", IdentifiableUtils.getIds( entities ), 2048 )
                 .collect( Collectors.toMap( row -> ( Long ) row[0], row -> ( Long ) row[1] ) );
         for ( ArrayDesignValueObject vo : entities ) {

gemma-core/src/main/java/ubic/gemma/persistence/service/expression/designElement/CompositeSequenceDaoImpl.java

@@ -99,7 +99,7 @@ protected Query getFilteringQuery( @Nullable Filters filters, @Nullable Sort sor
                         + ( !SecurityUtil.isUserAdmin() ? " and ad.curationDetails.troubled = false" : "" )
                         + FilterQueryUtils.formRestrictionClause( filters )
                         + FilterQueryUtils.formOrderByClause( sort ) );
-        AclQueryUtils.addAclParameters( query, ArrayDesign.class );
+        AclQueryUtils.setAclParameters( query, ArrayDesign.class );
         FilterQueryUtils.addRestrictionParameters( query, filters );
         return query;
     }
@@ -119,7 +119,7 @@ protected Query getFilteringIdQuery( @Nullable Filters filters, @Nullable Sort s
                         + ( !SecurityUtil.isUserAdmin() ? " and ad.curationDetails.troubled = false" : "" )
                         + FilterQueryUtils.formRestrictionClause( filters )
                         + FilterQueryUtils.formOrderByClause( sort ) );
-        AclQueryUtils.addAclParameters( query, ArrayDesign.class );
+        AclQueryUtils.setAclParameters( query, ArrayDesign.class );
         FilterQueryUtils.addRestrictionParameters( query, filters );
         return query;
     }
@@ -132,7 +132,7 @@ protected Query getFilteringCountQuery( @Nullable Filters filters ) {
                 + AclQueryUtils.formAclRestrictionClause( "ad.id" )
                 + ( !SecurityUtil.isUserAdmin() ? " and ad.curationDetails.troubled = false" : "" )
                 + FilterQueryUtils.formRestrictionClause( filters ) );
-        AclQueryUtils.addAclParameters( query, ArrayDesign.class );
+        AclQueryUtils.setAclParameters( query, ArrayDesign.class );
         FilterQueryUtils.addRestrictionParameters( query, filters );
         return query;
     }