Prod

.env

@@ -0,0 +1,2 @@
+CONF_REPO=https://github.com/VirtualFlyBrain/vfb-pipeline-config.git
+CONF_BRANCH=dev

config/collectdata/config.env

@@ -0,0 +1,3 @@
+KBserver=http://192.168.0.1:7474
+KBuser=neo4j
+KBpassword=password

config/collectdata/shacl/kb.shacl

@@ -0,0 +1,15 @@
+@prefix dash: <http://datashapes.org/dash#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix vfb: <http://virtualflybrain.org/> .
+@prefix sh: <http://www.w3.org/ns/shacl#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix FBbt:  <http://purl.obolibrary.org/obo/FBbt_> .
+
+vfb:DataSetCountShape
+			a sh:NodeShape ;
+			sh:targetNode FBbt:00050095 ;
+			sh:property [
+					sh:path [ sh:inversePath rdf:type ] ;
+					sh:minCount 1 ;
+			] .

config/collectdata/shex/kb.shex

@@ -0,0 +1,9 @@
+PREFIX ex: <http://ex.example/#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX vfb: <http://virtualflybrain.org/>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+vfb:ImageShape {
+  dct:source IRI
+}

config/collectdata/sparql/delete_blocked_entities.ru

@@ -0,0 +1,18 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+DELETE {
+  ?s <http://n2o.neo/custom/block> ?blocked .
+  ?s ?p ?o .
+}
+WHERE {
+  ?s <http://n2o.neo/custom/block> ?blocked .
+  ?s ?p ?o .
+  FILTER(?blocked=true) .
+  FILTER(isIRI(?s)) 
+}
+
+### EDIT: this was obsoleted in the end in favour of a cypher solution, see process.sh.

config/collectdata/sparql/delete_blocked_relations.ru

@@ -0,0 +1,30 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+
+
+DELETE {
+  ?s ?p ?o .
+  ?r rdf:type owl:Axiom ;
+        owl:annotatedSource ?s ;
+        owl:annotatedProperty ?p ;
+        owl:annotatedTarget ?o ;
+        <http://n2o.neo/custom/block> ?blocked;
+        ?bp ?bo;
+
+} WHERE {
+  ?s ?p ?o .
+  ?r rdf:type owl:Axiom ;
+      owl:annotatedSource ?s ;
+      owl:annotatedProperty ?p ;
+      owl:annotatedTarget ?o ;
+      <http://n2o.neo/custom/block> ?blocked;
+      ?bp ?bo;
+
+  FILTER(?blocked=true) .
+}
+
+### EDIT: this was obsoleted in the end in favour of a cypher solution, see process.sh.

config/collectdata/sparql/delete_embargoed_channels.ru

@@ -0,0 +1,31 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+#Delete all ds:DataSet where ds.production is False
+#Delete all i:Individual where (ds)-[:has_source]-(i:Individual)<-[:depicts]-(ch:Individual) WHERE ds.production is False
+
+DELETE {
+  ?channel ?channelrel ?channelval .
+}
+
+WHERE {
+
+	?dataset n2o:nodeLabel ?nodelabel . # This selects all datasets
+
+	OPTIONAL {
+		?dataset n2oc:production ?production .
+		# n2oc:production is a bit brittle because IRI might be changed (risk!)
+	}
+
+	?image dct:source ?dataset .
+	?channel <http://xmlns.com/foaf/0.1/depicts> ?image . # There does not always seem to be a channel
+	?channel ?channelrel ?channelval .
+
+	FILTER(?production=false || !bound(?production)) .
+	FILTER(?nodelabel="DataSet") 
+}
+
+### EDIT: this was obsoleted in the end in favour of a ROBOT solution, see process.sh. Using SPARQL this way is too memory consuming.

config/collectdata/sparql/delete_embargoed_datasets.ru

@@ -0,0 +1,28 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+#Delete all ds:DataSet where ds.production is False
+#Delete all i:Individual where (ds)-[:has_source]-(i:Individual)<-[:depicts]-(ch:Individual) WHERE ds.production is False
+
+DELETE {
+  ?dataset ?dsrel ?dsval .
+}
+
+WHERE {
+
+	?dataset n2o:nodeLabel ?nodelabel . # This selects all datasets	
+	OPTIONAL {
+		?dataset n2oc:production ?production .
+		# n2oc:production is a bit brittle because IRI might be changed (risk!)
+	}
+
+	?dataset ?dsrel ?dsval .
+
+	FILTER(?production=false || !bound(?production)) .
+	FILTER(?nodelabel="DataSet") 
+}
+
+### EDIT: this was obsoleted in the end in favour of a ROBOT solution, see process.sh. Using SPARQL this way is too memory consuming.

config/collectdata/sparql/delete_embargoed_images.ru

@@ -0,0 +1,35 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+#Delete all ds:DataSet where ds.production is False
+#Delete all i:Individual where (ds)-[:has_source]-(i:Individual)<-[:depicts]-(ch:Individual) WHERE ds.production is False
+
+DELETE {
+	?image ?imgrel ?imgval .
+	?imgval  ?p1  ?o1 .
+}
+
+WHERE {
+
+	?dataset n2o:nodeLabel ?nodelabel . # This selects all datasets
+
+	OPTIONAL {
+		?dataset n2oc:production ?production .
+		# n2oc:production is a bit brittle because IRI might be changed (risk!)
+	}
+
+	?image dct:source ?dataset . #in case a dataset does not have images yet this is an optional clause
+	?image ?imgrel ?imgval .
+	OPTIONAL {
+        ?imgval  ?p1  ?o1  .
+        FILTER (isBlank(?imgval)) 
+  }
+
+	FILTER(?production=false || !bound(?production)) .
+	FILTER(?nodelabel="DataSet") 
+}
+
+### EDIT: this was obsoleted in the end in favour of a ROBOT solution, see process.sh. Using SPARQL this way is too memory consuming.

config/collectdata/sparql/embargoed_datasets_dev.sparql

@@ -0,0 +1,25 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+SELECT DISTINCT ?dataset
+
+WHERE {
+
+	?dataset n2o:nodeLabel ?nodelabel . # This selects all datasets	
+	OPTIONAL {
+		?dataset n2oc:production ?production .
+		# n2oc:production is a bit brittle because IRI might be changed (risk!)
+	}
+
+	OPTIONAL {
+		?dataset n2oc:staging ?staged .
+	}
+
+	IF((staging=false || unbound(staging)) && (prod = false || unbound(prod)) ) -----> EMBARGO
+
+	FILTER( (?production=false || !bound(?production)) && (?staged=false || !bound(?staged)) ) .
+	FILTER(?nodelabel="DataSet") 
+}

config/collectdata/sparql/embargoed_datasets_prod.sparql

@@ -0,0 +1,19 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+SELECT DISTINCT ?dataset
+
+WHERE {
+
+	?dataset n2o:nodeLabel ?nodelabel . # This selects all datasets	
+	OPTIONAL {
+		?dataset n2oc:production ?production .
+		# n2oc:production is a bit brittle because IRI might be changed (risk!)
+	}
+
+	FILTER(?production=false || !bound(?production)) .
+	FILTER(?nodelabel="DataSet") 
+}

config/collectdata/sparql/select_blocked_entities.sparql

@@ -0,0 +1,15 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+SELECT ?s ?p ?o .
+WHERE {
+  ?s <http://n2o.neo/custom/block> ?blocked .
+  ?s ?p ?o .
+  FILTER(?blocked=true) .
+  FILTER(isIRI(?s)) 
+}
+
+### EDIT: this was obsoleted in the end in favour of a cypher solution, see process.sh.

config/collectdata/sparql/select_blocked_relations.sparql

@@ -0,0 +1,22 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+
+
+SELECT ?s ?p ?o
+WHERE {
+  ?s ?p ?o .
+  ?r rdf:type owl:Axiom ;
+      owl:annotatedSource ?s ;
+      owl:annotatedProperty ?p ;
+      owl:annotatedTarget ?o ;
+      <http://n2o.neo/custom/block> ?blocked;
+      ?bp ?bo;
+
+  FILTER(?blocked=true) .
+}
+
+### EDIT: this was obsoleted in the end in favour of a cypher solution, see process.sh.

config/collectdata/sparql/select_embargoed_channels_dev.sparql

@@ -0,0 +1,29 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+#Delete all ds:DataSet where ds.production is False
+#Delete all i:Individual where (ds)-[:has_source]-(i:Individual)<-[:depicts]-(ch:Individual) WHERE ds.production is False
+
+SELECT DISTINCT ?channel 
+WHERE {
+
+	?dataset n2o:nodeLabel ?nodelabel . # This selects all datasets
+
+	OPTIONAL {
+		?dataset n2oc:production ?production .
+		# n2oc:production is a bit brittle because IRI might be changed (risk!)
+	}
+
+	OPTIONAL {
+		?dataset n2oc:staging ?staged .
+	}
+
+	?image dct:source ?dataset .
+	?channel <http://xmlns.com/foaf/0.1/depicts> ?image . # There does not always seem to be a channel
+
+	FILTER( (?production=false || !bound(?production)) && (?staged=false || !bound(?staged)) ) .
+	FILTER(?nodelabel="DataSet") 
+}

config/collectdata/sparql/select_embargoed_channels_prod.sparql

@@ -0,0 +1,25 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+#Delete all ds:DataSet where ds.production is False
+#Delete all i:Individual where (ds)-[:has_source]-(i:Individual)<-[:depicts]-(ch:Individual) WHERE ds.production is False
+
+SELECT DISTINCT ?channel 
+WHERE {
+
+	?dataset n2o:nodeLabel ?nodelabel . # This selects all datasets
+
+	OPTIONAL {
+		?dataset n2oc:production ?production .
+		# n2oc:production is a bit brittle because IRI might be changed (risk!)
+	}
+
+	?image dct:source ?dataset .
+	?channel <http://xmlns.com/foaf/0.1/depicts> ?image . # There does not always seem to be a channel
+
+	FILTER(?production=false || !bound(?production)) .
+	FILTER(?nodelabel="DataSet") 
+}

config/collectdata/sparql/select_embargoed_datasets_dev.sparql

@@ -0,0 +1,26 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+#Delete all ds:DataSet where ds.production is False
+#Delete all i:Individual where (ds)-[:has_source]-(i:Individual)<-[:depicts]-(ch:Individual) WHERE ds.production is False
+
+SELECT DISTINCT ?dataset 
+WHERE {
+
+	?dataset n2o:nodeLabel ?nodelabel . # This selects all datasets	
+
+	OPTIONAL {
+		?dataset n2oc:production ?production .
+		# n2oc:production is a bit brittle because IRI might be changed (risk!)
+	}
+
+	OPTIONAL {
+		?dataset n2oc:staging ?staged .
+	}
+
+	FILTER( (?production=false || !bound(?production)) && (?staged=false || !bound(?staged)) ) .
+	FILTER(?nodelabel="DataSet") 
+}

config/collectdata/sparql/select_embargoed_datasets_prod.sparql

@@ -0,0 +1,22 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX n2o: <http://n2o.neo/property/>
+PREFIX n2oc: <http://n2o.neo/custom/>
+PREFIX dct: <http://purl.org/dc/terms/>
+
+#Delete all ds:DataSet where ds.production is False
+#Delete all i:Individual where (ds)-[:has_source]-(i:Individual)<-[:depicts]-(ch:Individual) WHERE ds.production is False
+
+SELECT DISTINCT ?dataset 
+WHERE {
+
+	?dataset n2o:nodeLabel ?nodelabel . # This selects all datasets	
+
+	OPTIONAL {
+		?dataset n2oc:production ?production .
+		# n2oc:production is a bit brittle because IRI might be changed (risk!)
+	}
+
+	FILTER(?production=false || !bound(?production)) .
+	FILTER(?nodelabel="DataSet") 
+}