Allow for harvesting of multiple formats

[srappel]

GeoDiscovery-Utils/opendataharvest/DCAT_Harvester.py

Lines 391 to 414 in 0352de7

	@staticmethod
	def process_distribution(distribution):
	url = AardvarkDataProcessor.getURL(distribution)
	if "format" not in distribution or url == "invalid":
	return None
	format_to_reference = {
	"ArcGIS GeoServices REST API": {
	"FeatureServer": "urn:x-esri:serviceType:ArcGIS#FeatureLayer",
	"ImageServer": "urn:x-esri:serviceType:ArcGIS#ImageMapLayer",
	"MapServer": "urn:x-esri:serviceType:ArcGIS#DynamicMapLayer",
	},
	"ZIP": "http://schema.org/downloadUrl",
	}
	format_references = format_to_reference.get(distribution["format"], {})
	if isinstance(format_references, dict):
	for key, value in format_references.items():
	if key in url:
	return {value: url}
	else:
	return {format_references: url}
	return None

This will have implications for how we implement dct_format_s, but it should be okay since the field is conditionally required, required only if we aren't using the multiple downloads option.

Intended Behavior would be to harvest all the "Distributions" listed for this file from Wisconsin Rapids:

{
			"@type": "dcat:Dataset",
			"identifier": "https://www.arcgis.com/home/item.html?id=04d526725a1e4f13bc5fb734028adc37&sublayer=20",
			"landingPage": "https://rapidsdata-wisconsinrapids.opendata.arcgis.com/maps/04d526725a1e4f13bc5fb734028adc37_20",
			"title": "Trails and Paths",
			"description": "Public services available to the residents of Wisconsin Rapids",
			"keyword": [
				"Wisconsin Rapids",
				"hunting",
				"zoning",
				"trails",
				"parks",
				"alderman districts",
				"developable property",
				"refuse",
				"garbage",
				"recycling",
				"bowhunting",
				"construction",
				"projects",
				"Basemaps"
			],
			"issued": "2015-07-16T20:37:06.000Z",
			"modified": "2020-02-25T17:50:31.000Z",
			"publisher": {
				"name": "WiscRapids"
			},
			"contactPoint": {
				"@type": "vcard:Contact",
				"fn": "WiscRapids"
			},
			"accessLevel": "public",
			"spatial": "-89.9218,44.3120,-89.7083,44.4595",
			"license": "",
			"distribution": [
				{
					"@type": "dcat:Distribution",
					"title": "ArcGIS Hub Dataset",
					"format": "Web Page",
					"mediaType": "text/html",
					"accessURL": "https://rapidsdata-wisconsinrapids.opendata.arcgis.com/maps/04d526725a1e4f13bc5fb734028adc37_20"
				},
				{
					"@type": "dcat:Distribution",
					"title": "ArcGIS GeoService",
					"format": "ArcGIS GeoServices REST API",
					"mediaType": "application/json",
					"accessURL": "https://gis.wirapids.org/arcgis/rest/services/Public_Basemap_svc/MapServer/20"
				},
				{
					"@type": "dcat:Distribution",
					"title": "CSV",
					"format": "CSV",
					"mediaType": "text/csv",
					"accessURL": "https://rapidsdata-wisconsinrapids.opendata.arcgis.com/api/download/v1/items/04d526725a1e4f13bc5fb734028adc37/csv?layers=20"
				},
				{
					"@type": "dcat:Distribution",
					"title": "GeoJSON",
					"format": "GeoJSON",
					"mediaType": "application/vnd.geo+json",
					"accessURL": "https://rapidsdata-wisconsinrapids.opendata.arcgis.com/api/download/v1/items/04d526725a1e4f13bc5fb734028adc37/geojson?layers=20"
				},
				{
					"@type": "dcat:Distribution",
					"title": "Shapefile",
					"format": "ZIP",
					"mediaType": "application/zip",
					"accessURL": "https://rapidsdata-wisconsinrapids.opendata.arcgis.com/api/download/v1/items/04d526725a1e4f13bc5fb734028adc37/shapefile?layers=20"
				},
				{
					"@type": "dcat:Distribution",
					"title": "KML",
					"format": "KML",
					"mediaType": "application/vnd.google-earth.kml+xml",
					"accessURL": "https://rapidsdata-wisconsinrapids.opendata.arcgis.com/api/download/v1/items/04d526725a1e4f13bc5fb734028adc37/kml?layers=20"
				}
			],
			"theme": [
				"geospatial"
			]
		},

This type of generated download does not work gracefully right now in geoblacklight, as some "in progress" type responses are retrieved from the URL as the download is generated by the source server (e.g. Wisconsin Rapids's ArcGIS Portal).