HDXDSYS-2865 Use paging for all_fields True #109
Merged
GitHub Actions / Test Results
failed
Apr 7, 2026 in 0s
1 fail, 156 pass in 1m 25s
157 tests 156 ✅ 1m 25s ⏱️
1 suites 0 💤
1 files 1 ❌
Results for commit d2423b1.
Annotations
Check warning on line 0 in tests.hdx.api.test_ckan.TestCKAN
github-actions / Test Results
test_create_dataset (tests.hdx.api.test_ckan.TestCKAN) failed
test-results.xml [took 1m 10s]
Raw output
AssertionError: assert 'HDX Python API test' == 'HDX Python A...changed again'
#x1B[0m#x1B[91m- HDX Python API test changed again#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
#x1B[92m+ HDX Python API test#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
self = <tests.hdx.api.test_ckan.TestCKAN object at 0x7f2d0fd85590>
datasetmetadata = PosixPath('tests/fixtures/CKAN/hdx_dataset_static.yaml')
test_data = PosixPath('tests/fixtures/test_data.csv')
setup_teardown_folder = (<gspread.client.Client object at 0x7f2d0f735e80>, '1AJOFSsgtfT0_5LgvDNL_aEPftCnw3GEe')
params = {'corpora': 'teamDrive', 'includeItemsFromAllDrives': True, 'supportsAllDrives': True, 'teamDriveId': '0AKCBfHI3H-hcUk9PVA'}
def test_create_dataset(
self,
datasetmetadata,
test_data,
setup_teardown_folder,
params,
):
today = now_utc()
gclient, folderid = setup_teardown_folder
def create_gsheet(name, update):
payload = {
"name": name,
"mimeType": "application/vnd.google-apps.spreadsheet",
"parents": [folderid],
}
r = gclient.http_client.request(
"post", DRIVE_FILES_API_V3_URL, json=payload, params=params
)
spreadsheetid = r.json()["id"]
gsheet = gclient.open_by_key(spreadsheetid)
wks = gsheet.sheet1
wks.update(update, "A1")
gsheet.share("", role="reader", perm_type="anyone")
return wks, f"{gsheet.url}/export?format=xlsx"
name = "hdx_python_api_test"
dataset = Dataset.read_from_hdx(name)
if dataset:
dataset.delete_from_hdx()
title = "HDX Python API test"
dataset = Dataset({"name": name, "title": title})
dataset.update_from_yaml(datasetmetadata)
maintainer_id = "196196be-6037-4488-8b71-d786adf4c081"
dataset.set_maintainer(maintainer_id)
dataset.set_organization("5a63012e-6c41-420c-8c33-e84b277fdc90")
dataset.set_time_period(today)
dataset.set_expected_update_frequency("Every week")
dataset.set_subnational(True)
countryiso3s = ["AFG", "PSE", "SYR", "YEM"]
dataset.add_country_locations(countryiso3s)
tags = ["conflict-violence", "displacement"]
dataset.add_tags(tags)
resource_no = 0
def create_resource():
nonlocal resource_no
resource = Resource(
{
"name": f"test_resource_{resource_no}",
"description": f"Test Resource {resource_no}",
"last_modified": today.replace(tzinfo=None).isoformat(),
}
)
filestore = resource_no % 2 == 0
if filestore:
resource.set_format("csv")
resource.set_file_to_upload(test_data)
else:
wks, url = create_gsheet(
"resource1",
[[random.random() for i in range(10)] for j in range(10)],
)
resource.set_format("xlsx")
resource["url"] = url
resource_no += 1
dataset.add_update_resource(resource)
# add resources
for i in range(10):
create_resource()
dataset.create_in_hdx(updated_by_script="hdx_python_api_ignore")
# check created dataset
dataset = Dataset.read_from_hdx(name)
assert dataset["name"] == name
assert dataset["title"] == title
assert dataset.get_tags() == tags
assert dataset.get_maintainer()["id"] == maintainer_id
assert dataset.get_organization()["display_name"] == "INNAGO (inactive)"
resources = dataset.get_resources()
for i, resource in enumerate(resources):
assert resource["name"] == f"test_resource_{i}"
if i % 2 == 0:
assert resource.get_format() == "csv"
assert resource["url_type"] == "upload"
assert "humdata" in resource["url"]
else:
assert resource.get_format() == "xlsx"
assert resource["url_type"] == "api"
assert "humdata" not in resource["url"]
# modify dataset
dataset_id = dataset["id"]
title = "HDX Python API test changed"
notes = "added some notes"
caveats = "added some caveats"
# starting from a newly created Dataset object
dataset = Dataset(
{
"id": dataset_id,
"title": title,
"notes": notes,
"caveats": caveats,
}
)
tags.remove("displacement")
dataset.add_tags(tags)
countryiso3s.remove("YEM")
dataset.add_country_locations(countryiso3s)
resources.pop(3)
resources.pop()
gsheet_resource = resources[5]
gsheet_resource.set_format("csv")
gsheet_resource.set_file_to_upload(test_data)
for resource in resources:
del resource["package_id"]
dataset.add_update_resources(resources)
sleep(2)
dataset.update_in_hdx(remove_additional_resources=True)
sleep(2)
# check updated dataset
dataset = Dataset.read_from_hdx(name)
assert dataset["name"] == name
assert dataset["title"] == title
assert dataset["notes"] == notes
assert dataset["caveats"] == caveats
assert dataset.get_tags() == tags
assert dataset.get_maintainer()["id"] == maintainer_id
assert dataset.get_organization()["display_name"] == "INNAGO (inactive)"
updated_resources = dataset.get_resources()
for i, updated_resource in enumerate(updated_resources):
resource = resources[i]
assert updated_resource["name"] == resource["name"]
assert updated_resource.get_format() == resource.get_format()
assert updated_resource["url_type"].lower() == resource["url_type"]
url = resource.get("url")
if url:
if "humdata" in url:
assert "humdata" in updated_resource["url"]
else:
assert "humdata" not in updated_resource["url"]
else:
assert "humdata" in updated_resource["url"]
assert updated_resource.get("size") == resource.get("size")
assert updated_resource.get("hash") == resource.get("hash")
# modify dataset again starting with existing dataset
title = "HDX Python API test changed again"
dataset["title"] = title
del dataset["caveats"]
tags = ["agriculture-livestock", "climate-weather"]
dataset["tags"] = []
dataset.add_tags(tags)
countryiso3s.append("YEM")
dataset.add_country_location("YEM")
dataset.delete_resource(updated_resources[5])
updated_resources[0].set_file_to_upload(test_data)
create_resource()
resources = dataset.get_resources()
sleep(2)
dataset.create_in_hdx(
remove_additional_resources=True,
keys_to_delete=("caveats",),
)
sleep(2)
# check dataset updated for second time
dataset = Dataset.read_from_hdx(name)
assert dataset["name"] == name
> assert dataset["title"] == title
E AssertionError: assert 'HDX Python API test' == 'HDX Python A...changed again'
E
E #x1B[0m#x1B[91m- HDX Python API test changed again#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
E #x1B[92m+ HDX Python API test#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
tests/hdx/api/test_ckan.py:273: AssertionError
Loading