Skip to content

Use paging for all_fields True

d2423b1
Select commit
Loading
Failed to load commit list.
Sign in for the full log view
Merged

HDXDSYS-2865 Use paging for all_fields True #109

Use paging for all_fields True
d2423b1
Select commit
Loading
Failed to load commit list.
GitHub Actions / Test Results failed Apr 7, 2026 in 0s

1 fail, 156 pass in 1m 25s

157 tests   156 ✅  1m 25s ⏱️
  1 suites    0 💤
  1 files      1 ❌

Results for commit d2423b1.

Annotations

Check warning on line 0 in tests.hdx.api.test_ckan.TestCKAN

See this annotation in the file changed.

@github-actions github-actions / Test Results

test_create_dataset (tests.hdx.api.test_ckan.TestCKAN) failed

test-results.xml [took 1m 10s]
Raw output
AssertionError: assert 'HDX Python API test' == 'HDX Python A...changed again'
  
  #x1B[0m#x1B[91m- HDX Python API test changed again#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
  #x1B[92m+ HDX Python API test#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
self = <tests.hdx.api.test_ckan.TestCKAN object at 0x7f2d0fd85590>
datasetmetadata = PosixPath('tests/fixtures/CKAN/hdx_dataset_static.yaml')
test_data = PosixPath('tests/fixtures/test_data.csv')
setup_teardown_folder = (<gspread.client.Client object at 0x7f2d0f735e80>, '1AJOFSsgtfT0_5LgvDNL_aEPftCnw3GEe')
params = {'corpora': 'teamDrive', 'includeItemsFromAllDrives': True, 'supportsAllDrives': True, 'teamDriveId': '0AKCBfHI3H-hcUk9PVA'}

    def test_create_dataset(
        self,
        datasetmetadata,
        test_data,
        setup_teardown_folder,
        params,
    ):
        today = now_utc()
        gclient, folderid = setup_teardown_folder
    
        def create_gsheet(name, update):
            payload = {
                "name": name,
                "mimeType": "application/vnd.google-apps.spreadsheet",
                "parents": [folderid],
            }
            r = gclient.http_client.request(
                "post", DRIVE_FILES_API_V3_URL, json=payload, params=params
            )
            spreadsheetid = r.json()["id"]
            gsheet = gclient.open_by_key(spreadsheetid)
            wks = gsheet.sheet1
            wks.update(update, "A1")
            gsheet.share("", role="reader", perm_type="anyone")
            return wks, f"{gsheet.url}/export?format=xlsx"
    
        name = "hdx_python_api_test"
        dataset = Dataset.read_from_hdx(name)
        if dataset:
            dataset.delete_from_hdx()
        title = "HDX Python API test"
        dataset = Dataset({"name": name, "title": title})
        dataset.update_from_yaml(datasetmetadata)
        maintainer_id = "196196be-6037-4488-8b71-d786adf4c081"
        dataset.set_maintainer(maintainer_id)
        dataset.set_organization("5a63012e-6c41-420c-8c33-e84b277fdc90")
        dataset.set_time_period(today)
        dataset.set_expected_update_frequency("Every week")
        dataset.set_subnational(True)
        countryiso3s = ["AFG", "PSE", "SYR", "YEM"]
        dataset.add_country_locations(countryiso3s)
        tags = ["conflict-violence", "displacement"]
        dataset.add_tags(tags)
        resource_no = 0
    
        def create_resource():
            nonlocal resource_no
    
            resource = Resource(
                {
                    "name": f"test_resource_{resource_no}",
                    "description": f"Test Resource {resource_no}",
                    "last_modified": today.replace(tzinfo=None).isoformat(),
                }
            )
            filestore = resource_no % 2 == 0
            if filestore:
                resource.set_format("csv")
                resource.set_file_to_upload(test_data)
            else:
                wks, url = create_gsheet(
                    "resource1",
                    [[random.random() for i in range(10)] for j in range(10)],
                )
                resource.set_format("xlsx")
                resource["url"] = url
    
            resource_no += 1
            dataset.add_update_resource(resource)
    
        # add resources
        for i in range(10):
            create_resource()
    
        dataset.create_in_hdx(updated_by_script="hdx_python_api_ignore")
    
        # check created dataset
        dataset = Dataset.read_from_hdx(name)
        assert dataset["name"] == name
        assert dataset["title"] == title
        assert dataset.get_tags() == tags
        assert dataset.get_maintainer()["id"] == maintainer_id
        assert dataset.get_organization()["display_name"] == "INNAGO (inactive)"
        resources = dataset.get_resources()
        for i, resource in enumerate(resources):
            assert resource["name"] == f"test_resource_{i}"
            if i % 2 == 0:
                assert resource.get_format() == "csv"
                assert resource["url_type"] == "upload"
                assert "humdata" in resource["url"]
            else:
                assert resource.get_format() == "xlsx"
                assert resource["url_type"] == "api"
                assert "humdata" not in resource["url"]
    
        # modify dataset
        dataset_id = dataset["id"]
        title = "HDX Python API test changed"
        notes = "added some notes"
        caveats = "added some caveats"
        # starting from a newly created Dataset object
        dataset = Dataset(
            {
                "id": dataset_id,
                "title": title,
                "notes": notes,
                "caveats": caveats,
            }
        )
        tags.remove("displacement")
        dataset.add_tags(tags)
        countryiso3s.remove("YEM")
        dataset.add_country_locations(countryiso3s)
        resources.pop(3)
        resources.pop()
        gsheet_resource = resources[5]
        gsheet_resource.set_format("csv")
        gsheet_resource.set_file_to_upload(test_data)
        for resource in resources:
            del resource["package_id"]
        dataset.add_update_resources(resources)
    
        sleep(2)
        dataset.update_in_hdx(remove_additional_resources=True)
        sleep(2)
    
        # check updated dataset
        dataset = Dataset.read_from_hdx(name)
        assert dataset["name"] == name
        assert dataset["title"] == title
        assert dataset["notes"] == notes
        assert dataset["caveats"] == caveats
        assert dataset.get_tags() == tags
        assert dataset.get_maintainer()["id"] == maintainer_id
        assert dataset.get_organization()["display_name"] == "INNAGO (inactive)"
        updated_resources = dataset.get_resources()
        for i, updated_resource in enumerate(updated_resources):
            resource = resources[i]
            assert updated_resource["name"] == resource["name"]
            assert updated_resource.get_format() == resource.get_format()
            assert updated_resource["url_type"].lower() == resource["url_type"]
            url = resource.get("url")
            if url:
                if "humdata" in url:
                    assert "humdata" in updated_resource["url"]
                else:
                    assert "humdata" not in updated_resource["url"]
            else:
                assert "humdata" in updated_resource["url"]
            assert updated_resource.get("size") == resource.get("size")
            assert updated_resource.get("hash") == resource.get("hash")
    
        # modify dataset again starting with existing dataset
        title = "HDX Python API test changed again"
        dataset["title"] = title
        del dataset["caveats"]
        tags = ["agriculture-livestock", "climate-weather"]
        dataset["tags"] = []
        dataset.add_tags(tags)
        countryiso3s.append("YEM")
        dataset.add_country_location("YEM")
        dataset.delete_resource(updated_resources[5])
        updated_resources[0].set_file_to_upload(test_data)
        create_resource()
        resources = dataset.get_resources()
    
        sleep(2)
        dataset.create_in_hdx(
            remove_additional_resources=True,
            keys_to_delete=("caveats",),
        )
        sleep(2)
    
        # check dataset updated for second time
        dataset = Dataset.read_from_hdx(name)
        assert dataset["name"] == name
>       assert dataset["title"] == title
E       AssertionError: assert 'HDX Python API test' == 'HDX Python A...changed again'
E         
E         #x1B[0m#x1B[91m- HDX Python API test changed again#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
E         #x1B[92m+ HDX Python API test#x1B[39;49;00m#x1B[90m#x1B[39;49;00m

tests/hdx/api/test_ckan.py:273: AssertionError