diff --git a/databank/management/commands/sources/RELIEFWEB.py b/databank/management/commands/sources/RELIEFWEB.py index cc7227c84..308b8b5a2 100644 --- a/databank/management/commands/sources/RELIEFWEB.py +++ b/databank/management/commands/sources/RELIEFWEB.py @@ -21,6 +21,30 @@ def parse_date(date): return datetime.datetime.strptime(date.split("T")[0], RELIEFWEB_DATETIME_FORMAT) +def _post_reliefweb(query_params, url, context): + try: + response = requests.post(url, data=query_params) + response.raise_for_status() + return response.json() + except requests.HTTPError as exc: + status_code = exc.response.status_code if exc.response else None + if status_code == 410: + logger.warning( + "ReliefWeb API returned 410 Gone for %s. Skipping %s prefetch. URL: %s", + DISASTER_API, + context, + url, + ) + return None + raise + except requests.RequestException: + logger.exception("ReliefWeb API request failed for %s. URL: %s", context, url) + raise + except ValueError: + logger.exception("ReliefWeb API returned invalid JSON for %s. URL: %s", context, url) + raise + + def _crises_event_prefetch(): query_params = json.dumps( { @@ -42,9 +66,9 @@ def _crises_event_prefetch(): url = DISASTER_API data = {} while True: - response = requests.post(url, data=query_params) - response.raise_for_status() - response = response.json() + response = _post_reliefweb(query_params, url, "crises_event") + if response is None: + return {} for disaster in response["data"]: disaster = disaster["fields"] @@ -90,9 +114,9 @@ def _epidemics_prefetch(): url = DISASTER_API data = {} while True: - response = requests.post(url, data=query_params) - response.raise_for_status() - response = response.json() + response = _post_reliefweb(query_params, url, "epidemics") + if response is None: + return {} for epidemic in response["data"]: epidemic = epidemic["fields"] diff --git a/deploy/helm/ifrcgo-helm/values.yaml b/deploy/helm/ifrcgo-helm/values.yaml index 49767eacc..3ffaaeb8b 100644 --- a/deploy/helm/ifrcgo-helm/values.yaml +++ b/deploy/helm/ifrcgo-helm/values.yaml @@ -315,3 +315,4 @@ sshBastion: thenav56.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN/f/A3qkaTHSdbKn8Hv75YiJvRMEXvWTDdIiR7tyAjJ navin@nav-machine david.pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3FzrQdVh5Qwp5Y6KQGcpqHxKErxCW103iEECuutR/jBZe6X0xjD+cW7e+H8SrUsPQwj87fzOsMAc6v6n+3hdYFa6ekgRG/USEIUR5C/GD1Xjva3Xpp45PasBhJEtYt2ON+dlzwvRyOuv2hvqv2WHBO020ewIlVuQ4pU4Qj5ysvwWGj8GAv/jITiVERmjLTStbFwxeIDT3jQEbwnfV1zZZKiGxIecB/y51nk6oIQ00ZGrYEo5ieWsUSVfLHOX0/lZ0mtrdqxDEgMaCbNaUbICAimsJPamNpoirKc7FoKIKKrLQsK8qE1lClWQEecbW+dgSiwxracooKeWhHq+BkKUCNgEL/C0ff2l9e8sJcLmYZUdPtDCdtUDC8BAlELA5HR6tdCTfFcc0nXltclSSODMnZkQohh5/2fixJTwN5p5csEfBLzbdrturKtT/TbYSoaodg4muPqY4YE5jiJfrHVAGS1DVWz/cRcm1vOxT2V4iW2SNvo8fS2PZOpU5furrvbM= ifrcds\david.muchatiza@5CG41911S1 paola.pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGql4RrbxSQTW5QrTh+P+94jGCXOCeZgc23hxL9zFCYQrzL0SMw1F53Z5SFZimIhJswYPqV2pT8L4oTRqIrTCM+looWi7b9/9u+m/KmA+FWbo3u6uRrckkA3nVIKsKHvlOucX2GxE6i+tXdeXEisW49ZpMtuvxMLJ3Eg4MK10d/2d3FKuzTsrxCTlJn8FAE3yOsVow0jdu+381IrkAqRE2GINeQ87hVlQpbo+bL2N/2QZmNjDhBBQkRJLDisW0+UNgo+S9wN7HbpV5LheSJS9wGN7LlmcqlpZFrDO/lVyoMxEQ0588wUI8BVfqAZDEBJPdGtzq513r+5iXEX/9A1Mendlvxfl6ANNRcH9PVZHkRN1dxY3rckQ+Lk3qqIjjfYFYvl5Gybidb1BM2VNWHAuzaDDQzJpeTHIbQnDt7Ke4oX2xWYgyu+kVhqz0HnAV28qMXbMEsrMIrtwl7IjcrorgdduHghZvWFbaJZNtXOfgnf1IYNXkZ9eWPS+Bz9nWMhE= ifrcds\paola.yela@5CG41911RT + ranjan.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGJA0ec4Gavc+m1MjEZGoUce51yWouMTRTYJZV3s/jgD rsh@rsh-XPS-15-9510