From 20a59960440a0ea9d49dd8c7f31f2b52db9d7712 Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 22 Apr 2026 18:18:00 -0700 Subject: [PATCH 1/4] api threading speed up --- src/api.py | 136 +++++++++++++++++++--------------------------- tests/test_api.py | 72 +++++++++--------------- 2 files changed, 83 insertions(+), 125 deletions(-) diff --git a/src/api.py b/src/api.py index bff7b3a..8aa6e25 100644 --- a/src/api.py +++ b/src/api.py @@ -3,21 +3,20 @@ """ import logging +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta from functools import lru_cache from http import HTTPStatus from threading import Lock import numpy as np -import openmeteo_requests import pandas as pd import requests -import requests_cache from cachetools import TTLCache, cached from geopy.geocoders import Nominatim -from retry_requests import retry from src import helper +from src.open_meteo import openmeteo_client logger = logging.getLogger(__name__) @@ -25,26 +24,19 @@ # data expires after 600 seconds (10 min) _TTL = 600 -_ocean_cache = TTLCache(maxsize=300, ttl=_TTL) -_uv_cache = TTLCache(maxsize=300, ttl=_TTL) -uv_history_cache = TTLCache(maxsize=300, ttl=_TTL) -ocean_history_cache = TTLCache(maxsize=300, ttl=_TTL) -_wind_temp_cache = TTLCache(maxsize=300, ttl=_TTL) -_rain_cache = TTLCache(maxsize=300, ttl=_TTL) -forecast_cache = TTLCache(maxsize=300, ttl=_TTL) -_hourlyforecast_cache = TTLCache(maxsize=300, ttl=_TTL) +# max size = 300 items +_MAXSIZE = 300 +_ocean_cache = TTLCache(maxsize=_MAXSIZE, ttl=_TTL) +_uv_cache = TTLCache(maxsize=_MAXSIZE, ttl=_TTL) +uv_history_cache = TTLCache(maxsize=_MAXSIZE, ttl=_TTL) +ocean_history_cache = TTLCache(maxsize=_MAXSIZE, ttl=_TTL) +_wind_temp_cache = TTLCache(maxsize=_MAXSIZE, ttl=_TTL) +_rain_cache = TTLCache(maxsize=_MAXSIZE, ttl=_TTL) +forecast_cache = TTLCache(maxsize=_MAXSIZE, ttl=_TTL) +_hourlyforecast_cache = TTLCache(maxsize=_MAXSIZE, ttl=_TTL) _ocean_lock = Lock() -def _create_openmeteo_client() -> openmeteo_requests.Client: - """Creates a cached, retry-enabled Open-Meteo API client.""" - cache_session = requests_cache.CachedSession( - "/tmp/.cache", expire_after=3600 - ) - retry_session = retry(cache_session, retries=5, backoff_factor=0.2) - return openmeteo_requests.Client(session=retry_session) - - @lru_cache(maxsize=128) def get_coordinates(args: tuple) -> list | str: """ @@ -98,8 +90,6 @@ def get_uv( Get UV at coordinates (lat, long) Calling the API here: https://open-meteo.com/en/docs """ - openmeteo = _create_openmeteo_client() - url = "https://air-quality-api.open-meteo.com/v1/air-quality" params = { "latitude": lat, @@ -108,7 +98,7 @@ def get_uv( "current": "uv_index", } try: - responses = openmeteo.weather_api(url, params=params) + responses = openmeteo_client.weather_api(url, params=params) except ValueError: return "No data" @@ -146,8 +136,6 @@ def get_uv_history( API Documentation: https://open-meteo.com/en/docs/air-quality-api """ - openmeteo = _create_openmeteo_client() - # Calculate the date one year ago and the current hour one_year_ago = datetime.now() - timedelta(days=365) formatted_date_one_year_ago = one_year_ago.strftime("%Y-%m-%d") @@ -169,7 +157,7 @@ def get_uv_history( if testing == 1: # Attempt to fetch the UV index data from the API try: - responses = openmeteo.weather_api(url, params=params) + responses = openmeteo_client.weather_api(url, params=params) except ValueError: return "No data" @@ -198,8 +186,6 @@ def ocean_information( Get Ocean Data at coordinates API: https://open-meteo.com/en/docs/marine-weather-api """ - openmeteo = _create_openmeteo_client() - url = "https://marine-api.open-meteo.com/v1/marine" params = { "latitude": lat, @@ -210,7 +196,7 @@ def ocean_information( "forecast_days": 3, } try: - responses = openmeteo.weather_api(url, params=params) + responses = openmeteo_client.weather_api(url, params=params) except ValueError: return "No data" @@ -255,8 +241,6 @@ def ocean_information_history( API Documentation: https://open-meteo.com/en/docs/marine-weather-api """ - openmeteo = _create_openmeteo_client() - # Calculate the date and current hour one year ago one_year_ago = datetime.now() - timedelta(days=365) formatted_date_one_year_ago = one_year_ago.strftime("%Y-%m-%d") @@ -278,7 +262,7 @@ def ocean_information_history( if testing == 1: # Attempt to fetch the UV index data from the API try: - responses = openmeteo.weather_api(url, params=params) + responses = openmeteo_client.weather_api(url, params=params) except ValueError: return "No data" @@ -312,8 +296,6 @@ def current_wind_temp( """ Gathers the wind and temperature data """ - openmeteo = _create_openmeteo_client() - url = "https://api.open-meteo.com/v1/forecast" params = { "latitude": lat, @@ -322,7 +304,7 @@ def current_wind_temp( "temperature_unit": temp_unit, "wind_speed_unit": "mph", } - responses = openmeteo.weather_api(url, params=params) + responses = openmeteo_client.weather_api(url, params=params) response = responses[0] @@ -345,15 +327,13 @@ def get_rain(lat: float, long: float) -> tuple[float, float]: Get rain data at coordinates (lat, long) Calling the API here: https://open-meteo.com/en/docs """ - openmeteo = _create_openmeteo_client() - url = "https://api.open-meteo.com/v1/forecast" params = { "latitude": lat, "longitude": long, "daily": ["rain_sum", "precipitation_probability_max"], } - responses = openmeteo.weather_api(url, params=params) + responses = openmeteo_client.weather_api(url, params=params) response = responses[0] # Process daily data. The order of variables needs to be the @@ -374,8 +354,6 @@ def forecast(lat: float, long: float, decimal: int, days: int = 0) -> dict: Number of forecast days. Max is 7 API: https://open-meteo.com/en/docs/marine-weather-api """ - openmeteo = _create_openmeteo_client() - # First URL is the marine API. Second is for general weather/UV index urls = ( "https://marine-api.open-meteo.com/v1/marine", @@ -413,8 +391,12 @@ def forecast(lat: float, long: float, decimal: int, days: int = 0) -> dict: "forecast_days": days, } - responses_marine = openmeteo.weather_api(urls[0], params=params_marine) - responses_general = openmeteo.weather_api(urls[1], params=params_general) + responses_marine = openmeteo_client.weather_api( + urls[0], params=params_marine + ) + responses_general = openmeteo_client.weather_api( + urls[1], params=params_general + ) response_marine = responses_marine[0] response_general = responses_general[0] @@ -474,8 +456,6 @@ def get_hourly_forecast( """ Gets hourly weather data """ - openmeteo = _create_openmeteo_client() - # The order of variables in hourly or daily is important # to assign them correctly below url = "https://api.open-meteo.com/v1/forecast" @@ -489,7 +469,7 @@ def get_hourly_forecast( "forecast_days": days, } - responses = openmeteo.weather_api(url, params=params) + responses = openmeteo_client.weather_api(url, params=params) response = responses[0] hourly = response.Hourly() @@ -524,43 +504,42 @@ def gather_data(lat: float | str, long: float | str, arguments: dict) -> dict: in a dictionary (ocean_data_dict) """ lat, long = float(lat), float(long) - ocean_data = ocean_information( - lat, long, arguments["decimal"], arguments["unit"] - ) - - uv_index = get_uv(lat, long, arguments["decimal"], arguments["unit"]) - - hourly_dict = get_hourly_forecast(lat, long) + dec, unit = arguments["decimal"], arguments["unit"] + + with ThreadPoolExecutor(max_workers=8) as executor: + futures = { + "ocean": executor.submit(ocean_information, lat, long, dec, unit), + "uv": executor.submit(get_uv, lat, long, dec, unit), + "hourly": executor.submit(get_hourly_forecast, lat, long), + "wind_temp": executor.submit(current_wind_temp, lat, long, dec), + "rain": executor.submit(get_rain, lat, long), + "forecast": executor.submit(forecast, lat, long, dec, 7), + "ocean_hist": executor.submit( + ocean_information_history, lat, long, dec, unit + ), + "uv_hist": executor.submit(get_uv_history, lat, long, dec, unit), + } + results = {k: f.result() for k, f in futures.items()} - air_temp, wind_speed, wind_dir = current_wind_temp( - lat, long, arguments["decimal"] - ) - rain_sum, precipitation_probability_max = get_rain(lat, long) + arguments["ocean_data"] = results["ocean"] + arguments["uv_index"] = results["uv"] - arguments["ocean_data"] = ocean_data - arguments["uv_index"] = uv_index - spot_forecast = forecast(lat, long, arguments["decimal"], 7) - json_forecast = helper.forecast_to_json( - spot_forecast, arguments["decimal"] - ) + air_temp, wind_speed, wind_dir = results["wind_temp"] + rain_sum, precipitation_probability_max = results["rain"] + json_forecast = helper.forecast_to_json(results["forecast"], dec) - ocean_history = ocean_information_history( - lat, long, arguments["decimal"], arguments["unit"] - ) - ocean_data_dict = { + return { "Lat": lat, "Long": long, "Location": arguments["city"], - "Height": ocean_data[0], - "Height one year ago": ocean_history[0], - "Swell Direction": ocean_data[1], - "Swell Direction one year ago": ocean_history[1], - "Period": ocean_data[2], - "Period one year ago": ocean_history[2], - "UV Index": uv_index, - "UV Index one year ago": ( - get_uv_history(lat, long, arguments["decimal"], arguments["unit"]) - ), + "Height": results["ocean"][0], + "Height one year ago": results["ocean_hist"][0], + "Swell Direction": results["ocean"][1], + "Swell Direction one year ago": results["ocean_hist"][1], + "Period": results["ocean"][2], + "Period one year ago": results["ocean_hist"][2], + "UV Index": results["uv"], + "UV Index one year ago": results["uv_hist"], "Air Temperature": air_temp, "Wind Speed": wind_speed, "Wind Direction": wind_dir, @@ -568,10 +547,9 @@ def gather_data(lat: float | str, long: float | str, arguments: dict) -> dict: "Unit": arguments["unit"], "Rain Sum": rain_sum, "Precipitation Probability Max": precipitation_probability_max, - "Cloud Cover": hourly_dict["cloud_cover"], - "Visibility": hourly_dict["visibility"], + "Cloud Cover": results["hourly"]["cloud_cover"], + "Visibility": results["hourly"]["visibility"], } - return ocean_data_dict def separate_args_and_get_location(args: list) -> dict: diff --git a/tests/test_api.py b/tests/test_api.py index 27ce305..dfc6f88 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -90,7 +90,7 @@ def test_get_coordinates(mock_nominatim): assert isinstance(result[2], str) -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") def test_get_uv(mock_create_client): UV_INDEX = 5.0 mock_variable = MagicMock() @@ -102,9 +102,7 @@ def test_get_uv(mock_create_client): mock_response = MagicMock() mock_response.Current.return_value = mock_current - mock_client = MagicMock() - mock_client.weather_api.return_value = [mock_response] - mock_create_client.return_value = mock_client + mock_create_client.weather_api.return_value = [mock_response] result = get_uv(31.41, -84.92, 2, "imperial") @@ -112,7 +110,7 @@ def test_get_uv(mock_create_client): assert isinstance(result, float) -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") def test_ocean_information(mock_create_client): # fake each variable (height, direction, period) mock_var_0 = MagicMock() @@ -131,16 +129,14 @@ def test_ocean_information(mock_create_client): mock_response = MagicMock() mock_response.Current.return_value = mock_current - mock_client = MagicMock() - mock_client.weather_api.return_value = [mock_response] - mock_create_client.return_value = mock_client + mock_create_client.weather_api.return_value = [mock_response] result = ocean_information(31.41, -84.92, 2, "imperial") assert result == [3.5, 180.0, 12.0] -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") def test_forecast(mock_create_client): """ Test forecast() at an arbitrary location(palm beach), @@ -165,12 +161,10 @@ def test_forecast(mock_create_client): mock_general_response = MagicMock() mock_general_response.Daily.return_value = mock_general_daily - mock_client = MagicMock() - mock_client.weather_api.side_effect = [ + mock_create_client.weather_api.side_effect = [ [mock_marine_response], [mock_general_response], ] - mock_create_client.return_value = mock_client forecast_cache.clear() fc = forecast(26.705, -80.036, 1, 7) @@ -243,7 +237,7 @@ def test_seperate_args_and_get_location(mock_nominatim): assert "Pleasure Point" in str(city) -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") def test_get_uv_history_basic_functionality(mock_create_client): """ Test the basic functionality of the get_uv_history function. @@ -256,16 +250,14 @@ def test_get_uv_history_basic_functionality(mock_create_client): mock_hourly.Variables.return_value.ValuesAsNumpy.return_value = fake_uv mock_response = MagicMock() mock_response.Hourly.return_value = mock_hourly - mock_client = MagicMock() - mock_client.weather_api.return_value = [mock_response] - mock_create_client.return_value = mock_client + mock_create_client.weather_api.return_value = [mock_response] uv_history_cache.clear() uv = get_uv_history(31.9505, 115.8605, 2) assert isinstance(uv, str) -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") def test_get_uv_history_invalid_coordinates(mock_create_client): """ Test get_uv_history with invalid coordinates. @@ -273,18 +265,16 @@ def test_get_uv_history_invalid_coordinates(mock_create_client): This test checks that the function raises an OpenMeteoRequestsError when provided with latitude and longitude values that are out of range. """ - mock_client = MagicMock() - mock_client.weather_api.side_effect = OpenMeteoRequestsError( + mock_create_client.weather_api.side_effect = OpenMeteoRequestsError( "out of range" ) # noqa: E501 - mock_create_client.return_value = mock_client uv_history_cache.clear() with pytest.raises(OpenMeteoRequestsError): get_uv_history(1000, -2000, 2) -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") @patch("src.api.testing", new=0) # Set testing variable to 0 def test_get_uv_history_api_response(mock_create_client): """ @@ -293,15 +283,13 @@ def test_get_uv_history_api_response(mock_create_client): This test verifies that the function returns the expected values when called with valid coordinates while patching the API call request. """ - mock_create_client.return_value = MagicMock() - uv_history_cache.clear() result = get_uv_history(31.9505, 115.8605, 1) expected_result = "0.6" assert result == expected_result -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") def test_ocean_information_history_basic_functionality(mock_create_client): """ Test the basic functionality of the ocean_information_history function. @@ -314,9 +302,7 @@ def test_ocean_information_history_basic_functionality(mock_create_client): mock_hourly.Variables.return_value.ValuesAsNumpy.return_value = fake_array mock_response = MagicMock() mock_response.Hourly.return_value = mock_hourly - mock_client = MagicMock() - mock_client.weather_api.return_value = [mock_response] - mock_create_client.return_value = mock_client + mock_create_client.weather_api.return_value = [mock_response] ocean_history_cache.clear() waves = ocean_information_history(31.9505, 115.8605, 2) @@ -325,7 +311,7 @@ def test_ocean_information_history_basic_functionality(mock_create_client): assert waves[2] is not None -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") def test_ocean_information_history_invalid_coordinates(mock_create_client): """ Test ocean_information_history with invalid coordinates. @@ -333,18 +319,16 @@ def test_ocean_information_history_invalid_coordinates(mock_create_client): This test ensures that the function raises an OpenMeteoRequestsError when provided with latitude and longitude values that are out of range. """ - mock_client = MagicMock() - mock_client.weather_api.side_effect = OpenMeteoRequestsError( + mock_create_client.weather_api.side_effect = OpenMeteoRequestsError( "out of range" ) # noqa: E501 - mock_create_client.return_value = mock_client ocean_history_cache.clear() with pytest.raises(OpenMeteoRequestsError): ocean_information_history(1000, -2000, 2) -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") def test_ocean_information_history_response_format(mock_create_client): """ Test the response format of ocean_information_history. @@ -357,9 +341,7 @@ def test_ocean_information_history_response_format(mock_create_client): mock_hourly.Variables.return_value.ValuesAsNumpy.return_value = fake_array mock_response = MagicMock() mock_response.Hourly.return_value = mock_hourly - mock_client = MagicMock() - mock_client.weather_api.return_value = [mock_response] - mock_create_client.return_value = mock_client + mock_create_client.weather_api.return_value = [mock_response] ocean_history_cache.clear() waves = ocean_information_history(31.9505, 115.8605, 2) @@ -369,7 +351,7 @@ def test_ocean_information_history_response_format(mock_create_client): assert len(waves) == expected_wave_count -@patch("src.api._create_openmeteo_client") +@patch("src.api.openmeteo_client") @patch("src.api.testing", new=0) # Set testing variable to 0 def test_ocean_information_history(mock_create_client): """ @@ -378,8 +360,6 @@ def test_ocean_information_history(mock_create_client): This test verifies that the function returns the expected values when called with valid coordinates while patching the API call request. """ - mock_create_client.return_value = MagicMock() - ocean_history_cache.clear() result = ocean_information_history(31.9505, 115.8605, 1) expected_result = ["0.6", "0.6", "0.6"] @@ -421,28 +401,28 @@ def test_get_coordinates_invalid_location_falls_back_to_default( def test_get_uv_returns_no_data_on_value_error(mocker): """get_uv returns 'No data' when Open-Meteo client raises ValueError.""" - mock_client = mocker.patch("src.api._create_openmeteo_client") - mock_client.return_value.weather_api.side_effect = ValueError("bad coords") + mock_client = mocker.patch("src.api.openmeteo_client") + mock_client.weather_api.side_effect = ValueError("bad coords") assert get_uv(1000, -2000, 2) == "No data" def test_get_uv_history_returns_no_data_on_value_error(mocker): """get_uv_history returns 'No data' when the API raises ValueError.""" uv_history_cache.clear() - mock_client = mocker.patch("src.api._create_openmeteo_client") - mock_client.return_value.weather_api.side_effect = ValueError("bad coords") + mock_client = mocker.patch("src.api.openmeteo_client") + mock_client.weather_api.side_effect = ValueError("bad coords") assert get_uv_history(31.9505, 115.8605, 2) == "No data" def test_ocean_information_returns_no_data_on_value_error(mocker): """ocean_information returns 'No data' when the API raises ValueError.""" - mock_client = mocker.patch("src.api._create_openmeteo_client") - mock_client.return_value.weather_api.side_effect = ValueError("bad coords") + mock_client = mocker.patch("src.api.openmeteo_client") + mock_client.weather_api.side_effect = ValueError("bad coords") assert ocean_information(1000, -2000, 2) == "No data" def test_ocean_information_history_returns_no_data_on_value_error(mocker): """ocean_information_history returns 'No data' on ValueError.""" - mock_client = mocker.patch("src.api._create_openmeteo_client") - mock_client.return_value.weather_api.side_effect = ValueError("bad coords") + mock_client = mocker.patch("src.api.openmeteo_client") + mock_client.weather_api.side_effect = ValueError("bad coords") assert ocean_information_history(1000, -2000, 2) == "No data" From fcdb30b386895adf2271662474a2bc106d51db91 Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 22 Apr 2026 18:18:20 -0700 Subject: [PATCH 2/4] singleton open meteo client pattern --- src/open_meteo.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/open_meteo.py diff --git a/src/open_meteo.py b/src/open_meteo.py new file mode 100644 index 0000000..cb90701 --- /dev/null +++ b/src/open_meteo.py @@ -0,0 +1,12 @@ +import requests +import openmeteo_requests +from retry_requests import retry + + +def _create_client() -> openmeteo_requests.Client: + """Creates a retry-enabled Open-Meteo API client.""" + retry_session = retry(requests.Session(), retries=5, backoff_factor=0.2) + return openmeteo_requests.Client(session=retry_session) + + +openmeteo_client = _create_client() From 58ef7d3eb947be5d30a04e0a58660b6a8ebcdabc Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 22 Apr 2026 18:18:40 -0700 Subject: [PATCH 3/4] formatting fix --- src/open_meteo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/open_meteo.py b/src/open_meteo.py index cb90701..0a58865 100644 --- a/src/open_meteo.py +++ b/src/open_meteo.py @@ -1,5 +1,5 @@ -import requests import openmeteo_requests +import requests from retry_requests import retry From 2af32dfecca747cd261515611bcd163a9e20a01f Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 22 Apr 2026 18:39:18 -0700 Subject: [PATCH 4/4] cleanup of unused lambda zip logic --- makefile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/makefile b/makefile index 83cb9ff..0f9365a 100644 --- a/makefile +++ b/makefile @@ -66,13 +66,6 @@ clean: .PHONY: all all: format lint test -.PHONY: lambda-zip -lambda-zip: - poetry export -f requirements.txt --output requirements.txt --without-hashes - pip install -r requirements.txt -t ./package - cp -r src/ ./package/src/ - cd package && zip -r ../terraform/lambda.zip . && cd .. - ECR_REGION ?= us-west-1 ECR_ACCOUNT_ID ?= $(shell aws sts get-caller-identity --query Account --output text) ECR_REPO = $(ECR_ACCOUNT_ID).dkr.ecr.$(ECR_REGION).amazonaws.com/cli-surf