build_parameters.py: use local cache and server cache to get files

Author: khancyr

build_parameters.py

@@ -36,6 +36,7 @@
 import sys
 import time  # noqa: F401
 from html.parser import HTMLParser
+from urllib.parse import quote
 
 import requests
 
@@ -44,6 +45,15 @@
 parser.add_argument("--ardupilotRepoFolder", dest='gitFolder', default="../ardupilot", help="Ardupilot git folder. ")
 parser.add_argument("--destination", dest='destFolder', default="../../../../new_params_mversion", help="Parameters*.rst destination folder.")  # noqa: E501
 parser.add_argument('--vehicle', dest='single_vehicle', help="If you just want to copy to one vehicle, you can do this. Otherwise it will work for all vehicles (Copter, Plane, Rover, AntennaTracker, Sub, Blimp)")  # noqa: E501
+
+DEFAULT_CACHE_TIME = 6 * 3600
+
+# Get the directory where this script is located
+script_dir = os.path.dirname(os.path.abspath(__file__))
+default_http_request_cache_dir = os.path.join(script_dir, '.cache')
+
+parser.add_argument("--cache-dir", dest='cache_dir', default=default_http_request_cache_dir,
+                    help="Directory to cache HTTP responses")
 args = parser.parse_args()
 
 
@@ -94,6 +104,102 @@ def format(self, record):
 })
 
 
+def fetch_url_with_cache(url, cache_dir=None):
+    """Fetch URL content with caching to avoid repeated downloads."""
+    if cache_dir is None:
+        cache_dir = args.cache_dir
+
+    os.makedirs(cache_dir, exist_ok=True)
+
+    # Create cache filename from URL
+    cache_filename = quote(url, safe='') + '.cache'
+    cache_path = os.path.join(cache_dir, cache_filename)
+    cache_meta_path = cache_path + '.meta'
+
+    def load_cached_content():
+        with open(cache_path, 'r', encoding='utf-8') as f:
+            return f.read()
+
+    def load_cache_metadata():
+        if not os.path.exists(cache_meta_path):
+            return {}
+        try:
+            with open(cache_meta_path, 'r', encoding='utf-8') as f:
+                return json.load(f)
+        except OSError:
+            return {}
+
+    def save_cache(content, response):
+        with open(cache_path, 'w', encoding='utf-8') as f:
+            f.write(content)
+
+        metadata = {}
+        etag = response.headers.get('ETag')
+        last_modified = response.headers.get('Last-Modified')
+        if etag:
+            metadata['etag'] = etag
+        if last_modified:
+            metadata['last_modified'] = last_modified
+        if metadata:
+            with open(cache_meta_path, 'w', encoding='utf-8') as f:
+                json.dump(metadata, f)
+
+    def refresh_cache_mtime():
+        try:
+            os.utime(cache_path, None)
+        except OSError:
+            pass
+
+    if os.path.exists(cache_path):
+        cache_age = time.time() - os.path.getmtime(cache_path)
+        if cache_age < DEFAULT_CACHE_TIME:
+            debug(f"Using cached content for {url}")
+            return load_cached_content()
+
+    cache_metadata = {}
+    headers = {}
+    if os.path.exists(cache_path):
+        cache_metadata = load_cache_metadata()
+        if cache_metadata.get('etag'):
+            headers['If-None-Match'] = cache_metadata['etag']
+        if cache_metadata.get('last_modified'):
+            headers['If-Modified-Since'] = cache_metadata['last_modified']
+
+    if headers:
+        try:
+            debug(f"HEAD checking server for {url}")
+            head_response = session.head(url, timeout=30, headers=headers, allow_redirects=True)
+            if head_response.status_code == 304:
+                debug(f"Cache still valid for {url}")
+                refresh_cache_mtime()
+                return load_cached_content()
+
+            head_response.raise_for_status()
+            if (head_response.headers.get('ETag') == cache_metadata.get('etag') and
+                    head_response.headers.get('Last-Modified') == cache_metadata.get('last_modified')):
+                debug(f"Server metadata unchanged for {url}, using local cache")
+                refresh_cache_mtime()
+                return load_cached_content()
+        except requests.RequestException as e:
+            debug(f"HEAD request failed for {url}: {e}")
+            # Fallback to GET if the HEAD request is unsupported or fails.
+
+    try:
+        debug(f"Fetching full content from {url}")
+        response = session.get(url, timeout=30, allow_redirects=True)
+        response.raise_for_status()
+        content = response.text
+    except requests.RequestException as e:
+        error(f"Failed to fetch {url}: {e}")
+        if os.path.exists(cache_path):
+            debug(f"Using stale cached content for {url}")
+            return load_cached_content()
+        raise
+
+    save_cache(content, response)
+    return content
+
+
 def run_git(cmd, cwd=None, check=True, max_retries=3):
     """Run git command with retry logic for lock conflicts"""
     if cwd is None:
@@ -406,9 +512,7 @@ def handle_starttag(self, tag, attrs):
         try:
             debug(f"Fetching {firmware_url}{vehicle}")
 
-            response = session.get(firmware_url + vehicle, timeout=30)
-            response.raise_for_status()
-            content = response.text
+            content = fetch_url_with_cache(firmware_url + vehicle)
             html_parser.feed(content)
         except Exception as e:
             error(f"Vehicles folders list download error: {e}")
@@ -460,9 +564,7 @@ def handle_starttag(self, tag, attrs):
         try:
             debug(f"Fetching {url}")
 
-            response = session.get(url, timeout=30)
-            response.raise_for_status()
-            content = response.text
+            content = fetch_url_with_cache(url)
             html_parser.feed(content)
         except Exception as e:
             error(f"Board folders list download error: {e}")
@@ -483,11 +585,9 @@ def fetch_commit_hash(version_link, board, file):
         progress(f"Processing link...\t{fetch_link}")
 
         try:
-            response = session.get(fetch_link, timeout=30)
-            response.raise_for_status()
-            fecth_response = response.text
+            fetch_response = fetch_url_with_cache(fetch_link)
 
-            commit_details = fecth_response.split("\n")
+            commit_details = fetch_response.split("\n")
             commit_hash = commit_details[0][7:]
             # version =  commit_details[6] the sizes cary
             version = commit_details.pop(-2)