Create a pipeline to archive_urls

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

archize.py

@@ -0,0 +1,91 @@
+import json
+import time
+from datetime import datetime
+
+import requests
+
+# ── CONFIG ────────────────────────────────────────────────────────────────────
+URLS = [
+    "https://github.com/aboutcode-org/vulnerablecode/issues/17",
+]
+
+DELAY_SECONDS = 5  # pause between requests to avoid rate-limiting
+LOG_FILE = "archive_log.json"
+# ─────────────────────────────────────────────────────────────────────────────
+
+SPN_ENDPOINT = "https://web.archive.org/save/"
+
+
+def save_url(url: str) -> dict:
+    """Submit a single URL to the Wayback Machine."""
+    try:
+        response = requests.post(
+            SPN_ENDPOINT,
+            data={"url": url},
+            headers={"User-Agent": "ArchiveBot/1.0"},
+            timeout=30,
+        )
+
+        if response.status_code == 200:
+            # Archive.org returns the archived URL in the Content-Location header
+            location = response.headers.get("Content-Location", "")
+            archived_url = f"https://web.archive.org{location}" if location else "check manually"
+            return {"url": url, "status": "success", "archived_url": archived_url}
+
+        else:
+            return {
+                "url": url,
+                "status": "failed",
+                "http_code": response.status_code,
+                "reason": response.text[:200],
+            }
+
+    except requests.exceptions.Timeout:
+        return {"url": url, "status": "error", "reason": "Request timed out"}
+    except requests.exceptions.RequestException as e:
+        return {"url": url, "status": "error", "reason": str(e)}
+
+
+def archive_all(urls: list[str]) -> list[dict]:
+    results = []
+    total = len(urls)
+
+    print(f"Starting archive of {total} URL(s)...\n")
+
+    for i, url in enumerate(urls, start=1):
+        print(f"[{i}/{total}] Submitting: {url}")
+        result = save_url(url)
+        result["timestamp"] = datetime.utcnow().isoformat()
+        results.append(result)
+
+        if result["status"] == "success":
+            print(f"  ✓ Archived → {result['archived_url']}")
+        else:
+            print(f"  ✗ {result.get('reason') or result.get('http_code')}")
+
+        if i < total:
+            time.sleep(DELAY_SECONDS)
+
+    return results
+
+
+def save_log(results: list[dict], path: str) -> None:
+    with open(path, "w") as f:
+        json.dump(results, f, indent=2)
+    print(f"\nLog saved to {path}")
+
+
+def print_summary(results: list[dict]) -> None:
+    success = sum(1 for r in results if r["status"] == "success")
+    failed = len(results) - success
+    print(f"\n── Summary ──────────────────────")
+    print(f"  Total   : {len(results)}")
+    print(f"  Success : {success}")
+    print(f"  Failed  : {failed}")
+    print(f"─────────────────────────────────")
+
+
+if __name__ == "__main__":
+    results = archive_all(URLS)
+    print_summary(results)
+    save_log(results, LOG_FILE)

vulnerabilities/improvers/__init__.py

@@ -19,6 +19,7 @@
 from vulnerabilities.pipelines import flag_ghost_packages
 from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
 from vulnerabilities.pipelines import remove_duplicate_advisories
+from vulnerabilities.pipelines.v2_improvers import archive_urls
 from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees
 from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
@@ -74,5 +75,6 @@
         compute_advisory_todo.ComputeToDo,
         collect_ssvc_trees.CollectSSVCPipeline,
         relate_severities.RelateSeveritiesPipeline,
+        archive_urls.ArchiveImproverPipeline,
     ]
 )

vulnerabilities/migrations/0116_advisoryreference_archive_url.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.2.11 on 2026-03-20 19:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0115_impactedpackageaffecting_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryreference",
+            name="archive_url",
+            field=models.URLField(
+                help_text="URL to the backup vulnerability reference", max_length=1024, null=True
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -2672,6 +2672,12 @@ class AdvisoryReference(models.Model):
         help_text="URL to the vulnerability reference",
     )
 
+    archive_url = models.URLField(
+        max_length=1024,
+        null=True,
+        help_text="URL to the backup vulnerability reference",
+    )
+
     ADVISORY = "advisory"
     EXPLOIT = "exploit"
     COMMIT = "commit"

vulnerabilities/pipelines/v2_improvers/archive_urls.py

@@ -0,0 +1,74 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import time
+import requests
+from vulnerabilities.models import AdvisoryReference
+from vulnerabilities.pipelines import VulnerableCodePipeline
+from vulnerabilities.wayback_machine import WaybackMachineSaveAPI
+
+
+class ArchiveImproverPipeline(VulnerableCodePipeline):
+    """
+    Archive Improver Pipeline
+    """
+
+    pipeline_id = "archive_improver_pipeline"
+
+    @classmethod
+    def steps(cls):
+        return (cls.archive_urls,)
+
+    def archive_urls(self):
+        advisory_refs = AdvisoryReference.objects.filter(archive_url__isnull=True).only("id", "url")
+
+        for advisory_ref in advisory_refs.iterator():
+
+            if not advisory_ref.url.startswith("http"):
+                continue
+
+            if not self.is_reachable_url(advisory_ref.url):
+                self.log(
+                    f"Skipping archival: URL is unreachable or returned 404: {advisory_ref.url}"
+                )
+                continue
+
+            self.log(f"Requesting archival for: {advisory_ref.url}")
+            try:
+                time.sleep(300)
+                archive_url = self.request_archival(advisory_ref.url)
+                if not archive_url:
+                    continue
+
+                AdvisoryReference.objects.filter(id=advisory_ref.id).update(archive_url=archive_url)
+                self.log(f"Successfully added archival URL for advisory reference: {archive_url}")
+            except Exception as e:
+                self.log(f"Failed to archive {advisory_ref.url}: {str(e)}")
+
+    def request_archival(self, url):
+        user_agent = "Mozilla/5.0 (Windows NT 5.1; rv:40.0) Gecko/20100101 Firefox/40.0"
+        try:
+            save_api = WaybackMachineSaveAPI(url, user_agent)
+            return save_api.save()
+        except Exception as e:
+            self.log(f"Failed to archive {url}: {str(e)}")
+            return None
+
+    def is_reachable_url(self, url):
+        try:
+            with requests.Session() as session:
+                head_res = session.head(url, allow_redirects=True, timeout=10)
+                if not head_res.status_code == 200:
+                    return False
+
+                get_res = session.get(url, allow_redirects=True, stream=True, timeout=10)
+                return get_res.status_code == 200
+
+        except requests.RequestException:
+            return False

vulnerabilities/wayback_machine.ABOUT

@@ -0,0 +1,10 @@
+about_resource: wayback_machine.py
+version: 5407681c34ac299ad5c0622ea0c2d186a330fec1
+download_url: https://raw.githubusercontent.com/akamhy/waybackpy/5407681c34ac299ad5c0622ea0c2d186a330fec1/waybackpy/save_api.py
+package_url: pkg:github/akamhy/waybackpy@5407681c34ac299ad5c0622ea0c2d186a330fec1#waybackpy/save_api.py
+homepage_url: https://github.com/akamhy/waybackpy
+notes: There are two files extracted from waybackpy that handle API calls—`save_api.py` and `exceptions.py` and they are saved in `wayback_machine.py`
+
+copyright: Copyright (c) 2010 United States Government. All Rights Reserved.
+license_expression: MIT License
+license_file: wayback_machine.py.LICENSE

vulnerabilities/wayback_machine.LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020-2022 waybackpy contributors ( https://github.com/akamhy/waybackpy/graphs/contributors )
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.