IPBlocklist/.github/workflows/aggregate-feeds.yml at master · tn3w/IPBlocklist · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
name: Update Feeds

on:
  schedule:
    - cron: '0 0 * * *'
  push:
    branches: [master]
    paths:
      - 'feeds.json'
      - 'feeds-intel.json'
      - 'aggregator.py'
      - 'builder/**'
      - 'cidr_minimizer/**'
  workflow_dispatch:

jobs:
  update:
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - uses: dtolnay/rust-toolchain@stable

      - name: Build cidr_minimizer
        run: cargo build --release
        working-directory: cidr_minimizer

      - name: Build intel builder
        run: cargo build --release
        working-directory: builder

      - name: Download and process feeds (legacy)
        run: python aggregator.py

      - name: Build intel.bin
        run: ./builder/target/release/builder update
        env:
          FEEDS_FILE: feeds-intel.json
          OUT_FILE: intel.bin
          PEERINGDB_API_KEY: ${{ secrets.PEERINGDB_API_KEY }}

      - name: Create Release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC")
          TAG_NAME="v$(date -u +%Y%m%d-%H%M%S)"

          gh release create "$TAG_NAME" \
            blocklist.bin \
            blocklist.txt \
            intel.bin \
            asns.json \
            asn_prefixes.json \
            --title "Blocklist Update - $TIMESTAMP" \
            --notes "Automated blocklist update from threat intelligence feeds.

          **Files:**
          - \`blocklist.bin\` - Legacy scored blocklist (binary format)
          - \`blocklist.txt\` - Scored and CIDR-minimized blocklist (text format)
          - \`intel.bin\` - Threat-intel SoA database built from feeds-intel.json
          - \`asns.json\` - Normalized ASN lists keyed by feed name
          - \`asn_prefixes.json\` - Cached ASN to announced prefixes mapping

          **Download:**
          \`\`\`bash
          wget https://github.com/tn3w/IPBlocklist/releases/download/$TAG_NAME/blocklist.bin
          wget https://github.com/tn3w/IPBlocklist/releases/download/$TAG_NAME/blocklist.txt
          wget https://github.com/tn3w/IPBlocklist/releases/download/$TAG_NAME/intel.bin
          wget https://github.com/tn3w/IPBlocklist/releases/download/$TAG_NAME/asns.json
          wget https://github.com/tn3w/IPBlocklist/releases/download/$TAG_NAME/asn_prefixes.json
          \`\`\`"

      - name: Clean up old releases
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          KEEP=5
          gh release list --limit 100 --json tagName --jq '.[].tagName' \
            | tail -n +$((KEEP + 1)) \
            | while read -r tag; do
                echo "Deleting release and tag: $tag"
                gh release delete "$tag" --yes --cleanup-tag
              done