Add script to get ips block

Author: Rub21

.github/workflows/update-waf-firewall.yml

@@ -16,8 +16,9 @@ env:
   SCOPE: REGIONAL
   IPSET_NAME_STAGING: ipset-block-ohm-staging
   IPSET_NAME_PROD: ipset-block-ohm-production
-  FILE_STAGING: firewall/ip-blacklist-staging.txt
-  FILE_PROD: firewall/ip-blacklist-production.txt
+  # --- Archivos actualizados a .yaml ---
+  FILE_STAGING: firewall/ip-blacklist-staging.yaml
+  FILE_PROD: firewall/ip-blacklist-production.yaml
 
 jobs:
   update-waf-ipset:
@@ -27,54 +28,51 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # Configure AWS credentials from secrets (access keys)
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Install jq
-        run: sudo apt-get update && sudo apt-get install -y jq
+      - name: Install jq and yq
+        run: |
+          sudo apt-get update && sudo apt-get install -y jq
+          sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq
+          sudo chmod +x /usr/bin/yq
 
       - name: Resolve target env, IP set and file
         id: target
         run: |
-          ENV_IN="${{ inputs.environment }}"
-          if [[ "$ENV_IN" == "main" ]]; then
+          # This logic remains the same, but will now point to .yaml files
+          if [[ "${{ github.ref_name }}" == "main" ]]; then
             echo "IPSET_NAME=${IPSET_NAME_PROD}" >> $GITHUB_OUTPUT
             echo "IP_FILE=${FILE_PROD}" >> $GITHUB_OUTPUT
           else
             echo "IPSET_NAME=${IPSET_NAME_STAGING}" >> $GITHUB_OUTPUT
             echo "IP_FILE=${FILE_STAGING}" >> $GITHUB_OUTPUT
           fi
 
-      - name: Build IP list
+      - name: Build IP list from YAML
         id: iplist
         shell: bash
         run: |
           TMP=$(mktemp)
-          if [[ -n "${{ inputs.ips }}" ]]; then
-            # Convert space/newline-separated input to one-per-line
-            echo "${{ inputs.ips }}" | tr ' ' '\n' | sed '/^\s*$/d' > "$TMP"
-          else
-            FILE="${{ steps.target.outputs.IP_FILE }}"
-            if [[ ! -f "$FILE" ]]; then
-              echo "File $FILE not found" >&2
-              exit 1
-            fi
-            # Remove comments and blank lines
-            sed '/^\s*#/d;/^\s*$/d' "$FILE" > "$TMP"
+          FILE="${{ steps.target.outputs.IP_FILE }}"
+          if [[ ! -f "$FILE" ]]; then
+            echo "File $FILE not found" >&2
+            exit 1
           fi
+          # --- Cambio principal: Usar yq para leer el YAML ---
+          # Extrae cada IP de la lista 'block_ips' y la pone en una nueva línea
+          yq '.block_ips[]' "$FILE" > "$TMP"
 
-          # Basic validation for IPv4/IPv6 with optional CIDR
+          # La validación y el resto del script no necesitan cambios
           INVALID=$(grep -Ev '^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$|^([0-9a-fA-F:]+)(/[0-9]{1,3})?$' "$TMP" || true)
           if [[ -n "$INVALID" ]]; then
             echo "Invalid entries:"; echo "$INVALID"; exit 1
           fi
 
-          # Unique and sorted list
           sort -u "$TMP" > "${TMP}.uniq"
           LIST=$(paste -sd' ' "${TMP}.uniq")
           echo "addresses=$LIST" >> $GITHUB_OUTPUT
@@ -105,4 +103,4 @@ jobs:
 
       - name: Summary
         run: |
-          echo "Updated IP set: ${{ steps.target.outputs.IPSET_NAME }}"
+          echo "Updated IP set: ${{ steps.target.outputs.IPSET_NAME }}"

firewall/README.md

@@ -0,0 +1,4 @@
+## Blocking High-Probability Bot IPs Based on Traffic Patterns
+
+We’re blocking IPs with a high probability of being bots. Analysis shows these IPs generated excessive traffic on the site, following clear bot-like patterns.
+

firewall/ip-blacklist-production.txt

@@ -0,0 +1,5 @@
+block_ips:
+   ## Test Rub21 ips
+  200.60.4.59/32
+  200.37.252.67/32
+  200.10.69.30/32

firewall/ip-blacklist-production.yaml

@@ -0,0 +1,5 @@
+block_ips:
+  ## Test Rub21 ips
+  200.60.4.59/32
+  200.37.252.67/32
+  200.10.69.30/32

firewall/ip-blacklist-staging.txt

@@ -0,0 +1,61 @@
+/* This query is used to create the external table for the alb logs. */
+
+CREATE EXTERNAL TABLE IF NOT EXISTS alb_logs (
+    type STRING,
+    time STRING,
+    elb STRING,
+    client_ip STRING,
+    client_port INT,
+    target_ip STRING,
+    target_port INT,
+    request_processing_time DOUBLE,
+    target_processing_time DOUBLE,
+    response_processing_time DOUBLE,
+    elb_status_code INT,
+    target_status_code STRING,
+    received_bytes BIGINT,
+    sent_bytes BIGINT,
+    request_verb STRING,
+    request_url STRING,
+    request_proto STRING,
+    user_agent STRING,
+    ssl_cipher STRING,
+    ssl_protocol STRING,
+    target_group_arn STRING,
+    trace_id STRING,
+    domain_name STRING,
+    chosen_cert_arn STRING,
+    matched_rule_priority STRING,
+    request_creation_time STRING,
+    actions_executed STRING,
+    redirect_url STRING,
+    error_reason STRING,
+    target_port_list STRING,
+    target_status_code_list STRING,
+    classification STRING,
+    classification_reason STRING
+)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe'
+WITH SERDEPROPERTIES (
+    'serialization.format' = '1',
+    'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) ([^ ]*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"'
+)
+LOCATION 's3://openhistoricalmap-elb-logs/alb_production/AWSLogs/618380242247/elasticloadbalancing/us-east-1/';
+
+/* This query is used to find the bots in the alb logs. */
+SELECT
+    user_agent,
+    client_ip,
+    COUNT(*) AS request_count
+FROM
+    alb_logs
+WHERE
+    from_iso8601_timestamp(time) >= (now() - interval '48' hour)
+    AND
+    (LOWER(user_agent) LIKE '%bot%' OR LOWER(user_agent) LIKE '%spider%' OR LOWER(user_agent) LIKE '%crawler%')
+GROUP BY
+    user_agent,
+    client_ip
+ORDER BY
+    request_count DESC
+LIMIT 50;