From 04fc112b04140bab776152051867a36496112182 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 8 Oct 2025 16:11:27 -0500 Subject: [PATCH 1/6] Add config to update block ipsets --- .github/workflows/update-waf-firewall.yml | 108 ++++++++++++++++++++++ firewall/ip-blacklist-production.txt | 2 + firewall/ip-blacklist-staging.txt | 5 + 3 files changed, 115 insertions(+) create mode 100644 .github/workflows/update-waf-firewall.yml create mode 100644 firewall/ip-blacklist-production.txt create mode 100644 firewall/ip-blacklist-staging.txt diff --git a/.github/workflows/update-waf-firewall.yml b/.github/workflows/update-waf-firewall.yml new file mode 100644 index 00000000..c50014d4 --- /dev/null +++ b/.github/workflows/update-waf-firewall.yml @@ -0,0 +1,108 @@ +name: Update WAF IP blacklists + +on: + push: + branches: + - 'main' + - 'staging' + - 'waf_block_ip' + +permissions: + id-token: write + contents: read + +env: + AWS_REGION: us-east-1 + SCOPE: REGIONAL + IPSET_NAME_STAGING: ipset-block-ohm-staging + IPSET_NAME_PROD: ipset-block-ohm-production + FILE_STAGING: firewall/ip-blacklist-staging.txt + FILE_PROD: firewall/ip-blacklist-production.txt + +jobs: + update-waf-ipset: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Configure AWS credentials from secrets (access keys) + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Resolve target env, IP set and file + id: target + run: | + ENV_IN="${{ inputs.environment }}" + if [[ "$ENV_IN" == "main" ]]; then + echo "IPSET_NAME=${IPSET_NAME_PROD}" >> $GITHUB_OUTPUT + echo "IP_FILE=${FILE_PROD}" >> $GITHUB_OUTPUT + else + echo "IPSET_NAME=${IPSET_NAME_STAGING}" >> $GITHUB_OUTPUT + echo "IP_FILE=${FILE_STAGING}" >> $GITHUB_OUTPUT + fi + + - name: Build IP list + id: iplist + shell: bash + run: | + TMP=$(mktemp) + if [[ -n "${{ inputs.ips }}" ]]; then + # Convert space/newline-separated input to one-per-line + echo "${{ inputs.ips }}" | tr ' ' '\n' | sed '/^\s*$/d' > "$TMP" + else + FILE="${{ steps.target.outputs.IP_FILE }}" + if [[ ! -f "$FILE" ]]; then + echo "File $FILE not found" >&2 + exit 1 + fi + # Remove comments and blank lines + sed '/^\s*#/d;/^\s*$/d' "$FILE" > "$TMP" + fi + + # Basic validation for IPv4/IPv6 with optional CIDR + INVALID=$(grep -Ev '^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$|^([0-9a-fA-F:]+)(/[0-9]{1,3})?$' "$TMP" || true) + if [[ -n "$INVALID" ]]; then + echo "Invalid entries:"; echo "$INVALID"; exit 1 + fi + + # Unique and sorted list + sort -u "$TMP" > "${TMP}.uniq" + LIST=$(paste -sd' ' "${TMP}.uniq") + echo "addresses=$LIST" >> $GITHUB_OUTPUT + echo "Addresses to apply:"; cat "${TMP}.uniq" + + - name: Get IP set Id and LockToken + id: getipset + run: | + NAME="${{ steps.target.outputs.IPSET_NAME }}" + DATA=$(aws wafv2 list-ip-sets --scope $SCOPE --region $AWS_REGION --query "IPSets[?Name=='${NAME}'].[Id,ARN]" --output json) + if [[ "$DATA" == "[]" ]]; then + echo "IP set ${NAME} not found in ${AWS_REGION}" >&2; exit 1 + fi + ID=$(echo "$DATA" | jq -r '.[0][0]') + LOCK=$(aws wafv2 get-ip-set --scope $SCOPE --region $AWS_REGION --id "$ID" --name "$NAME" --query "LockToken" --output text) + echo "IPSET_ID=$ID" >> $GITHUB_OUTPUT + echo "LOCK_TOKEN=$LOCK" >> $GITHUB_OUTPUT + + - name: Update IP set (replace full list) + run: | + aws wafv2 update-ip-set \ + --scope $SCOPE \ + --region $AWS_REGION \ + --id "${{ steps.getipset.outputs.IPSET_ID }}" \ + --name "${{ steps.target.outputs.IPSET_NAME }}" \ + --lock-token "${{ steps.getipset.outputs.LOCK_TOKEN }}" \ + --addresses ${{ steps.iplist.outputs.addresses }} + + - name: Summary + run: | + echo "Updated IP set: ${{ steps.target.outputs.IPSET_NAME }}" diff --git a/firewall/ip-blacklist-production.txt b/firewall/ip-blacklist-production.txt new file mode 100644 index 00000000..dd029078 --- /dev/null +++ b/firewall/ip-blacklist-production.txt @@ -0,0 +1,2 @@ + +.github/workflows/update-waf-firewall.yml \ No newline at end of file diff --git a/firewall/ip-blacklist-staging.txt b/firewall/ip-blacklist-staging.txt new file mode 100644 index 00000000..f62d9a1d --- /dev/null +++ b/firewall/ip-blacklist-staging.txt @@ -0,0 +1,5 @@ + +# IPv4 single IP +203.0.113.45/32 +# IPv4 range +198.51.100.0/24 \ No newline at end of file From 08528c4c4d356e4ad42aa6298684d3e203e38a79 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 8 Oct 2025 16:26:58 -0500 Subject: [PATCH 2/6] Test block my ip --- firewall/ip-blacklist-staging.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/firewall/ip-blacklist-staging.txt b/firewall/ip-blacklist-staging.txt index f62d9a1d..d3810595 100644 --- a/firewall/ip-blacklist-staging.txt +++ b/firewall/ip-blacklist-staging.txt @@ -1,5 +1,8 @@ # IPv4 single IP -203.0.113.45/32 +200.60.4.59/32 +200.37.252.67/32 +200.10.69.30/32 + # IPv4 range -198.51.100.0/24 \ No newline at end of file +# 198.51.100.0/24 From 94ca25ff84fd69adae1716a8119098cfa84b94f7 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 9 Oct 2025 13:06:23 -0500 Subject: [PATCH 3/6] Add script to get ips block --- .github/workflows/update-waf-firewall.yml | 42 ++++++++-------- firewall/README.md | 4 ++ firewall/ip-blacklist-production.txt | 2 - firewall/ip-blacklist-production.yaml | 5 ++ firewall/ip-blacklist-staging.txt | 8 --- firewall/ip-blacklist-staging.yaml | 5 ++ firewall/queries/boots.sql | 61 +++++++++++++++++++++++ 7 files changed, 95 insertions(+), 32 deletions(-) create mode 100644 firewall/README.md delete mode 100644 firewall/ip-blacklist-production.txt create mode 100644 firewall/ip-blacklist-production.yaml delete mode 100644 firewall/ip-blacklist-staging.txt create mode 100644 firewall/ip-blacklist-staging.yaml create mode 100644 firewall/queries/boots.sql diff --git a/.github/workflows/update-waf-firewall.yml b/.github/workflows/update-waf-firewall.yml index c50014d4..5f59c00f 100644 --- a/.github/workflows/update-waf-firewall.yml +++ b/.github/workflows/update-waf-firewall.yml @@ -16,8 +16,9 @@ env: SCOPE: REGIONAL IPSET_NAME_STAGING: ipset-block-ohm-staging IPSET_NAME_PROD: ipset-block-ohm-production - FILE_STAGING: firewall/ip-blacklist-staging.txt - FILE_PROD: firewall/ip-blacklist-production.txt + # --- Archivos actualizados a .yaml --- + FILE_STAGING: firewall/ip-blacklist-staging.yaml + FILE_PROD: firewall/ip-blacklist-production.yaml jobs: update-waf-ipset: @@ -27,7 +28,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - # Configure AWS credentials from secrets (access keys) - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -35,14 +35,17 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.AWS_REGION }} - - name: Install jq - run: sudo apt-get update && sudo apt-get install -y jq + - name: Install jq and yq + run: | + sudo apt-get update && sudo apt-get install -y jq + sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq - name: Resolve target env, IP set and file id: target run: | - ENV_IN="${{ inputs.environment }}" - if [[ "$ENV_IN" == "main" ]]; then + # This logic remains the same, but will now point to .yaml files + if [[ "${{ github.ref_name }}" == "main" ]]; then echo "IPSET_NAME=${IPSET_NAME_PROD}" >> $GITHUB_OUTPUT echo "IP_FILE=${FILE_PROD}" >> $GITHUB_OUTPUT else @@ -50,31 +53,26 @@ jobs: echo "IP_FILE=${FILE_STAGING}" >> $GITHUB_OUTPUT fi - - name: Build IP list + - name: Build IP list from YAML id: iplist shell: bash run: | TMP=$(mktemp) - if [[ -n "${{ inputs.ips }}" ]]; then - # Convert space/newline-separated input to one-per-line - echo "${{ inputs.ips }}" | tr ' ' '\n' | sed '/^\s*$/d' > "$TMP" - else - FILE="${{ steps.target.outputs.IP_FILE }}" - if [[ ! -f "$FILE" ]]; then - echo "File $FILE not found" >&2 - exit 1 - fi - # Remove comments and blank lines - sed '/^\s*#/d;/^\s*$/d' "$FILE" > "$TMP" + FILE="${{ steps.target.outputs.IP_FILE }}" + if [[ ! -f "$FILE" ]]; then + echo "File $FILE not found" >&2 + exit 1 fi + # --- Cambio principal: Usar yq para leer el YAML --- + # Extrae cada IP de la lista 'block_ips' y la pone en una nueva línea + yq '.block_ips[]' "$FILE" > "$TMP" - # Basic validation for IPv4/IPv6 with optional CIDR + # La validación y el resto del script no necesitan cambios INVALID=$(grep -Ev '^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$|^([0-9a-fA-F:]+)(/[0-9]{1,3})?$' "$TMP" || true) if [[ -n "$INVALID" ]]; then echo "Invalid entries:"; echo "$INVALID"; exit 1 fi - # Unique and sorted list sort -u "$TMP" > "${TMP}.uniq" LIST=$(paste -sd' ' "${TMP}.uniq") echo "addresses=$LIST" >> $GITHUB_OUTPUT @@ -105,4 +103,4 @@ jobs: - name: Summary run: | - echo "Updated IP set: ${{ steps.target.outputs.IPSET_NAME }}" + echo "Updated IP set: ${{ steps.target.outputs.IPSET_NAME }}" \ No newline at end of file diff --git a/firewall/README.md b/firewall/README.md new file mode 100644 index 00000000..e8fc9786 --- /dev/null +++ b/firewall/README.md @@ -0,0 +1,4 @@ +## Blocking High-Probability Bot IPs Based on Traffic Patterns + +We’re blocking IPs with a high probability of being bots. Analysis shows these IPs generated excessive traffic on the site, following clear bot-like patterns. + diff --git a/firewall/ip-blacklist-production.txt b/firewall/ip-blacklist-production.txt deleted file mode 100644 index dd029078..00000000 --- a/firewall/ip-blacklist-production.txt +++ /dev/null @@ -1,2 +0,0 @@ - -.github/workflows/update-waf-firewall.yml \ No newline at end of file diff --git a/firewall/ip-blacklist-production.yaml b/firewall/ip-blacklist-production.yaml new file mode 100644 index 00000000..1b50b13d --- /dev/null +++ b/firewall/ip-blacklist-production.yaml @@ -0,0 +1,5 @@ +block_ips: + ## Test Rub21 ips + 200.60.4.59/32 + 200.37.252.67/32 + 200.10.69.30/32 \ No newline at end of file diff --git a/firewall/ip-blacklist-staging.txt b/firewall/ip-blacklist-staging.txt deleted file mode 100644 index d3810595..00000000 --- a/firewall/ip-blacklist-staging.txt +++ /dev/null @@ -1,8 +0,0 @@ - -# IPv4 single IP -200.60.4.59/32 -200.37.252.67/32 -200.10.69.30/32 - -# IPv4 range -# 198.51.100.0/24 diff --git a/firewall/ip-blacklist-staging.yaml b/firewall/ip-blacklist-staging.yaml new file mode 100644 index 00000000..bc31e948 --- /dev/null +++ b/firewall/ip-blacklist-staging.yaml @@ -0,0 +1,5 @@ +block_ips: + ## Test Rub21 ips + 200.60.4.59/32 + 200.37.252.67/32 + 200.10.69.30/32 \ No newline at end of file diff --git a/firewall/queries/boots.sql b/firewall/queries/boots.sql new file mode 100644 index 00000000..2f6b683c --- /dev/null +++ b/firewall/queries/boots.sql @@ -0,0 +1,61 @@ +/* This query is used to create the external table for the alb logs. */ + +CREATE EXTERNAL TABLE IF NOT EXISTS alb_logs ( + type STRING, + time STRING, + elb STRING, + client_ip STRING, + client_port INT, + target_ip STRING, + target_port INT, + request_processing_time DOUBLE, + target_processing_time DOUBLE, + response_processing_time DOUBLE, + elb_status_code INT, + target_status_code STRING, + received_bytes BIGINT, + sent_bytes BIGINT, + request_verb STRING, + request_url STRING, + request_proto STRING, + user_agent STRING, + ssl_cipher STRING, + ssl_protocol STRING, + target_group_arn STRING, + trace_id STRING, + domain_name STRING, + chosen_cert_arn STRING, + matched_rule_priority STRING, + request_creation_time STRING, + actions_executed STRING, + redirect_url STRING, + error_reason STRING, + target_port_list STRING, + target_status_code_list STRING, + classification STRING, + classification_reason STRING +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' +WITH SERDEPROPERTIES ( + 'serialization.format' = '1', + 'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) ([^ ]*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"' +) +LOCATION 's3://openhistoricalmap-elb-logs/alb_production/AWSLogs/618380242247/elasticloadbalancing/us-east-1/'; + +/* This query is used to find the bots in the alb logs. */ +SELECT + user_agent, + client_ip, + COUNT(*) AS request_count +FROM + alb_logs +WHERE + from_iso8601_timestamp(time) >= (now() - interval '48' hour) + AND + (LOWER(user_agent) LIKE '%bot%' OR LOWER(user_agent) LIKE '%spider%' OR LOWER(user_agent) LIKE '%crawler%') +GROUP BY + user_agent, + client_ip +ORDER BY + request_count DESC +LIMIT 50; From 0146bbdb88bb86f7e3227a03c2c33968219fe752 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 9 Oct 2025 13:37:12 -0500 Subject: [PATCH 4/6] Fix format of block ips --- firewall/ip-blacklist-production.yaml | 7 +++---- firewall/ip-blacklist-staging.yaml | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/firewall/ip-blacklist-production.yaml b/firewall/ip-blacklist-production.yaml index 1b50b13d..4ae8e71d 100644 --- a/firewall/ip-blacklist-production.yaml +++ b/firewall/ip-blacklist-production.yaml @@ -1,5 +1,4 @@ block_ips: - ## Test Rub21 ips - 200.60.4.59/32 - 200.37.252.67/32 - 200.10.69.30/32 \ No newline at end of file + - 200.60.4.59/32 + - 200.37.252.67/32 + - 200.10.69.30/32 \ No newline at end of file diff --git a/firewall/ip-blacklist-staging.yaml b/firewall/ip-blacklist-staging.yaml index bc31e948..6ef7c22a 100644 --- a/firewall/ip-blacklist-staging.yaml +++ b/firewall/ip-blacklist-staging.yaml @@ -1,5 +1,5 @@ block_ips: ## Test Rub21 ips - 200.60.4.59/32 - 200.37.252.67/32 - 200.10.69.30/32 \ No newline at end of file + - 200.60.4.59/32 + - 200.37.252.67/32 + - 200.10.69.30/32 \ No newline at end of file From f8b3e993935d53b210658c33f4fd4b1668b5c121 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 9 Oct 2025 14:37:20 -0500 Subject: [PATCH 5/6] Add ips to block access to staging --- firewall/ip-blacklist-staging.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/firewall/ip-blacklist-staging.yaml b/firewall/ip-blacklist-staging.yaml index 6ef7c22a..940a2c3b 100644 --- a/firewall/ip-blacklist-staging.yaml +++ b/firewall/ip-blacklist-staging.yaml @@ -1,5 +1,11 @@ block_ips: - ## Test Rub21 ips - - 200.60.4.59/32 - - 200.37.252.67/32 - - 200.10.69.30/32 \ No newline at end of file + # DataForSEO Bot + - 136.243.220.208/29 + - 136.243.228.176/29 + - 136.243.228.192/29 + # Bingbot (Microsoft) + - 157.55.39.0/24 + - 207.46.13.0/24 + # Thinkbot &&& APNIC (Asia-Pacific Network Information Centre), No need acces to staging + - 43.0.0.0/8 + \ No newline at end of file From f72a29aad92012a7d9453fa76b37c7cd357bf16c Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 9 Oct 2025 16:00:53 -0500 Subject: [PATCH 6/6] Block ips fro last atack --- firewall/ip-blacklist-production.yaml | 48 +++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/firewall/ip-blacklist-production.yaml b/firewall/ip-blacklist-production.yaml index 4ae8e71d..2f4c23b7 100644 --- a/firewall/ip-blacklist-production.yaml +++ b/firewall/ip-blacklist-production.yaml @@ -1,4 +1,46 @@ block_ips: - - 200.60.4.59/32 - - 200.37.252.67/32 - - 200.10.69.30/32 \ No newline at end of file + - 2.189.5.0/24 + - 31.46.225.0/24 + - 43.128.0.0/11 + - 46.151.194.0/24 + - 47.79.196.0/24 + - 47.79.218.0/23 + - 49.51.0.0/16 + - 101.32.0.0/15 + - 106.71.168.0/24 + - 119.28.0.0/16 + - 124.156.0.0/16 + - 129.226.0.0/16 + - 138.199.0.0/16 + - 143.244.49.0/24 + - 143.244.50.0/24 + - 143.244.56.0/24 + - 143.244.60.0/24 + - 150.109.0.0/16 + - 156.146.43.0/24 + - 156.146.56.0/24 + - 157.131.223.0/24 + - 160.238.138.0/24 + - 162.62.0.0/16 + - 162.198.71.0/24 + - 169.150.207.0/24 + - 169.150.220.0/24 + - 169.150.236.0/24 + - 169.150.247.0/24 + - 169.150.249.0/24 + - 170.106.0.0/16 + - 172.56.13.0/24 + - 173.88.145.0/24 + - 180.191.169.0/24 + - 185.59.220.0/24 + - 185.93.1.0/24 + - 185.93.2.0/24 + - 185.111.111.0/24 + - 190.197.0.0/24 + - 192.184.146.0/24 + - 195.91.2.0/24 + - 195.181.163.0/24 + - 205.194.32.0/24 + - 209.184.121.0/24 + - 212.102.40.0/24 + - 213.136.70.0/24 \ No newline at end of file