#!/bin/bash
set -euo pipefail
SET=google_crawlers
TMP=/tmp/google_crawlers_ranges.txt
URLS=(
  "https://developers.google.com/search/apis/ipranges/googlebot.json"
  "https://developers.google.com/search/apis/ipranges/special-crawlers.json"
)
python3 - <<"PY" > "$TMP"
import urllib.request,json
urls=[
 "https://developers.google.com/search/apis/ipranges/googlebot.json",
 "https://developers.google.com/search/apis/ipranges/special-crawlers.json",
]
seen=set()
for u in urls:
 data=json.load(urllib.request.urlopen(u,timeout=30))
 for p in data.get("prefixes",[]):
  v=p.get("ipv4Prefix")
  if v and v not in seen:
   seen.add(v); print(v)
PY
if ! command -v ipset >/dev/null 2>&1; then
  echo "ipset not found" >&2
  exit 1
fi
ipset create "$SET" hash:net family inet -exist
# flush and reload official Google bot/special-crawler ranges
ipset flush "$SET"
while read -r net; do [ -n "$net" ] && ipset add "$SET" "$net" -exist; done < "$TMP"
# Put priority allow on top of INPUT
while iptables -C INPUT -m set --match-set "$SET" src -j ACCEPT 2>/dev/null; do
  iptables -D INPUT -m set --match-set "$SET" src -j ACCEPT || true
done
iptables -I INPUT 1 -m set --match-set "$SET" src -j ACCEPT
# save where possible
ipset save > /etc/ipset.conf 2>/dev/null || true
iptables-save > /etc/iptables.rules 2>/dev/null || true
if command -v netfilter-persistent >/dev/null 2>&1; then netfilter-persistent save || true; fi
rm -f "$TMP"
echo "OK $(ipset list "$SET" | awk /Number of entries:/ {print }) ranges loaded"