|
| 1 | +package com.openelements.content; |
| 2 | + |
| 3 | +import java.net.URI; |
| 4 | +import java.time.Duration; |
| 5 | +import java.util.ArrayList; |
| 6 | +import java.util.List; |
| 7 | +import java.util.Locale; |
| 8 | +import java.util.Map; |
| 9 | +import java.util.concurrent.ConcurrentHashMap; |
| 10 | +import java.util.function.LongSupplier; |
| 11 | +import org.slf4j.Logger; |
| 12 | +import org.slf4j.LoggerFactory; |
| 13 | +import org.springframework.beans.factory.annotation.Autowired; |
| 14 | +import org.springframework.stereotype.Component; |
| 15 | +import org.springframework.web.client.RestClient; |
| 16 | + |
| 17 | +/** |
| 18 | + * {@link RobotsPolicy} that fetches and caches each host's {@code robots.txt}. |
| 19 | + * |
| 20 | + * <p>Rules are fetched once per host (over HTTPS) and cached with a TTL. The group matching our bot |
| 21 | + * {@code User-Agent} is used, falling back to the {@code *} group; a missing or unreachable |
| 22 | + * {@code robots.txt} allows everything. Path matching is longest-prefix with {@code Allow} winning |
| 23 | + * ties (the common robots semantics). The clock is injectable so cache expiry is testable. |
| 24 | + */ |
| 25 | +@Component |
| 26 | +public class HttpRobotsPolicy implements RobotsPolicy { |
| 27 | + |
| 28 | + private static final Logger log = LoggerFactory.getLogger(HttpRobotsPolicy.class); |
| 29 | + private static final Duration CACHE_TTL = Duration.ofHours(1); |
| 30 | + |
| 31 | + private final RestClient restClient; |
| 32 | + private final String userAgent; |
| 33 | + private final LongSupplier clockNanos; |
| 34 | + private final Map<String, CachedRules> cache = new ConcurrentHashMap<>(); |
| 35 | + |
| 36 | + @Autowired |
| 37 | + public HttpRobotsPolicy(RestClient.Builder restClientBuilder, ContentSourceProperties properties) { |
| 38 | + this(restClientBuilder, properties, System::nanoTime); |
| 39 | + } |
| 40 | + |
| 41 | + HttpRobotsPolicy(RestClient.Builder restClientBuilder, ContentSourceProperties properties, LongSupplier clockNanos) { |
| 42 | + this.restClient = restClientBuilder.build(); |
| 43 | + this.userAgent = properties.userAgent(); |
| 44 | + this.clockNanos = clockNanos; |
| 45 | + } |
| 46 | + |
| 47 | + @Override |
| 48 | + public boolean isAllowed(String url) { |
| 49 | + String host = hostOf(url); |
| 50 | + if (host.isEmpty()) { |
| 51 | + return true; |
| 52 | + } |
| 53 | + boolean allowed = rulesFor(host).allows(pathOf(url)); |
| 54 | + if (!allowed) { |
| 55 | + log.info("robots.txt disallows {} for {}", url, userAgent); |
| 56 | + } |
| 57 | + return allowed; |
| 58 | + } |
| 59 | + |
| 60 | + @Override |
| 61 | + public Duration crawlDelay(String host) { |
| 62 | + return host == null || host.isEmpty() ? Duration.ZERO : rulesFor(host).crawlDelay(); |
| 63 | + } |
| 64 | + |
| 65 | + private RobotsRules rulesFor(String host) { |
| 66 | + CachedRules cached = cache.get(host); |
| 67 | + if (cached != null && cached.expiryNanos() > clockNanos.getAsLong()) { |
| 68 | + return cached.rules(); |
| 69 | + } |
| 70 | + RobotsRules rules = fetchAndParse(host); |
| 71 | + cache.put(host, new CachedRules(rules, clockNanos.getAsLong() + CACHE_TTL.toNanos())); |
| 72 | + return rules; |
| 73 | + } |
| 74 | + |
| 75 | + private RobotsRules fetchAndParse(String host) { |
| 76 | + try { |
| 77 | + String body = restClient.get() |
| 78 | + .uri(URI.create("https://" + host + "/robots.txt")) |
| 79 | + .header("User-Agent", userAgent) |
| 80 | + .retrieve() |
| 81 | + .body(String.class); |
| 82 | + return parse(body == null ? "" : body, userAgent); |
| 83 | + } catch (Exception e) { |
| 84 | + // Missing/unreachable robots.txt (e.g. 404) means allow everything. |
| 85 | + log.debug("No usable robots.txt for {} ({}); allowing all", host, e.toString()); |
| 86 | + return RobotsRules.ALLOW_ALL; |
| 87 | + } |
| 88 | + } |
| 89 | + |
| 90 | + /** Parses robots.txt, selecting the group for our user agent (falling back to {@code *}). */ |
| 91 | + static RobotsRules parse(String body, String userAgent) { |
| 92 | + List<Group> groups = new ArrayList<>(); |
| 93 | + Group current = null; |
| 94 | + boolean lastWasRule = false; |
| 95 | + for (String rawLine : body.split("\n")) { |
| 96 | + String line = stripComment(rawLine).trim(); |
| 97 | + if (line.isEmpty()) { |
| 98 | + continue; |
| 99 | + } |
| 100 | + int colon = line.indexOf(':'); |
| 101 | + if (colon < 0) { |
| 102 | + continue; |
| 103 | + } |
| 104 | + String field = line.substring(0, colon).trim().toLowerCase(Locale.ROOT); |
| 105 | + String value = line.substring(colon + 1).trim(); |
| 106 | + if (field.equals("user-agent")) { |
| 107 | + if (current == null || lastWasRule) { |
| 108 | + current = new Group(); |
| 109 | + groups.add(current); |
| 110 | + } |
| 111 | + current.agents.add(value.toLowerCase(Locale.ROOT)); |
| 112 | + lastWasRule = false; |
| 113 | + } else if (current != null) { |
| 114 | + switch (field) { |
| 115 | + case "disallow" -> current.disallow.add(value); |
| 116 | + case "allow" -> current.allow.add(value); |
| 117 | + case "crawl-delay" -> current.crawlDelay = parseDelay(value); |
| 118 | + default -> { /* ignore other directives */ } |
| 119 | + } |
| 120 | + lastWasRule = true; |
| 121 | + } |
| 122 | + } |
| 123 | + return selectGroup(groups, userAgent).toRules(); |
| 124 | + } |
| 125 | + |
| 126 | + private static Group selectGroup(List<Group> groups, String userAgent) { |
| 127 | + String ua = userAgent.toLowerCase(Locale.ROOT); |
| 128 | + Group wildcard = null; |
| 129 | + for (Group group : groups) { |
| 130 | + for (String agent : group.agents) { |
| 131 | + if (agent.equals("*")) { |
| 132 | + wildcard = group; |
| 133 | + } else if (!agent.isEmpty() && ua.contains(agent)) { |
| 134 | + return group; // a group naming our agent wins over the wildcard |
| 135 | + } |
| 136 | + } |
| 137 | + } |
| 138 | + return wildcard == null ? new Group() : wildcard; |
| 139 | + } |
| 140 | + |
| 141 | + private static Duration parseDelay(String value) { |
| 142 | + try { |
| 143 | + return Duration.ofMillis(Math.round(Double.parseDouble(value) * 1000)); |
| 144 | + } catch (NumberFormatException e) { |
| 145 | + return Duration.ZERO; |
| 146 | + } |
| 147 | + } |
| 148 | + |
| 149 | + private static String stripComment(String line) { |
| 150 | + int hash = line.indexOf('#'); |
| 151 | + return hash >= 0 ? line.substring(0, hash) : line; |
| 152 | + } |
| 153 | + |
| 154 | + private static String hostOf(String url) { |
| 155 | + try { |
| 156 | + String host = URI.create(url).getHost(); |
| 157 | + return host == null ? "" : host; |
| 158 | + } catch (IllegalArgumentException e) { |
| 159 | + return ""; |
| 160 | + } |
| 161 | + } |
| 162 | + |
| 163 | + private static String pathOf(String url) { |
| 164 | + try { |
| 165 | + String path = URI.create(url).getPath(); |
| 166 | + return path == null || path.isEmpty() ? "/" : path; |
| 167 | + } catch (IllegalArgumentException e) { |
| 168 | + return "/"; |
| 169 | + } |
| 170 | + } |
| 171 | + |
| 172 | + private static final class Group { |
| 173 | + private final List<String> agents = new ArrayList<>(); |
| 174 | + private final List<String> disallow = new ArrayList<>(); |
| 175 | + private final List<String> allow = new ArrayList<>(); |
| 176 | + private Duration crawlDelay = Duration.ZERO; |
| 177 | + |
| 178 | + private RobotsRules toRules() { |
| 179 | + return new RobotsRules(List.copyOf(disallow), List.copyOf(allow), crawlDelay); |
| 180 | + } |
| 181 | + } |
| 182 | + |
| 183 | + private record CachedRules(RobotsRules rules, long expiryNanos) { |
| 184 | + } |
| 185 | + |
| 186 | + /** Parsed robots rules for the applicable group. */ |
| 187 | + record RobotsRules(List<String> disallow, List<String> allow, Duration crawlDelay) { |
| 188 | + |
| 189 | + static final RobotsRules ALLOW_ALL = new RobotsRules(List.of(), List.of(), Duration.ZERO); |
| 190 | + |
| 191 | + boolean allows(String path) { |
| 192 | + int longestDisallow = longestMatch(disallow, path); |
| 193 | + if (longestDisallow < 0) { |
| 194 | + return true; |
| 195 | + } |
| 196 | + return longestMatch(allow, path) >= longestDisallow; // Allow wins ties |
| 197 | + } |
| 198 | + |
| 199 | + private static int longestMatch(List<String> rules, String path) { |
| 200 | + int longest = -1; |
| 201 | + for (String rule : rules) { |
| 202 | + if (!rule.isEmpty() && path.startsWith(rule) && rule.length() > longest) { |
| 203 | + longest = rule.length(); |
| 204 | + } |
| 205 | + } |
| 206 | + return longest; |
| 207 | + } |
| 208 | + } |
| 209 | +} |
0 commit comments