We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 15f47a9 commit 54d0e21Copy full SHA for 54d0e21
1 file changed
core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
@@ -315,7 +315,8 @@ public void execute(Tuple tuple) {
315
// nofollow
316
String[] relkeywords = link.attr("rel").split(" ");
317
boolean noFollow =
318
- Stream.of(relkeywords).anyMatch(x -> x.equalsIgnoreCase("nofollow"));
+ Stream.of(relkeywords)
319
+ .anyMatch(x -> x.equalsIgnoreCase("nofollow"));
320
321
// remove altogether
322
if (noFollow && robotsNoFollowStrict) {
0 commit comments