From eee01170fdfcc4f5208f0f20aa6a8752c8c0fd7f Mon Sep 17 00:00:00 2001 From: Mark Davis Date: Sat, 27 Jun 2026 12:09:20 +0200 Subject: [PATCH 1/8] ICU-23431 Add link detection/formatting tools Very rough first cut --- .../icu/impl/links/LinkHandlingUtilities.java | 1130 +++++++++++++++++ .../com/ibm/icu/impl/locale/XCldrStub.java | 42 + .../java/com/ibm/icu/util/LinkUtilities.java | 99 ++ .../com/ibm/icu/dev/test/links/LinkTest.java | 53 + .../data/testdata/links/LinkDetectionTest.txt | 432 +++++++ .../testdata/links/LinkFormattingTest.txt | 278 ++++ 6 files changed, 2034 insertions(+) create mode 100644 icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java create mode 100644 icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java create mode 100644 icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java create mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkDetectionTest.txt create mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkFormattingTest.txt diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java new file mode 100644 index 000000000000..e99420751518 --- /dev/null +++ b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java @@ -0,0 +1,1130 @@ +package com.ibm.icu.impl.links; + +import com.ibm.icu.impl.IDNA2003; +import com.ibm.icu.impl.UnicodeMap; +import com.ibm.icu.impl.Utility; +import com.ibm.icu.impl.locale.XCldrStub; +import com.ibm.icu.impl.locale.XCldrStub.Splitter; +import com.ibm.icu.lang.UCharacter; +import com.ibm.icu.lang.UProperty; +import com.ibm.icu.lang.UProperty.NameChoice; +import com.ibm.icu.text.StringPrepParseException; +import com.ibm.icu.text.UnicodeSet; +import com.ibm.icu.text.UnicodeSet.EntryRange; +import com.ibm.icu.text.UnicodeSet.SpanCondition; +import com.ibm.icu.util.ICUException; +import com.ibm.icu.util.Output; +import java.nio.charset.StandardCharsets; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.Set; +import java.util.Stack; +import java.util.function.Function; +import java.util.regex.MatchResult; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class LinkHandlingUtilities { + + private static final UnicodeSet SOFAR = new UnicodeSet(); + + /** + * Based on https://url.spec.whatwg.org/#percent-encoded-bytes. Is the default for maximal + * encoding, but can be customized + */ + private enum WHATWG_PERCENT_ENCODED { + /** C0 controls and all code points greater than U+007E (~). */ + C0(null, "[\\u0000-\\u001F\\u007E-\\x{10FFFF}]"), + + /** C0 and U+0020 SPACE, U+0022 ("), U+003C (<), U+003E (>), andU+0060 (`). */ + FRAGMENT(C0, "[\\ \"<>`]"), + + /** C0 and U+0020 SPACE, U+0022 ("), U+0023 (#), U+003C (<), and U+003E (>) */ + QUERY(C0, "[\\ \"<>]"), + + /** query percent-encode set and U+0027 ('). Used for http://... */ + SPECIAL_QUERY(QUERY, "[']"), + + /** + * query percent-encode set and
+ * U+003F (?), U+005E (^), U+0060 (`), U+007B ({), and U+007D (}). + */ + PATH(QUERY, "[?\\^`\\{\\}]"), + + /** + * path and
+ * U+002F (/), U+003A (:), U+003B (;), U+003D (=), U+0040 (@), U+005B ([) to U+005D (]), + * inclusive, and U+007C (|). + */ + USERINFO(PATH, "[/:;=@\\[-\\]|]"), + + /** + * userinfo and
+ * U+0024 ($) to U+0026 (&), inclusive, U+002B (+), and U+002C (,). + */ + COMPONENT(USERINFO, "[\\$-\\&+,]"); + + public final UnicodeSet set; + + private WHATWG_PERCENT_ENCODED(WHATWG_PERCENT_ENCODED base, String uset) { + set = new UnicodeSet(uset).addAll(base == null ? UnicodeSet.EMPTY : base.set).freeze(); + } + } + + private static String quote(String s) { + return s; // later, escape for HTML + } + + /** TODO use real property */ + private static int getOpening(int cp) { + return cp == '>' ? '<' : UCharacter.getIntPropertyValue(cp, UProperty.BIDI_PAIRED_BRACKET); + } + + /** Defines the LinkTermination property TODO use real property */ + private enum LinkTermination { + Hard("[\\p{whitespace}\\p{NChar}[\\p{C}-\\p{Cf}]\\p{deprecated}]"), + Soft("[\\p{Term}\\p{lb=qu}-\\p{deprecated}]"), + Close("[\\p{Bidi_Paired_Bracket_Type=Close}[>]-\\p{deprecated}]"), + Open("[\\p{Bidi_Paired_Bracket_Type=Open}[<]-\\p{deprecated}]"), + Include(null), // all else + ; + + public final UnicodeSet base; + + public UnicodeSet getBase() { + return base; + } + + private LinkTermination(String uset) { + if (uset == null) { // only called with Include, the "none of the above" option + this.base = SOFAR.complement().freeze(); + } else { + java.text.ParsePosition parsePosition = new java.text.ParsePosition(0); + this.base = new UnicodeSet(uset).freeze(); + SOFAR.addAll(this.base); + } + } + + private static final UnicodeMap PROPERTY_MAP = new UnicodeMap<>(); + + static { + // Verify consistency + for (LinkTermination pv1 : values()) { + for (LinkTermination pv2 : values()) { + if (pv1.compareTo(pv2) <= 0) { + continue; + } + if (pv1.base.containsSome(pv2.base)) { + throw new IllegalArgumentException( + "Values in LinkTermination overlap! " + + pv1 + + ", " + + pv2 + + ": " + + new UnicodeSet(pv1.base).retainAll(pv2.base)); + } + } + } + for (LinkTermination lt : values()) { + PROPERTY_MAP.putAll(lt.base, lt); + } + PROPERTY_MAP.freeze(); + } + + private static final Set NON_MISSING = + XCldrStub.setsDifference(EnumSet.allOf(LinkTermination.class), Set.of(Hard)); + } + + // https://www.rfc-editor.org/rfc/rfc5322.html#section-3.2.3 has the full list for ASCII part + // See also https://en.wikipedia.org/wiki/Email_address#Local-part + // We add dot (ascii '.'), and then check after for the special dot constraints. + + static final UnicodeSet EMAIL_ASCII_INCLUDES = + new UnicodeSet("[[a-zA-Z][0-9][_ \\- ! ? ' \\{ \\} * / \\& # % ` \\^ + = | ~ \\$]]") + .add('.') + .freeze(); + static final UnicodeSet validEmailLocalPart = + new UnicodeSet("[\\p{XID_Continue}-\\p{block=basic_latin}]") + .addAll(EMAIL_ASCII_INCLUDES) + .freeze(); + + public static int scanEmailBackwards(CharSequence source, int hardStart, int domainStart) { + if (UCharacter.codePointBefore(source, domainStart) != '@') { + throw new IllegalArgumentException("Scanning must start after an '@' sign"); + } + int result = validEmailLocalPart.spanBack(source, domainStart - 1, SpanCondition.SIMPLE); + return result >= hardStart ? result : domainStart - 1; + } + + private static String getGeneralCategory(int property, int codePoint, int nameChoice) { + return UCharacter.getPropertyValueName( + property, UCharacter.getIntPropertyValue(codePoint, property), nameChoice); + } + + private enum Structure { + single, + listP("/", "𝑷"), + listF(":~:", "𝑭"), // directive + listQ2("&", "𝑸", "=", "𝑽"); + public final String sub; + public final String sub2; + public final String prefix; + public final String prefix2; + + Structure() { + this(null, null, null, null); + } + + Structure(String sub, String prefix) { + this(sub, prefix, null, null); + } + + Structure(String sub, String prefix, String sub2, String prefix2) { + this.sub = sub; + this.sub2 = sub2; + this.prefix = prefix; + this.prefix2 = prefix2; + } + } + + /** Parallels the spec parts table */ + private enum Part { + // initiator, terminators, clearStack + PROTOCOL(Structure.single, '\u0000', "[{//}]", "[]", "[]", null), + HOST(Structure.single, '\u0000', "[/?#]", "[]", "[]", null), + PATH(Structure.listP, '/', "[?#]", "[/]", "[]", WHATWG_PERCENT_ENCODED.PATH), + QUERY(Structure.listQ2, '?', "[#]", "[=\\&]", "[+]", WHATWG_PERCENT_ENCODED.SPECIAL_QUERY), + FRAGMENT( + Structure.listF, + '#', + "[]", + "[{:~:}]", + "[]", + WHATWG_PERCENT_ENCODED.FRAGMENT), // the :~: is handled by code + // FRAGMENT_DIRECTIVE(Structure.listF, '#', "[]", "[{:~:}]", "[]"), // the :~: is handled by + // code + ; + + static final Set ALL = XCldrStub.SetofOrdered(values()); + + static final int[] FRAGMENT_DIRECTIVE_STRING = ":~:".codePoints().toArray(); + + public final Structure structure; + final int initiator; + final UnicodeSet terminators; + final UnicodeSet clearStack; + final UnicodeSet extraQuoted; + final UnicodeSet whatwg_quoted; + + private Part( + Structure structure, + char initiator, + String terminators, + String clearStack, + String extraQuoted, + WHATWG_PERCENT_ENCODED whatwg_quoted) { + this.structure = structure; + this.initiator = initiator; + this.terminators = new UnicodeSet(terminators).freeze(); + this.clearStack = new UnicodeSet(clearStack).freeze(); + this.extraQuoted = + new UnicodeSet(extraQuoted) + .addAll(this.clearStack) + .addAll(this.terminators) + .freeze(); + this.whatwg_quoted = whatwg_quoted == null ? UnicodeSet.EMPTY : whatwg_quoted.set; + } + + static Part fromInitiator(int cp) { + for (Part part : Part.values()) { + if (part.initiator == cp) { + return part; + } + } + return null; + } + + /** + * Unescape a part. But don't unescape interior characters or terminators because they are + * content! For example "a/b%2Fc" as a path should not be turned into a/b/c, because that + * b/c is a path-part. + * + * @param substring + * @return + */ + private String unescape(String substring) { + return LinkHandlingUtilities.unescape(substring, extraQuoted); + } + + private String fullEscape(List> source) { + if (source.isEmpty()) { + return ""; + } + if (structure == Structure.single) { + return source.get(0).get(0); + } + StringBuilder result = new StringBuilder().appendCodePoint(initiator); + UnicodeSet allEscape = whatwg_quoted; + // new UnicodeSet(0, 0x21, 0x7F, 0x10FFFF).addAll(extraQuoted).freeze(); + boolean atStart = true; + for (List list : source) { + if (atStart) { + atStart = false; + } else { + result.append(structure.sub); + } + if (structure.sub2 == null) { + result.append(escape(list.get(0), allEscape)); + } else { + boolean atStart2 = true; + for (String string : list) { + if (atStart2) { + atStart2 = false; + } else { + result.append(structure.sub2); + } + result.append(escape(string, allEscape)); + } + } + } + return result.toString(); + } + + /** Append an unescaped part, escaping as necessary */ + private void appendPart(StringBuilder toAppendTo, String partValue) { + if (!partValue.isBlank()) { + if (initiator != 0) { + toAppendTo.appendCodePoint(initiator); + } + toAppendTo.append(partValue); + } + } + } + + public static class UrlInternals { + private NavigableMap>> + data; // note for some parts, the lists only ever have 1 member + + /** + * Pull apart a URL string into Parts.
+ * TODO: unescape the %escapes. + * + * @param source + * @param unescape TODO + * @return + */ + public static UrlInternals from(String source) { + Map>> result = new EnumMap<>(Part.class); + // quick and dirty + int partStart = 0; + int partEnd; + main: + for (Part part : Part.values()) { + switch (part) { + case PROTOCOL: + partEnd = source.indexOf("://"); // TODO fix for mailto + if (partEnd > 0) { + partEnd += 3; + result.put( + Part.PROTOCOL, List.of(List.of(source.substring(0, partEnd)))); + partStart = partEnd; + } + break; + default: + partEnd = + part.terminators.span( + source, partStart, SpanCondition.NOT_CONTAINED); + if (partStart != partEnd) { + if (part != Part.HOST) { + ++partStart; + } + String partString = source.substring(partStart, partEnd); + if (part.structure.sub == null) { // only host, no unescaping + result.put(part, List.of(List.of(partString))); + } else if (part.structure.sub2 == null) { // we have just a list + List> cleanList = cleanList(part, partString); + result.put(part, cleanList); + } else { // we have a list of lists (query) + List> cleanList = + Splitter.on(part.structure.sub) + .splitToStream(partString) + .map(x -> splitList(part, part.structure.sub2, x)) + .collect(Collectors.toList()); + result.put(part, cleanList); + } + } + if (partEnd == source.length()) { + break main; + } + partStart = partEnd; + break; + } + } + return new UrlInternals(result); + } + + private UrlInternals(Map>> data) { + this.data = XCldrStub.immutableSortedMap(data); + } + + public enum EndStatus { + MEDIAL, + FINAL + } + + /** + * Minimally escape. Presumes that the parts use \ for interior quoting.
+ * + * @param endStatus TODO + * @param escapedCounter TODO + */ + public String minimalEscape(EndStatus endStatus) { + StringBuilder output = new StringBuilder(); + // get the last part + List>>> ordered = List.copyOf(data.entrySet()); + Part lastPart = null; + + for (Entry>> entry : ordered) { + if (!entry.getValue().isEmpty()) { + lastPart = entry.getKey(); + } + } + // process all parts + for (Entry>> partEntry : ordered) { + Part part = partEntry.getKey(); + final List> partParts = partEntry.getValue(); + if (partParts.isEmpty()) { + continue; + } + if (part.structure.sub == null) { + output.append(partParts.get(0).get(0)); // just copy + continue; + } + String unified = + joinListListEscaping(part.structure.sub, part.structure.sub2, partParts); + + int[] cps = unified.codePoints().toArray(); + int n = cps.length; + if (cps[0] != part.initiator) { + output.appendCodePoint(part.initiator); + } + ; + int copiedAlready = 0; + Stack openingStack = new Stack<>(); + for (int i = 0; i < n; ++i) { + final int cp = cps[i]; + switch (cp) { // was just for test files + case '\\': // if we have \ followed by x, just emit the literal x; otherwise + // \ + // This is ONLY used for our test files; + // in production the parts of the path/query + // would be handled separately. + + // append soft code points + appendCodePointsBetween(output, cps, copiedAlready, i); + if (i < n - 1) { + // append next code point + ++i; + appendPercentEscaped(output, cps[i]); + copiedAlready = i + 1; + } else { + // append '\' alone (at end) + appendPercentEscaped(output, cp); + copiedAlready = i + 1; + } + continue; + // + // case '%': // if we have %xy, and x and y are + // hex, escape the % + // if (i < n - 2 && HEX.contains(cps[i + 1]) + // && HEX.contains(cps[i + 2])) { + // // append soft code points + // appendCodePointsBetween(output, cps, + // copiedAlready, i); + // appendPercentEscaped(output, cp, + // escapedCounter); + // copiedAlready = i + 1; + // continue; + // } + // break; + } + LinkTermination lt = + part.terminators.contains(cp) + ? LinkTermination.Hard + : LinkTermination.PROPERTY_MAP.get(cp); + switch (lt) { + case Include: + appendCodePointsBetween(output, cps, copiedAlready, i); + output.appendCodePoint(cp); + copiedAlready = i + 1; + break; + case Hard: + appendCodePointsBetween(output, cps, copiedAlready, i); + appendPercentEscaped(output, cp); + copiedAlready = i + 1; + continue; + case Soft: // fix + continue; + case Open: + openingStack.push(cp); + appendCodePointsBetween(output, cps, copiedAlready, i); + output.appendCodePoint(cp); + copiedAlready = i + 1; + continue; // fix + case Close: // fix + if (openingStack.empty()) { + appendCodePointsBetween(output, cps, copiedAlready, i); + appendPercentEscaped(output, cp); + } else { + Integer topOfStack = openingStack.pop(); + int matchingOpening = getOpening(cp); + if (matchingOpening == topOfStack) { + appendCodePointsBetween(output, cps, copiedAlready, i); + output.appendCodePoint(cp); + } else { // failed to match + appendCodePointsBetween(output, cps, copiedAlready, i); + appendPercentEscaped(output, cp); + } + } + copiedAlready = i + 1; + continue; + default: + throw new IllegalArgumentException(); + } + } + if (endStatus == EndStatus.MEDIAL || part != lastPart) { + appendCodePointsBetween(output, cps, copiedAlready, n); + } else if (copiedAlready < n) { + appendCodePointsBetween(output, cps, copiedAlready, n - 1); + appendPercentEscaped(output, cps[n - 1]); + } + } + + return output.toString(); + } + + public String fullEscape() { + Output last = new Output<>(); + String result = + data.entrySet().stream() + .map( + x -> { + Part part = x.getKey(); + last.value = part; + List> value = x.getValue(); + return part.fullEscape(value); + }) + .collect(Collectors.joining()); + // handle the very last character, if soft + if (last.value != null && last.value != Part.HOST && last.value != Part.PROTOCOL) { + int lastCodePoint = result.codePointBefore(result.length()); + if (LinkTermination.Soft.base.contains(lastCodePoint)) { + // escape final soft code point + int indexBeforeLast = result.length() - Character.charCount(lastCodePoint); + StringBuilder resultBuilder = + new StringBuilder(result.substring(0, indexBeforeLast)); + appendPercentEscaped(resultBuilder, lastCodePoint); + result = resultBuilder.toString(); + } + } + return result; + } + + public List> get(Part part) { + return data.get(part); + } + + @Override + public String toString() { + return toString(Part.ALL); // show all + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof UrlInternals) { + return data.equals(((UrlInternals) obj).data); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(data); + } + + public String toString(Set partsToShow) { + StringBuilder result = new StringBuilder("{"); + for (Entry>> partListList : data.entrySet()) { + Part part = partListList.getKey(); + List> listList = partListList.getValue(); + int index1 = -1; + for (List list1 : listList) { + ++index1; + int index2 = -1; + for (String value : list1) { + ++index2; + if (result.length() != 1) { + result.append(" "); + } + switch (part.structure) { + case single: + // we don't care about interior structure + result.append(part == Part.PROTOCOL ? "𝑺" : "𝑯"); + break; + case listP: + result.append(part.structure.prefix); + break; + case listF: + // Hack the difference between Fragment and FragmentDirective for + // now + result.append(index1 == 0 ? part.structure.prefix : "𝑫"); + break; + case listQ2: + result.append( + index2 == 0 + ? part.structure.prefix + : part.structure.prefix2); + break; + } + result.append('='); + result.append(value); + } + } + } + return result.append("}").toString(); + } + } + + private static String joinListListEscaping( + String sep, String sep2, List> partParts) { + if (sep.codePointCount(0, sep.length()) == 0 + || (sep2 != null && sep2.codePointCount(0, sep2.length()) == 0)) { + throw new IllegalArgumentException(); + } + UnicodeSet toEscape = new UnicodeSet().add(sep); + if (sep2 != null) { + toEscape.add(sep2); + } + return partParts.stream() + .map( + x -> { + return joinListEscaping(sep2, toEscape, x); + }) + .collect(Collectors.joining(sep)); + } + + private static String joinListEscaping( + String sep, UnicodeSet toEscape, List partParts) { + return sep == null + ? escape(partParts.get(0), toEscape) + : partParts.stream() + .map( + x -> { + return escape(x, toEscape); + }) + .collect(Collectors.joining(sep)); + } + + private static List> cleanList(Part part, String partString) { + return splitList(part, part.structure.sub, partString).stream() + .map( + x -> { + return List.of(x); + }) + .collect(Collectors.toUnmodifiableList()); + } + + private static List splitList(Part part, String separator, String partString) { + return Splitter.on(separator) + .splitToStream(partString) + .map(x -> unescape(x)) + .collect(Collectors.toList()); + } + + // private static final UnicodeSet idnMapped = + // IUP.getSet("Idn_Status=" + Idn_Status_Values.mapped); + // + // private static final UnicodeSet idnValid = IUP.getSet("Idn_Status=" + + // Idn_Status_Values.valid); + // static final Pattern protocol = Pattern.compile("(https?://|mailto:)"); + // public static final UnicodeSet validHost = + // new UnicodeSet(idnValid) + // .addAll(idnMapped) + // .removeAll(new UnicodeSet("[:Block=Basic_Latin:]")) + // .addAll(new UnicodeSet("[-a-zA-Z0-9..。]")) // the three dots are + // different: + // .freeze(); + // public static final UnicodeSet validHostNoDot = + // new UnicodeSet(validHost).removeAll("..。").freeze(); + + /** Status of link found. Note that if the link is imputed, https or mailto will be returned. */ + enum LinkStatus { + bogus, + http, + https, + mailto, + tld + } + + /** Immutable class for returning the start/end of link that is found, plus some information */ + private static final class LinkFound { + public final int start; + public final int limit; + public final LinkStatus linkStatus; + + public LinkFound(int start, int limit, LinkStatus linkStatus) { + this.start = start; + this.limit = limit; + this.linkStatus = linkStatus; + } + + public String substring(String source) { + return source.substring(start, limit); + } + } + + // OLDER Code, leaving here for now, for comparison + // /** + // * Parses a restricted set of URLs, for testing the PathQueryFragment portion. That is, it + // is of + // * the form <host><domain_name>?
+ // * The host is currently just http, https, and mailto.
+ // * The domain_name is just approximated for testing. + // * + // * @return null if we run out of string, otherwise a LinkFound. If what is found is + // malformed in + // * some way, indicate with LinkFound.bogus. + // */ + // private static LinkFound parseLink(String source, int startCodePointOffset) { + // Matcher findStartMatcher = protocol.matcher(source); + // + // findStartMatcher.region(startCodePointOffset, source.length()); + // if (!findStartMatcher.find()) { + // return null; // we are at the end + // } + // // if we found something, and there was a character before it, + // // and that character was not soft, then exit + // int start = findStartMatcher.start(); + // int protocolEnd = findStartMatcher.end(); + // if (start != 0) { + // LinkTermination lt = + // LinkTermination.PROPERTY_MAP.get(UCharacter.codePointBefore(source, + // start)); + // if (lt == LinkTermination.Include) { + // return new LinkFound(start, protocolEnd, LinkStatus.bogus); + // } + // } + // + // String protocolValue = findStartMatcher.group(1); + // if (protocolValue.equals("mailto")) { + // return parseRestOfMailto(source, start, protocolEnd); + // } + // + // // dumb search for end of host, doesn't handle .. or edge cases, but this does not + // have to + // // be production-quality + // int hostLimit = findEndOfDomain(source, protocolEnd); + // if (protocolEnd == hostLimit) { + // return new LinkFound(start, protocolEnd, LinkStatus.bogus); + // } + // int limit = parsePathQueryFragment(source, hostLimit); + // return new LinkFound( + // start, limit, LinkStatus.valueOf(source.substring(start, protocolEnd))); + // } + // + // private static LinkFound parseRestOfMailto(String source, int start, int hostStart) { + // // basic implementation at first + // int atPosition = source.indexOf('@', hostStart); + // if (atPosition == -1) { + // return new LinkFound(start, hostStart, LinkStatus.bogus); + // } + // // TBD we could be in the middle of a quoted string, check for that later + // + // // see if what is in front of the @ looks ok + // + // int limit = parsePathQueryFragment(source, atPosition + 1); + // return new LinkFound(start, limit, LinkStatus.mailto); + // } + // + // // Simple implementation for testing, since the spec doesn't define the content + // private static int findEndOfDomain(String source, int protocolLimit) { + // return validHost.span(source, protocolLimit, SpanCondition.CONTAINED); + // } + + /** + * Set lastSafe to 0 — this marks the last code point that is definitely included in the + * linkification.
+ * Set closingStack to empty
+ * Set the current code point position i to 0
+ * Loop from i = 0 to n
+ * Set LT to LinkTermination(cp[i])
+ * If LT == none, set lastSafe to be i+1, continue loop
+ * If LT == soft, continue loop
+ * If LT == hard, stop linkification and return lastSafe
+ * If LT == opening, push cp[i] onto closingStack
+ * If LT == closing, set open to the pop of closingStack, or 0 if the closingStack is empty
+ * If LinkPairedOpeners(cp[i]) == open, set lastSafe to be i+1, continue loop.
+ * Otherwise, stop linkification and return lastSafe
+ * If lastSafe == n+1, then the entire part is safe; continue to the next part
+ * Otherwise, stop linkification and return lastSafe
+ */ + public static int parsePathQueryFragment(String source, int codePointOffset) { + // For simplicity, and to match the spec, we just get the code points + // Production code would be optimized, of course. + + int[] codePoints = source.codePoints().toArray(); + int lastSafe = codePointOffset; + Part part = null; + + Stack openingStack = new Stack<>(); + LinkTermination lt = LinkTermination.Soft; + for (int i = codePointOffset; i < codePoints.length; ++i) { + int cp = codePoints[i]; + if (part == null) { + part = Part.fromInitiator(cp); + if (part == null) { + return i; // failed, don't move cursor + } + lastSafe = i + 1; + continue; + } + + lt = LinkTermination.PROPERTY_MAP.get(cp); + // handle terminators and interior syntax + if (part.terminators.contains(cp)) { + lastSafe = i; + part = Part.fromInitiator(cp); + if (part == null) { + return lastSafe; + } + lastSafe = i + 1; + continue; + } else if (part.clearStack.contains(cp)) { + openingStack.clear(); + } else if (part == Part.FRAGMENT + && matches(codePoints, i, Part.FRAGMENT_DIRECTIVE_STRING)) { + // there is one string form, so hard-code it + openingStack.clear(); + } + switch (lt) { + case Include: + lastSafe = i + 1; + break; + case Soft: // no action + break; + case Hard: + return lastSafe; + case Open: + // We're not testing for stack limits; not needed for test data + openingStack.push(cp); + lastSafe = i + 1; + break; + case Close: + if (openingStack.empty()) { + return lastSafe; + } + int matchingOpening = getOpening(cp); + Integer topOfStack = openingStack.pop(); + if (matchingOpening == topOfStack) { + lastSafe = i + 1; + break; + } // else failed to match + return lastSafe; + } + } + // if we hit the end, it acts like we hit a hard character *** + return lt == LinkTermination.Soft ? lastSafe : codePoints.length; + } + + /** Simple utility for finding matching int array regions */ + private static boolean matches(int[] codePoints, int cpIndex, int[] fragmentDirective) { + if (cpIndex + fragmentDirective.length > codePoints.length) { + return false; + } + for (int i = 0; i < fragmentDirective.length; ++i) { + if (codePoints[i + cpIndex] != fragmentDirective[i]) { + return false; + } + } + return true; + } + + private static void appendCodePointsBetween( + StringBuilder output, int[] cp, int copyEnd, int notToCopy) { + for (int i = copyEnd; i < notToCopy; ++i) { + output.appendCodePoint(cp[i]); + } + } + + /** Regex for percent escaping */ + private static final Pattern ESCAPED_BYTE_PATTERN = + Pattern.compile("(%[a-fA-F0-9][a-fA-F0-9])+"); + + private static final Pattern ESCAPED_SINGLE = Pattern.compile("%[a-fA-F0-9][a-fA-F0-9]"); + + /** Unescape a string. */ + private static String unescape(String stringWithEscapes) { + return unescape(stringWithEscapes, UnicodeSet.EMPTY); // no characters escaped bak + } + + /** Unescape a string; however, code points in toEscape are escaped back. */ + private static String unescape(String stringWithEscapes, UnicodeSet toEscape) { + StringBuilder result = new StringBuilder(); + Matcher matcher = ESCAPED_BYTE_PATTERN.matcher(stringWithEscapes); + int current = 0; + while (matcher.find(current)) { + result.append( + stringWithEscapes.substring( + current, matcher.start())); // append intervening text + String unescaped = percentUnescape(matcher.group()); + unescaped + .chars() + .forEach( + x -> { + if (toEscape.contains(x)) { + // quote it + appendPercentEscaped(result, x); + } else { + result.appendCodePoint(x); + } + }); + current = matcher.end(); + } + result.append(stringWithEscapes.substring(current, stringWithEscapes.length())); + return result.toString(); + } + + private static void appendPercentEscaped(StringBuilder output, int cp) { + byte[] bytes = Character.toString(cp).getBytes(StandardCharsets.UTF_8); + for (int i = 0; i < bytes.length; ++i) { + output.append('%'); + output.append(Utility.hex(bytes[i], 2)); + } + } + + private static void appendPercentEscaped(StringBuilder output, String source) { + byte[] bytes = source.getBytes(StandardCharsets.UTF_8); + for (int i = 0; i < bytes.length; ++i) { + output.append('%'); + output.append(Utility.hex(bytes[i], 2)); + } + } + + /** percent-escape single char or string. */ + private static String escape(String source, String cpToEscape) { + return escape(source, new UnicodeSet().add(cpToEscape)); + } + + /** percent-escape all the toEscape characters. */ + private static String escape(String source, UnicodeSet toEscape) { + StringBuilder result = new StringBuilder(); + int lastEnd = 0; + while (true) { + int start = toEscape.span(source, lastEnd, SpanCondition.NOT_CONTAINED); + appendCheckingPercent(result, source, lastEnd, start); + if (start == source.length()) { + break; + } + lastEnd = toEscape.span(source, start, SpanCondition.SIMPLE); + appendPercentEscaped(result, source.substring(start, lastEnd)); + if (lastEnd == source.length()) { + break; + } + } + return result.toString(); + } + + /** If we have in an unescaped string %xy, and xy are hex, then we have to escape the % sign. */ + private static void appendCheckingPercent( + StringBuilder toAppendTo, String source, int lastEnd, int start) { + int pos = source.indexOf("%", lastEnd); // quickcheck + if (pos < 0 || pos >= start) { + toAppendTo.append(source, lastEnd, start); + return; + } + String sub = source.substring(lastEnd, start); + Matcher matcher = ESCAPED_SINGLE.matcher(sub); + Function replacer = + x -> { + return "%25" + + x.group(0) + .substring(1); // separate the % from the xy, and return %25xy + }; + String fixed = matcher.replaceAll(replacer); + toAppendTo.append(fixed); + } + + /** We are guaranteed that string is all percent escaped utf8, %a3%c0 ... */ + private static String percentUnescape(String escapedSource) { + byte[] temp = new byte[escapedSource.length() / 3]; + int tempOffset = 0; + for (int i = 0; i < escapedSource.length(); i += 3) { + if (escapedSource.charAt(i) != '%') { + throw new IllegalArgumentException(); + } + byte b = (byte) Integer.parseInt(escapedSource.substring(i + 1, i + 3), 16); + temp[tempOffset++] = b; + } + return new String(temp, StandardCharsets.UTF_8); + } + + /** + * The wikipedia languages are not all BCP47. Convert the ones that are not. See:
+ * https://meta.wikimedia.org/wiki/Special_language_codes
+ * https://meta.wikimedia.org/wiki/List_of_Wikipedias#Nonstandard_language_codes + * + * @param languageCode + * @return + */ + private static String fixWiki(String languageCode) { + switch (languageCode) { + case "als": + return "gsw"; + case "roa-rup": + return "rup"; + case "bat-smg": + return "sgs"; + case "simple": + return "en"; + case "fiu-vro": + return "vro"; + case "zh-classical": + return "lzh"; + case "zh-min-nan": + return "nan"; + case "zh-yue": + return "yue"; + case "cbk-zam": + return "cbk"; + case "map-bms": + return "map"; + case "nrm": + return "nrf"; + case "roa-tara": + return "nap"; + default: + return languageCode; + } + } + + /** Hard-coded list of wikilanguages, for testing */ + // private static Set WIKI_LANGUAGES = + // of( + // SPLIT_COMMA.splitToList( + // + // "en,ceb,de,fr,sv,nl,ru,es,it,pl,arz,zh,ja,uk,vi,war,ar,pt,fa,ca,id,sr,ko,no,tr,ce,fi,cs,hu,tt,ro,sh,eu,zh-min-nan,ms,he,eo,hy,da,bg,uz,cy,simple,sk,et,be,azb,el,kk,min,hr,lt,gl,ur,az,sl,lld,ka,nn,ta,th,hi,bn,mk,zh-yue,la,ast,lv,af,tg,my,te,sq,mr,mg,bs,oc,be-tarask,ku,br,sw,ml,nds,ky,lmo,jv,pnb,ckb,new,ht,vec,pms,lb,ba,su,ga,is,szl,cv,pa,fy,io,ha,tl,an,mzn,wuu,diq,vo,ig,yo,sco,kn,ne,als,gu,ia,avk,crh,bar,ban,scn,bpy,mn,qu,nv,si,xmf,frr,ps,os,or,tum,sd,bcl,bat-smg,sah,cdo,gd,bug,glk,yi,ilo,am,li,nap,gor,as,fo,mai,hsb,map-bms,shn,zh-classical,eml,ace,ie,wa,sa,hyw,sat,zu,sn,mhr,lij,hif,km,bjn,mrj,mni,dag,ary,hak,pam,rue,roa-tara,ug,zgh,bh,nso,co,tly,so,vls,nds-nl,mi,se,myv,rw,kaa,sc,bo,kw,vep,mt,tk,mdf,kab,gv,gan,fiu-vro,ff,zea,ab,skr,smn,ks,gn,frp,pcd,udm,kv,csb,ay,nrm,lo,ang,fur,olo,lfn,lez,ln,pap,nah,mwl,tw,stq,rm,ext,lad,gom,dty,av,tyv,koi,dsb,lg,cbk-zam,dv,ksh,za,bxr,blk,gag,pfl,bew,szy,haw,tay,pag,pi,awa,tcy,krc,inh,gpe,xh,kge,fon,atj,to,pdc,mnw,arc,shi,om,tn,dga,ki,nia,jam,kbp,wo,xal,nov,kbd,anp,nqo,bi,kg,roa-rup,tpi,tet,guw,jbo,mad,fj,lbe,kcg,pcm,cu,ty,trv,dtp,sm,ami,st,iba,srn,btm,alt,ltg,gcr,ny,kus,mos,ss,chr,ee,ts,got,bbc,gur,bm,pih,ve,rmy,fat,chy,rn,igl,ik,guc,ch,ady,pnt,iu,ann,rsk,pwn,dz,ti,sg,din,tdd,kl,bdr,nr,cr")); + + /** Show termination values, for generating property files */ + private void showLinkTermination() { + for (LinkTermination lt : LinkTermination.values()) { + UnicodeSet value = LinkTermination.PROPERTY_MAP.getSet(lt); + String name = lt.toString(); + System.out.println("\n#\tLink_Termination=" + name); + if (lt == LinkTermination.Include) { + System.out.println("# " + "(All code points without other values)"); + continue; + } else { + System.out.println("# draft = " + lt.base); + } + if (lt == LinkTermination.Hard) { + value.removeAll(new UnicodeSet("[\\p{Cn}\\p{Cs}]")); + System.out.println("# (not listing Unassigned or Surrogates)"); + } + System.out.println(); + for (EntryRange range : value.ranges()) { + final String rangeString = + Utility.hex(range.codepoint) + + (range.codepoint == range.codepointEnd + ? "" + : ".." + Utility.hex(range.codepointEnd)); + System.out.println( + rangeString + + ";" + + " ".repeat(15 - rangeString.length()) + + lt + + "\t# " + + "(" + + getGeneralCategory( + UProperty.GENERAL_CATEGORY, + range.codepoint, + NameChoice.SHORT) + + ") " + + quote(UCharacter.getExtendedName(range.codepoint)) + + (range.codepoint == range.codepointEnd + ? "" + : ".." + + "(" + + getGeneralCategory( + UProperty.GENERAL_CATEGORY, + range.codepointEnd, + NameChoice.SHORT) + + ") " + + quote( + UCharacter.getExtendedName( + range.codepointEnd)))); + } + System.out.println(); + } + } + + /** Show paired openers, for generating property files */ + private void showLinkPairedOpeners() { + UnicodeSet value = LinkTermination.PROPERTY_MAP.getSet(LinkTermination.Close); + + System.out.println("\n#\tLink_Paired_Opener"); + System.out.println( + "# draft = BidiPairedBracket + (“>” GREATER-THAN SIGN 🡆 “<” LESS-THAN SIGN)"); + System.out.println(); + + for (String cpString : value) { + int cp = cpString.codePointAt(0); + String hex = Utility.hex(cp); + final int value2 = getOpening(cp); + System.out.println( + hex + + ";" + + " ".repeat(7 - hex.length()) + + Utility.hex(value2) + + "\t#" + + " “" + + quote(Character.toString(cp)) + + "” " + + UCharacter.getExtendedName(cp) + + " 🡆 " + + " “" + + quote(Character.toString(value2)) + + "” " + + UCharacter.getExtendedName(value2)); + } + } + + /** + * Regex to scan for possible TLDs. The result needs to be checked that there is no + * validHostNoDot before and after + */ + // private static final Pattern TLD_SCANNER; + // + // private static final SortedSet TLDS; + // + private static final String DOTSET_STRING = "[.。]"; + + private static final UnicodeSet DOTSET = new UnicodeSet("[.。]").freeze(); + private static final Splitter SPLIT_LABELS = Splitter.on(Pattern.compile("[.。]")); + + private static String toUnicode2(String x) { + try { + if (x.startsWith("XN--")) { + x = IDNA2003.convertIDNToUnicode(x, 0).toString(); + } + return x; + } catch (StringPrepParseException e) { + throw new ICUException(e); + } + } +} diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/impl/locale/XCldrStub.java b/icu4j/main/core/src/main/java/com/ibm/icu/impl/locale/XCldrStub.java index 527be9217823..c57c6228ef7d 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/impl/locale/XCldrStub.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/impl/locale/XCldrStub.java @@ -13,6 +13,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -21,11 +22,14 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** Stub class to make migration easier until we get either Guava or a higher level of Java. */ public class XCldrStub { @@ -314,6 +318,10 @@ public Splitter(char c) { this(Pattern.compile("\\Q" + c + "\\E")); } + public Splitter(String s) { + this(Pattern.compile("\\Q" + s + "\\E")); + } + public Splitter(Pattern p) { pattern = p; } @@ -322,6 +330,10 @@ public static Splitter on(char c) { return new Splitter(c); } + public static Splitter on(String sub) { + return new Splitter(sub); + } + public static Splitter on(Pattern p) { return new Splitter(p); } @@ -336,6 +348,10 @@ public List splitToList(String input) { return Arrays.asList(items); } + public Stream splitToStream(String input) { + return splitToList(input).stream(); + } + public Splitter trimResults() { trimResults = true; return this; @@ -458,4 +474,30 @@ public interface Predicate { */ boolean test(T t); } + + /** Utilities because regular java doesn't have some nice Guava features */ + public static Set SetofOrdered(T... source) { + return Arrays.stream(source) + .collect( + Collectors.collectingAndThen( + Collectors.toCollection(LinkedHashSet::new), Set::copyOf)); + } + + /** Not as good as Guava, since it makes a new set instead of a view */ + public static Set setsDifference(Set set1, Set set2) { + return set1.stream() + .filter(element -> !set2.contains(element)) + .collect( + Collectors.collectingAndThen( + Collectors.toCollection(LinkedHashSet::new), Set::copyOf)); + } + + public static NavigableMap immutableSortedMap(Map existingMap) { + return Collections.unmodifiableNavigableMap(new TreeMap<>(existingMap)); + } + + public static final Comparator LONGEST_FIRST = + Comparator.comparingInt(x -> x.length()) + .reversed() + .thenComparing(Comparator.naturalOrder()); } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java new file mode 100644 index 000000000000..74eda248f349 --- /dev/null +++ b/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java @@ -0,0 +1,99 @@ +package com.ibm.icu.util; + +import com.ibm.icu.impl.links.LinkHandlingUtilities; +import com.ibm.icu.impl.links.LinkHandlingUtilities.UrlInternals; +import com.ibm.icu.impl.links.LinkHandlingUtilities.UrlInternals.EndStatus; +import com.ibm.icu.text.UnicodeSet; + +/** + * Utility class for assisting with detecting links (URLs or emails) in text, and formatting them + * for display, implementing the algorithms in https://www.unicode.org/reports/tr58/ to handle + * Unicode characters properly. It supplies lower level APIs for use in augmenting existing scanners + * and formatters. + */ +public class LinkUtilities { + + /** + * Lower level utility for finding the end of a PathQueryFragment (PQF) in text. It assumes that + * the start position is immediately after an identified domain name. The purpose of this + * routine is for fitting into algorithms that are already in use, just taking over for the PQF + * scanning. For more information, see https://www.unicode.org/reports/tr58/. + * + * @param source the text to be scanned + * @param start the position in the text to be scanned from. It should be immediately after a + * domain name. + * @return the end position of the PQF, or the start value if there is none. + */ + public static int scanPathQueryFragment(CharSequence source, int start, int limit) { + return LinkHandlingUtilities.parsePathQueryFragment(source.toString(), start); + } + + /** + * Lower level utility for finding the start of an email address in text. It assumes that the + * limit position is immediately before an '@' + identified domain name. The purpose of this + * routine is for fitting into algorithms that are already in use, just handling for the email + * `local-part`. + * + * @param source the text to be scanned + * @param start the position that is the earliest that should be considered in a backwards scan + * @param limit the position to start scanning backwards from — should be just after @ and just + * before the domain_name. + * @return the start of the email locale part, or limit if no email local part is found + */ + public static int scanBackEmailLocalPart(CharSequence s, int start, int limit) { + return LinkHandlingUtilities.scanEmailBackwards(s, start, limit); + } + + /** Enum for determining whether any percent-escaping is minimal or maximal, for use */ + public enum Extent { + /** Minimal percent-escaping only percent-escapes non-ASCII where necessary. */ + MINIMAL, + /** Maximal percent-escaping percent-escapes all non-ASCII. */ + MAXIMAL + } + + /** + * Escapes a URL according to the Extent parameter + * + * @param source In the source, it is assumed that ASCII syntax characters requiring escaping + * have already been escaped. For example, a literal / in a path segment would already be + * percent-escaped. For more information, see https://www.unicode.org/reports/tr58/. + * @param extent either MINIMAL or MAXIMAL + * @return an escaped string according to the extent parameter. + */ + public static String escapePathQueryFragment(String source, Extent extent) { + UrlInternals ui = UrlInternals.from(source.toString()); + switch (extent) { + case MINIMAL: + return ui.minimalEscape(EndStatus.FINAL); + case MAXIMAL: + return ui.fullEscape(); + default: + throw new InternalError(); + } + } + + // NOTE for reviewers: These are only temporary, until ICU supplies \p{Link_Term=hard} and + // \p{Idn_Status≠disallowed} + + /** + * Returns a frozen set of Unicode characters that are guaranteed to never be part of a URL or + * email address. This allows implementations to make various optimizations because URLs and + * email addresses can never span these characters. For example, a span of characters between + * safe characters that doesn't have a sequence of domain-character + . + domain-character can + * be skipped in processing. + */ + public static UnicodeSet getSafeCharacters() { + return null; + } + + /** + * Returns a frozen set of Unicode characters that are possible characters in a domain name + * (pre-mapping) This allows implementations to make various optimizations because URLs and + * email addresses must contain a sequence of domain-character + . + domain-character. It is the + * same as the set of IDNA Mapping Table character with values ≠ disallowed + */ + public static UnicodeSet getDomainCharacters() { + return null; + } +} diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java new file mode 100644 index 000000000000..063dcd429a90 --- /dev/null +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java @@ -0,0 +1,53 @@ +package com.ibm.icu.dev.test.links; + +import com.ibm.icu.dev.test.CoreTestFmwk; +import com.ibm.icu.util.LinkUtilities; +import com.ibm.icu.util.LinkUtilities.Extent; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class LinkTest extends CoreTestFmwk { + + /** Constructor */ + public LinkTest() {} + + @Test + public void testSimpleScan() { + String source = "/αβγ?def#ghi "; + int expected = source.indexOf(" "); + int actual = LinkUtilities.scanPathQueryFragment(source, 0, source.length()); + assertEquals(source, expected, actual); + } + + @Test + public void testSimpleEscape() { + String source = "/%CE%B1%CE%B2%CE%B3/%CE%B4.%CE%B5%CE%B6%2E"; + String expected = "/αβγ/δ.εζ%2E"; + String actual = LinkUtilities.escapePathQueryFragment(source, Extent.MINIMAL); + assertEquals(source, expected, actual); + } + + @Test + public void testSimpleScanBackEmail() { + String source = "See αβγ@example.com"; + int start = 0; + int end = source.indexOf('@') + 1; + int expected = source.indexOf("αβγ"); + int actual = LinkUtilities.scanBackEmailLocalPart(source, start, end); + assertEquals(source, expected, actual); + } + + // @Test + // public void testAgainstFiles() { + // // for later + // try (Stream lines = Files.lines(Paths.get("file.txt"))) { + // lines.forEach(dosomething); + // } catch (IOException e) { + // e.printStackTrace(); + // } + // + // } + +} diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkDetectionTest.txt b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkDetectionTest.txt new file mode 100644 index 000000000000..8da4c44a45ee --- /dev/null +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkDetectionTest.txt @@ -0,0 +1,432 @@ +# LinkDetectionTest.txt +# Date: 2026-02-03, 23:10:02 GMT +# © 2026 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# The usage and stability of these values is covered in https://www.unicode.org/reports/tr58/ +# +# ================================================ +# +# Format: +# Each line contains zero or more marked links, such as ⸠abc.com⸡ +# +# Operation: +# For each line. +# • Create a copy of the line, with the characters ⸠ and ⸡ removed. +# • Run link detection on the line, inserting ⸠ and ⸡ around each detected link. +# • Report a failure if the result is not identical to the original line. +# Empty lines, and lines starting with # are ignored. +# Otherwise # is treated like any other character. +# ================================================ +# + +# Misc. test cases + +# Implementations may differ as to whether they linkify text where the TLD is not listed in IANA's tlds-alpha-by-domain.txt, such as example.invalid +See example.😎 on… +See .fr on… + +# Sample uppercase +See ⸠TEST.COM⸡ on… +See ⸠FOO.VERMÖGEN.com⸡ on… + +# Illegal domain names, so not linked +See http://.foo.example.com/αβγ on… +See http://foo..example.com/αβγ on… +See http://-foo.example-.com. on… + +# Legal but unusual. Because we might be at the end of a sentence, we don't include the . unless followed by a Path, Query, or Fragment +See ⸠http://foo.example.com⸡. on… +See ⸠http://foo.example.com./αβγ⸡ on… + +# Various Path/Query/Fragment cases +See ⸠example.com⸡! on… +See ⸠example.com/αβγ⸡ on… +See ⸠example.com?αβγ⸡ on… +See ⸠example.com#αβγ⸡ on… +See ⸠example.com/αβγ/δεζ?θικ#λμν⸡ on… +See ⸠example.com/αβγ/δεζ?δ.εφ#λμν⸡ on… +See ⸠https://example.com/αβγ⸡ on… +See ⸠http://example.com/αβγ⸡ on… + +# Illustrations of Soft vs Hard, bracketing + +# Break on hard (such as ' ') +See ⸠example.com/αβγ/δεζ?δ⸡ εφ#λμν on… + +# Break on soft (such as '.') followed by hard (such as ' ') +See ⸠example.com/αβγ/δεζ?δ⸡. εφ#λμν on… +See ⸠example.com/α/βγ?δ/ε?ζ#λ/μ?ν#π⸡ on… + +# Break on unmatched bracket +See ⸠example.com/αβ⸡) on… + +# Include matched bracket +See ⸠example.com/α(β)⸡ on… +See ⸠example.com/αβγ/δρς?α=θ&β=κ#λμν:~:text=φχψ⸡ on… + +# Continue past matching brackets within same part and not across interior syntax +See ⸠example.com/α(β)γ/δρς?α=(θ)&(β)=κ#λμν:~:text=(φχψ)⸡ on… +See ⸠example.com/α[(β)]γ/δρς?α=[(θ)]&[(β)]=κ#λμν:~:text=[(φχψ)]⸡ on… + +# Don't match mismatched brackets +See ⸠example.com/α[(β⸡])γ/δρς?α=[(θ)]&[(β)]=κ#λμν:~:text=φχψ on… +See ⸠example.com/α[(β)]γ/δρς?α=[(θ⸡])&[(β)]=κ#λμν:~:text=φχψ on… +See ⸠example.com/α[(β)]γ/δρς?α=[(θ)]&[(β⸡])=κ#λμν:~:text=φχψ on… +See ⸠example.com/α[(β)]γ/δρς?α=[(θ)]&[(β)]=κ#λμν:~:text=[(φχψ⸡]) on… + +# Don't match across interior syntax +See ⸠example.com/αβ(γ/δ⸡)ρς?α=θ&β=κ#λμν:~:text=φχψ on… +See ⸠example.com/αβγ/δρς?α(=⸡)θ&β=κ#λμν:~:text=φχψ on… +See ⸠example.com/αβγ/δρς?α=θ(&⸡)β=κ#λμν:~:text=φχψ on… +See ⸠example.com/αβγ/δρς?α=θ&β=κ#λμ(ν:~:text=φχ⸡)ψ on… + +⸠https://ja.wikipedia.org/wiki/フィンセント・ファン・ゴッホ⸡ +⸠https://ja.wikipedia.org/wiki/%E3%83%95%E3%82%A3%E3%83%B3%E3%82%BB%E3%83%B3%E3%83%88%E3%83%BB%E3%83%95%E3%82%A1%E3%83%B3%E3%83%BB%E3%82%B4%E3%83%83%E3%83%9B⸡ +⸠abc.com⸡ ⸠deΩ.uk⸡ ⸠foo.accountants⸡ ⸠foo.香港⸡ +⸠abc.com/d/e?xyz#w⸡ +⸠abc.XN--11B4C3D⸡ + +# Email addresses + +See ⸠αβγ.δεζ@example.com⸡ on… +See ⸠mailto:αβγ.δεζ@example.com⸡ on… + +# If a local-part is invalid, skip the domain name +See mailto:αβγ.δεζ.@example.com on… +See mailto:αβγ..δεζ@example.com on… +See mailto:.αβγ.δεζ@example.com on… + +#Stop backing up when a space is hit +See ⸠abcd@example.com⸡ + +#Include the medial dot +See ⸠x.abcd@example.com⸡. + +#Handle non-ASCII +See ⸠アルベルト.アルベルト@example.com⸡ + +#Invalid domain name, so not linked +See @example.😎 + +# No valid local-part, so not linked +See @example.com +See john.@example.com +See .john@example.com +See ..john.doe@example.com + +#Quoted local-part, so not linked (in the base algorithm) +See "john\ doe"@example.com + +# Note that john.doe@example.com is ambiguous: it could be an email address, or it could be a URL with a userinfo +# The latter, however, is extremely rare, so always favor the email address. +# (Even when the context would tip the balance towards the URL interpretation, for consistency.) + +See ⸠example.com:1234/forum/questions/?tag=new&order=newest#top⸡ on… +See ⸠john.doe@example.com⸡:1234/forum/questions/?tag=new&order=newest#top on… + + +⸠john.smith@example.com⸡ +⸠john.smith@example.com⸡/foo/bar + +john..smith@example.com +john..smith@example.com/foo/bar + +⸠mailto:john.smith@example.com⸡ +⸠mailto:john.smith@example.com⸡/foo/bar + +mailto:john..smith@example.com +mailto:john..smith@example.com/foo/bar + +# The following are strange. the "john.smith" could be a valid userinfo, except that that is deprecated for security + +http://john.smith@example.com +http://john.smith@example.com/foo/bar + +http://john..smith@example.com +http://john..smith@example.com/foo/bar + +# Sample pages from https://meta.wikimedia.org/wiki/List_of_articles_every_Wikipedia_should_have + +⸠https://als.wikipedia.org/wiki/Wikipedia:Artikel,_wos_i_allne_Wikipedene_gee_sött⸡ +⸠https://an.wikipedia.org/wiki/Wikipedia:Lista_d'articlos_que_toda_Wikipedia_habría_de_tener⸡ +⸠https://ang.wikipedia.org/wiki/Wikipǣdia:List_of_articles_all_languages_should_have⸡ +⸠https://anp.wikipedia.org/wiki/मूलभूत_लेखॊ_के_सूची⸡ +⸠https://ar.wikipedia.org/wiki/ويكيبيديا:قائمة_مقالات_يجب_أن_تحتويها_كل_ويكيبيديا⸡ +⸠https://arc.wikipedia.org/wiki/ܘܝܩܝܦܕܝܐ:ܡܟܬܒܘܬܐ_ܕܡܓܠܐ_ܠܟܠ_ܘܝܩܝܦܕܝܐ⸡ +⸠https://ary.wikipedia.org/wiki/ويكيپيديا:1000_مقالة_خاص_يكونو_ف_جميع_لويكيپيديات⸡ +⸠https://arz.wikipedia.org/wiki/ويكيبيديا:مقالات_لازم_تكون_فى_كل_ويكيبيديا⸡ +⸠https://as.wikipedia.org/wiki/ৱিকিপিডিয়া:অত্যাৱশ্যকীয়_প্ৰবন্ধসমূহৰ_তালিকা⸡ +⸠https://ast.wikipedia.org/wiki/Wikipedia:Llista_d'artículos_que_toles_Wikipedies_habríen_de_tener⸡ +⸠https://av.wikipedia.org/wiki/Википедия:Киналго_Википедиязда_рукӀине_кколел_макъалабазул_сияхӀ⸡ +⸠https://awa.wikipedia.org/wiki/विकिपीडिया:अवधी_भाषाकय_विकिपीडिया_में_होए_वाला_न्युनतम_लेख_कय_लिस्ट⸡ +⸠https://az.wikipedia.org/wiki/Vikipediya:1000_məqalə⸡ +⸠https://azb.wikipedia.org/wiki/ویکی‌پدیا:۱۰۰۰_مقاله⸡ +⸠https://ba.wikipedia.org/wiki/Википедия:Һәр_тел_бүлегендә_булырға_тейеш_мәҡәләләр_исемлеге⸡ +⸠https://bar.wikipedia.org/wiki/Wikipedia:Artikl,_de_wo_jede_Wikipedia_hom_soit⸡ +⸠https://bat-smg.wikipedia.org/wiki/Vikipedėjė:Būtėnū_straipsniu_sārašos_vikipedėjuo⸡ +⸠https://be-tarask.wikipedia.org/wiki/Вікіпэдыя:Істотныя_артыкулы⸡ +⸠https://be.wikipedia.org/wiki/Вікіпедыя:Артыкулы,_якія_мусяць_быць_у_кожнай_Вікіпедыі⸡ +⸠https://bew.wikipedia.org/wiki/Wikipédi:Daptar_makalah_nyang_saban_Wikipédi_kudu_gableg⸡ +⸠https://bg.wikipedia.org/wiki/Уикипедия:1000_статии⸡ +⸠https://bh.wikipedia.org/wiki/विकिपीडिया:जरूरी_लेख⸡ +⸠https://bn.wikipedia.org/wiki/উইকিপিডিয়া:আবশ্যকীয়_নিবন্ধ⸡ +⸠https://bo.wikipedia.org/wiki/Wikipedia:སྐད་རིགས་ཆ་ཚང་ཡོད་པའི་རྩོམ་ཡིག་ཟིན་ཐོ⸡། +⸠https://bpy.wikipedia.org/wiki/উইকিপিডিয়া:থানা_থকিসে_নিবন্ধ⸡ +⸠https://br.wikipedia.org/wiki/Wikipedia:Roll_ar_pennadoù_a_rankfe_pep_wikipedia_kaout⸡ +⸠https://bs.wikipedia.org/wiki/Wikipedia:1000_članaka⸡ +⸠https://btm.wikipedia.org/wiki/Wikipedia:Artikel_ponting⸡ +⸠https://bug.wikipedia.org/wiki/Wikipedia:Daftar_artikel_ᨆᨊᨛᨂ_ᨅᨔ_ᨄᨑᨛᨒᨘ_ᨂᨛᨀ⸡ +⸠https://bxr.wikipedia.org/wiki/Википеэди:1000_үгүүлэл⸡ +⸠https://ca.wikipedia.org/wiki/Viquipèdia:Llista_d'articles_que_totes_les_llengües_haurien_de_tenir⸡ +⸠https://cdo.wikipedia.org/wiki/Wikipedia:Ék-chiék_Wikipedia_dŭ_găi-dŏng_ô_gì_ùng⸡ +⸠https://ce.wikipedia.org/wiki/Википеди:Массо_а_маттахь_хилайеза_агӀонаш⸡ +⸠https://ceb.wikipedia.org/wiki/Wikipedia:Talaan_sa_mga_artikulo_nga_angayng_anaa_sa_matag_Wikipedya⸡ +⸠https://ch.wikipedia.org/wiki/Wikipedia:List_of_articles_all_languages_should_have⸡ +⸠https://ckb.wikipedia.org/wiki/ویکیپیدیا:وتارە_سەرەکییەکان⸡ +⸠https://co.wikipedia.org/wiki/Wikipedia:Lista_di_l'articuli_chì_ogni_Wikipedia_duverìa_avè⸡ +⸠https://crh.wikipedia.org/wiki/Vikipediya:Er_bir_Vikipediyada_olması_kerek_olğan_maqaleler⸡ +⸠https://cs.wikipedia.org/wiki/Wikipedie:1000_nejdůležitějších_článků⸡ +⸠https://cu.wikipedia.org/wiki/Википєдїꙗ:Важьни_члѣни⸡ +⸠https://cv.wikipedia.org/wiki/Википеди:Ку_статьясем_кашни_Википеди_уйрăмĕсенче_пулмалла⸡ +⸠https://cy.wikipedia.org/wiki/Wicipedia:Rhestr_erthyglau_sy'n_angenrheidiol_ym_mhob_iaith⸡ +⸠https://da.wikipedia.org/wiki/Wikipedia:Artikler_vi_bør_have⸡ +⸠https://de.wikipedia.org/wiki/Wikipedia:Artikel,_die_es_in_allen_Wikipedias_geben_sollte⸡ +⸠https://dz.wikipedia.org/wiki/Wikipedia:List_of_articles_all_Wikipedias_should_have⸡ +⸠https://ee.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://eml.wikipedia.org/wiki/Wikipedia:Léssta_ed_artéccuel_necesàri⸡ +⸠https://en.wikipedia.org/wiki/Wikipedia:List_of_articles_all_languages_should_have⸡ +⸠https://eo.wikipedia.org/wiki/Vikipedio:Listo_de_havendaj_artikoloj⸡ +⸠https://es.wikipedia.org/wiki/Wikipedia:Lista_de_artículos_que_toda_Wikipedia_debería_tener⸡ +⸠https://et.wikipedia.org/wiki/Vikipeedia:Oluliste_artiklite_loend⸡ +⸠https://eu.wikipedia.org/wiki/Wikipedia:Wikipedia_guztiek_izan_beharreko_artikuluen_zerrenda/3._maila⸡ +⸠https://ext.wikipedia.org/wiki/Lista_d'artícalus_que_toa_Wikipedia_deviera_de_tenel⸡ +⸠https://fat.wikipedia.org/wiki/Atekels_a_ɔsɛ_dɛ_Wikipedia_biara_enya⸡ +⸠https://ff.wikipedia.org/wiki/Wikipedia:Lissafin_labaran_duk_harsuna_yakamata_su_kasance_suna_da⸡ +⸠https://fi.wikipedia.org/wiki/Wikipedia:Luettelo_keskeisistä_tietosanakirja-artikkeleista⸡ +⸠https://fiu-vro.wikipedia.org/wiki/Wikipedia:Artikliq,_miä_egan_keelen_pidänü_olõma⸡ +⸠https://fr.wikipedia.org/wiki/Wikipédia:Liste_d'articles_que_toutes_les_encyclopédies_devraient_avoir⸡ +⸠https://frr.wikipedia.org/wiki/Wikipedia:1000_wichtag_artiikler⸡ +⸠https://fur.wikipedia.org/wiki/Vichipedie:Liste_dai_articui_che_dutis_lis_vichipedîs_a_varessin_di_vê⸡ +⸠https://fy.wikipedia.org/wiki/Wikipedy:Artikels_dy't_wichtich_binne_foar_eltse_Wikipedy⸡ +⸠https://ga.wikipedia.org/wiki/Vicipéid:Liosta_d'ábhair_riachtanach_do_gach_uile_Vicipéid⸡ +⸠https://gan.wikipedia.org/wiki/Wikipedia:基礎文章⸡ +⸠https://gl.wikipedia.org/wiki/Wikipedia:Lista_de_artigos_que_toda_Wikipedia_debería_ter⸡ +⸠https://gor.wikipedia.org/wiki/Wikipedia:Artikel_u_musti_woluwo_to_timi'idu_Wikipedia⸡ +⸠https://gv.wikipedia.org/wiki/Wikipedia:Artyn_er-laccal⸡ +⸠https://ha.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://he.wikipedia.org/wiki/ויקיפדיה:מיזמי_ויקיפדיה/מיזם_ערכים_חשובים⸡ +⸠https://hi.wikipedia.org/wiki/विकिपीडिया:कुछ_प्रारंभिक_लेख_जो_कि_हर_भाषा_के_विकिपीडिया_में_होने_चाहिए⸡ +⸠https://hif.wikipedia.org/wiki/Wikipedia:Articles_jiske_sab_wikipedia_me_hoe_ke_chaahi_ke_suchi⸡ +⸠https://hr.wikipedia.org/wiki/Wikipedija:Wikiprojekt_10000/Predloženi_članci⸡ +⸠https://hsb.wikipedia.org/wiki/Wikipedija:1000_wažnych_nastawkow⸡ +⸠https://ht.wikipedia.org/wiki/Wikipedya:Lis_atik_ke_tout_ansiklopedi_yo_ta_dwe_genyen⸡ +⸠https://hu.wikipedia.org/wiki/Wikipédia:Ezer_fontos_cikk⸡ +⸠https://hy.wikipedia.org/wiki/Վիքիպեդիա:Կարևորագույն_հոդվածներ⸡ +⸠https://ia.wikipedia.org/wiki/Wikipedia:Articulos_vital⸡ +⸠https://id.wikipedia.org/wiki/Wikipedia:Artikel_penting⸡ +⸠https://ie.wikipedia.org/wiki/Liste_de_articules_quel_omni_Wikipedia_deve_haver⸡ +⸠https://ilo.wikipedia.org/wiki/Wikipedia:Listaan_dagiti_artikulo_a_nasken_nga_adda_ti_amin_a_Wikipedia⸡ +⸠https://inh.wikipedia.org/wiki/Википеди:Массайолча_эршашка_хила_езаш_йола_оагӀонаш⸡ +⸠https://io.wikipedia.org/wiki/Wikipedio:Obligita_artikli⸡ +⸠https://is.wikipedia.org/wiki/Wikipedia:Grundvallargreinar⸡ +⸠https://it.wikipedia.org/wiki/Wikipedia:Lista_delle_voci_che_tutte_le_Wikipedie_dovrebbero_avere⸡ +⸠https://ja.wikipedia.org/wiki/Wikipedia:すべての言語版にあるべき項目の一覧⸡ +⸠https://jam.wikipedia.org/wiki/Wikipidia:Di_nesiseri_aatikl_dem⸡ +⸠https://jbo.wikipedia.org/wiki/uikipedi'as:liste_be_le_tcevai_ckupau⸡ +⸠https://jv.wikipedia.org/wiki/Wikipédia:Daftar_Artikel_Dhasar⸡ +⸠https://ka.wikipedia.org/wiki/ვიკიპედია:აუცილებელი_სტატიები⸡ +⸠https://kaa.wikipedia.org/wiki/Wikipedia:Hámme_Wikipedialarda_bolıwı_tiyis_maqalalar_dizimi⸡ +⸠https://kbd.wikipedia.org/wiki/Уикипедиэ:Бзэ_версиэу_хъуам_яӀэн_хуей_тхыгъэхэм_я_напэ⸡ +⸠https://kcg.wikipedia.org/wiki/A̱tsatsak_a̱yaati̱kut⸡ +⸠https://kg.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://kk.wikipedia.org/wiki/Уикипедия:Барлық_Уикипедияларда_болуы_тиіс_мақала_тізімі⸡ +⸠https://kn.wikipedia.org/wiki/ವಿಕಿಪೀಡಿಯ:ಅಗತ್ಯ_ಲೇಖನಗಳು⸡ +⸠https://ko.wikipedia.org/wiki/위키백과:모든_언어의_위키백과마다_꼭_있어야_하는_문서_목록⸡ +⸠https://krc.wikipedia.org/wiki/Википедия:Хар_Википедияда_болургъа_керек_статьяланы_тизмеси⸡ +⸠https://ks.wikipedia.org/wiki/وِکیٖپیٖڈیا:Vital_articles⸡ +⸠https://ksh.wikipedia.org/wiki/Wikipedia:Sigge_di_en_alle_Sproche_drin_sin_sullte⸡ +⸠https://ku.wikipedia.org/wiki/Wîkîpediya:Gotarên_ku_ji_bo_hemû_Wîkîpediyayan_pêwîst_in⸡ +⸠https://kw.wikipedia.org/wiki/Wikipedia:1,000_erthygel⸡ +⸠https://ky.wikipedia.org/wiki/Википедия:Vital_articles⸡ +⸠https://la.wikipedia.org/wiki/Vicipaedia:Paginae_quas_omnibus_Wikipediis_contineri_oportet⸡ +⸠https://lad.wikipedia.org/wiki/Vikipedya:Lista_de_artikolos_ke_toda_Vikipedya_kale_tener⸡ +⸠https://lb.wikipedia.org/wiki/Wikipedia:Artikelen,_déi_et_op_all_Wikipedia_sollt_ginn⸡ +⸠https://lez.wikipedia.org/wiki/Википедия:1000⸡ +⸠https://lfn.wikipedia.org/wiki/Vicipedia:Lista_de_articles_nesesada_per_vicipedia⸡ +⸠https://lg.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://li.wikipedia.org/wiki/Wikipedia:Lies_van_artikele_die_eder_Wikipedia_moot_höbbe⸡ +⸠https://lij.wikipedia.org/wiki/Wikipedia:Vôxe_de_bâze⸡ +⸠https://lmo.wikipedia.org/wiki/Wikipedia:Lista_di_articoi_che_tutt_i_Wikipedii_dovarissen_avègh⸡ +⸠https://lt.wikipedia.org/wiki/Vikipedija:Sąrašas_straipsnių,_būtinų_visomis_kalbomis⸡ +⸠https://lv.wikipedia.org/wiki/Vikipēdija:Nozīmīgi_raksti⸡ +⸠https://mad.wikipedia.org/wiki/Wikipèḍia:Artikel_penting⸡ +⸠https://mai.wikipedia.org/wiki/विकिपिडिया:महत्वपूर्ण_लेखसभ⸡ +⸠https://mdf.wikipedia.org/wiki/Википедиесь:1000_эрявикс_сёрматфксне⸡ +⸠https://min.wikipedia.org/wiki/Wikipedia:Artikel_nan_paralu_ado_di_satiok_Wikipedia⸡ +⸠https://mk.wikipedia.org/wiki/Википедија:Потребни_статии⸡ +⸠https://ml.wikipedia.org/wiki/വിക്കിപീഡിയ:സുപ്രധാന_ലേഖനങ്ങൾ⸡ +⸠https://mni.wikipedia.org/wiki/ꯋꯤꯀꯤꯄꯦꯗꯤꯌꯥ_ꯈꯨꯗꯤꯡꯗ_ꯌꯥꯎꯒꯗꯕ_ꯋꯥꯔꯦꯡꯁꯤꯡ⸡ +⸠https://mr.wikipedia.org/wiki/विकिपीडिया:सगळ्या_विकिपीडियांवर_अपेक्षित_लेखांची_यादी/आंतरभाषीय_परिपेक्ष⸡ +⸠https://ms.wikipedia.org/wiki/Wikipedia:Senarai_rencana_semua_bahasa_perlu_ada⸡ +⸠https://mt.wikipedia.org/wiki/Wikipedija:Lista_ta'_artikli_li_kull_Wikipedija_għandu_jkollha⸡ +⸠https://mwl.wikipedia.org/wiki/Biquipédia:1000_artigos_bitales⸡ +⸠https://my.wikipedia.org/wiki/ဝီကီပီးဒီးယား:မြန်မာဝီကီတွင်_ရှိသင့်သော_ဆောင်းပါးများ⸡ +⸠https://myv.wikipedia.org/wiki/Википедиясь:Лопань_потмокс,_конатат_эрявить_эрьва_келень_пельксэ⸡ +⸠https://nah.wikipedia.org/wiki/Huiquipedia:Amatlahcuilolli_tlen_monequi_ipan_nochi_Huiquipedia_ica_nahuatlahtolli⸡ +⸠https://nap.wikipedia.org/wiki/Wikipedia:Lista_d"e_vvoce_ca_tutt'_'e_Wikipedie_aveno_a_avé⸡ +⸠https://nds.wikipedia.org/wiki/Wikipedia:List_vun_Artikels,_de_in_all_Wikipedias_binnen_ween_schöölt⸡ +⸠https://ne.wikipedia.org/wiki/विकिपिडिया:प्रत्येक_भाषामा_हुनु_पर्ने_न्यूनतम_लेखहरू⸡ +⸠https://ng.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://nl.wikipedia.org/wiki/Wikipedia:Archief/Wikipedia:Artikelen_die_elke_Wikipedia_zou_moeten_hebben⸡ +⸠https://nn.wikipedia.org/wiki/Wikipedia:Artiklar_vi_bør_ha⸡ +⸠https://no.wikipedia.org/wiki/Wikipedia:Liste_over_artikler_vi_bør_ha⸡ +⸠https://nov.wikipedia.org/wiki/Wikipedia:Liste_de_artikles_kel_chaki_wikipedia_deve_kontena⸡ +⸠https://nso.wikipedia.org/wiki/Wikipedia:Ditemana_tše_Bohlokwa⸡ +⸠https://ny.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://oc.wikipedia.org/wiki/Wikipèdia:Tièra_de_1000_articles_que_totas_las_Wikipèdias_deurián_aver⸡ +⸠https://olo.wikipedia.org/wiki/Wikipedii:Vältämättömien_Wikipedii-kirjutuksien_luvettelo⸡ +⸠https://om.wikipedia.org/wiki/Wikipedia:Tarreeffama_barruu_Wiikiipiidiyaa_hundi_qabaachuu_qabu⸡ +⸠https://os.wikipedia.org/wiki/Википеди:Æппæт_Википедиты_чи_хъуамæ_уа,_уыцы_уацтæ⸡ +⸠https://pag.wikipedia.org/wiki/Wikipedia:Listaan_na_nakaukulan_ed_balang_salita⸡ +⸠https://pap.wikipedia.org/wiki/Wikipedia:Artíkulonan_importante⸡ +⸠https://pl.wikipedia.org/wiki/Wikipedia:Strony,_które_powinna_mieć_każda_Wikipedia⸡ +⸠https://pms.wikipedia.org/wiki/Wikipedia:Lista_dj'artìcoj_che_minca_Wikipedia_a_dovrìa_avèj⸡ +⸠https://pnt.wikipedia.org/wiki/Βικιπαίδεια:1000_σελίδας⸡ +⸠https://pt.wikipedia.org/wiki/Wikipédia:Lista_dos_artigos_que_toda_Wikipédia_deve_ter⸡ +⸠https://qu.wikipedia.org/wiki/Wikipidiya:Lliw_Wikipidiyapaq_qillqanakuna⸡ +⸠https://rm.wikipedia.org/wiki/Wikipedia:Artitgels_che_mintga_Vichipedia_duai_aver⸡ +⸠https://ro.wikipedia.org/wiki/Wikipedia:Articole_vitale⸡ +⸠https://roa-rup.wikipedia.org/wiki/Wikipedia:Frândzâ_anânghioasi_trâ_unâ_enciclopedie⸡ +⸠https://ru.wikipedia.org/wiki/Википедия:Список_статей,_которые_должны_быть_во_всех_языковых_версиях⸡ +⸠https://rue.wikipedia.org/wiki/Вікіпедія:Статї,_якы_повинны_быти⸡ +⸠https://rw.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://sa.wikipedia.org/wiki/विकिपीडिया:प्रारम्भिक-विषयाः⸡ +⸠https://sah.wikipedia.org/wiki/Бикипиэдьийэ:Ханнык_баҕарар_Бикипиэдьийэҕэ_баар_буолуохтаах_ыстатыйалар_тиһиликтэрэ⸡ +⸠https://sat.wikipedia.org/wiki/ᱞᱟᱹᱠᱛᱤᱭᱟᱱ_ᱚᱱᱚᱞᱠᱚ⸡ +⸠https://sc.wikipedia.org/wiki/Wikipedia:Lista_de_sos_artìculos_chi_totu_sas_Wikipedias_bisonzat_a_tenner⸡ +⸠https://scn.wikipedia.org/wiki/Wikipedia:Artìculi_nicissari⸡ +⸠https://sco.wikipedia.org/wiki/Wikipedia:Leet_o_airticles_that_aa_Wikipedias_shuid_hae⸡ +⸠https://sh.wikipedia.org/wiki/Wikipedija:Spisak_članaka_koje_sve_Wikipedije_trebaju_imati⸡ +⸠https://simple.wikipedia.org/wiki/Wikipedia:List_of_articles_all_languages_should_have⸡ +⸠https://sk.wikipedia.org/wiki/Wikipédia:1_000_najdôležitejších_článkov⸡ +⸠https://sl.wikipedia.org/wiki/Wikipedija:Članki,_ki_bi_jih_morala_imeti_vsaka_Wikipedija⸡ +⸠https://so.wikipedia.org/wiki/Wikipedia:Maqaalada_aan_u_baahan_laheen⸡ +⸠https://sq.wikipedia.org/wiki/Wikipedia:Artikujt_që_secila_Wikipedia_duhet_t'i_ketë⸡ +⸠https://sr.wikipedia.org/wiki/Википедија:Списак_чланака_које_свака_Википедија_треба_да_има⸡ +⸠https://srn.wikipedia.org/wiki/Wikipedia:Rei_fu_Peprewoysi_ala_Wikipedia_mu_Abi⸡ +⸠https://ss.wikipedia.org/wiki/Wikipedia:List_of_articles_all_languages_should_have⸡ +⸠https://st.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://stq.wikipedia.org/wiki/Wikipedia:Artikkele_do't_in_älke_Wikipedia_reeke_skäl⸡ +⸠https://su.wikipedia.org/wiki/Wikipedia:Artikel_nu_sadaya_basa_Wikipédia_kedah_gaduh⸡ +⸠https://sv.wikipedia.org/wiki/Wikipedia:Basartiklar⸡ +⸠https://sw.wikipedia.org/wiki/Wikipedia:Makala_za_msingi_za_kamusi_elezo⸡ +⸠https://ta.wikipedia.org/wiki/விக்கிப்பீடியா:முக்கிய_கட்டுரைகள்⸡ +⸠https://te.wikipedia.org/wiki/వికీపీడియా:వికీపీడియాలో_తప్పకుండా_ఉండవలసిన_వ్యాసాలు⸡ +⸠https://tet.wikipedia.org/wiki/Wikipedia:Lista_husi_artigu_sira_ne'ebé_Wikipédia_hotu-hotu_bele_iha⸡ +⸠https://tg.wikipedia.org/wiki/Википедиа:Феҳристи_мақолаҳои_муҳим⸡ +⸠https://th.wikipedia.org/wiki/วิกิพีเดีย:รายการบทความที่วิกิพีเดียทุกภาษาควรมี⸡ +⸠https://tk.wikipedia.org/wiki/Wikipediýa:Wikipediýada_bolmagy_gerek_sahypalar⸡ +⸠https://tl.wikipedia.org/wiki/Wikipedia:Talaan_ng_mga_artikulong_kailangang_mayroon_ang_bawat_Wikipedia⸡ +⸠https://tly.wikipedia.org/wiki/Vikipediá:Məǧolon_sijohi,_komon_bəpe_har_zyvonədə_bybun⸡ +⸠https://tn.wikipedia.org/wiki/Wikipedia:List_of_articles_we_should_have⸡ +⸠https://tpi.wikipedia.org/wiki/Wikipedia:Ol_pes_i_mas_stap_long_Wikipedia⸡ +⸠https://tr.wikipedia.org/wiki/Vikipedi:Her_Vikipedi'de_olması_gereken_maddeler⸡ +⸠https://ts.wikipedia.org/wiki/Wikipedia:Matsalwa_ya_Nkoka/xiyenge/3⸡ +⸠https://tt.wikipedia.org/wiki/Википедия:Һәрбер_тел_бүлегендә_булырга_тиешле_мәкаләләр⸡ +⸠https://tum.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://tw.wikipedia.org/wiki/Wikipidia:Atwerɛsɛm_dodoɔ_a_ɛwɔ_sɛ_Wikipidia_ɛnya⸡ +⸠https://ty.wikipedia.org/wiki/Wikipedia:Te_àpi_faufaa⸡ +⸠https://tyv.wikipedia.org/wiki/Википедия:Шупту_дылдарга_бижээн_турар_чүүлдер⸡ +⸠https://udm.wikipedia.org/wiki/Википедия:Тужгес_кулэ_статьяос⸡ +⸠https://uk.wikipedia.org/wiki/Вікіпедія:Статті,_які_повинні_бути_в_усіх_Вікіпедіях⸡ +⸠https://ur.wikipedia.org/wiki/ویکیپیڈیا:مضامین_تازہ⸡ +⸠https://uz.wikipedia.org/wiki/Vikipediya:Muhim_maqolalar⸡ +⸠https://ve.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://vec.wikipedia.org/wiki/Wikipedia:Lista_de_łe_voze_che_tuta_Wikipèdia_garia_da_ver⸡ +⸠https://vep.wikipedia.org/wiki/Vikipedii:Tarbhaižed_lehtpoled,_kudambad_tarbiž_säta_kaikihe_Vikipedijan_versijoihe⸡ +⸠https://wuu.wikipedia.org/wiki/Wikipedia:必有个页面⸡ +⸠https://xal.wikipedia.org/wiki/Wikipedia:Википедьт_иим_төрмүд_бəəх_зөвтə⸡ +⸠https://xh.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ +⸠https://xmf.wikipedia.org/wiki/ვიკიპედია:უციო_სტატიეფი⸡ +⸠https://yi.wikipedia.org/wiki/װיקיפּעדיע:וויכטיגע_ארטיקלען⸡ +⸠https://yo.wikipedia.org/wiki/Wikipedia:Àwọn_àyọkà_tó_ṣe_kóko⸡ +⸠https://zea.wikipedia.org/wiki/Wikipedia:Artikels_die_ieleken_Wikipedia_zou_motte_ha⸡ +⸠https://zgh.wikipedia.org/wiki/ⵡⵉⴽⵉⴱⵉⴷⵢⴰ:ⵜⴰⵍⴳⴰⵎⵜ_ⵏ_ⵉⵎⴳⵔⴰⴷⵏ_ⵏⵏⴰ_ⴷ_ⵉⵇⵇⴰⵏ_ⴰⴷ_ⵉⵍⵉⵏ_ⴳ_ⴽⵔⴰⵢⴳⴰⵜⵜ_ⵡⵉⴽⵉⴱⵉⴷⵢⴰ⸡ +⸠https://zh-classical.wikipedia.org/wiki/維基大典:當有之文⸡ +⸠https://zh-min-nan.wikipedia.org/wiki/Wikipedia:Só͘-ū_ê_Wikipedia_pán-pún_lóng_èng-kai_ū_ê_bûn-chiuⁿ⸡ +⸠https://zh-yue.wikipedia.org/wiki/Wikipedia:基礎文章⸡ +⸠https://zh.wikipedia.org/wiki/Wikipedia:基礎條目⸡ +⸠https://zu.wikipedia.org/wiki/Wikipedia:Vital_articles⸡ + +# Test cases contributed by ICANN + +a ⸠blogspot.com⸡ b +a com b +a ⸠example.com:443⸡ b +a ⸠example.com/123⸡ b +a ⸠example.com?123⸡ b +a ⸠example.com#123⸡ b +a ⸠example.com/123?123#123⸡ b +a (⸠example.com/123?123#123⸡) b +a (⸠example.com/123?123#(123)⸡) b +a ⸠example.com/123.html⸡ b +a ⸠example.com/123⸡. HTML is great +a ⸠example.com/123⸡... HTML is so great +a ⸠example.com/123⸡. +a ⸠xn-----ctdbabcfhu9c2b9l1acccr4c.xn--mgbah1a3hjkrd⸡ b +Lorem ipsum ⸠universal-acceptance-test.international⸡ dolor sit amet +Lorem ipsum ⸠universal-acceptance-test.icu⸡ dolor sit amet +Lorem ipsum ⸠تجربة-القبول-الشامل.موريتانيا⸡ dolor sit amet +Lorem ipsum ⸠համընդհանուր-ընկալում-թեստ.հայ⸡ dolor sit amet +Lorem ipsum ⸠সর্বজনীন-স্বীকৃতির-পরীক্ষা.ভারত⸡ dolor sit amet +Lorem ipsum ⸠универсальное-принятие-тест.москва⸡ dolor sit amet +Lorem ipsum ⸠सार्वभौमिक-स्वीकृति-परीक्षण.संगठन⸡ dolor sit amet +Lorem ipsum ⸠უნივერსალური-თავსობადობის-ტესტი.გე⸡ dolor sit amet +Lorem ipsum ⸠καθολική-αποδοχή-δοκιμή.ευ⸡ dolor sit amet +Lorem ipsum ⸠સાર્વત્રિક-સ્વીકૃતિ-પરીક્ષણ.ભારત⸡ dolor sit amet +Lorem ipsum ⸠ਸਰਵਵਿਆਪਕ-ਪ੍ਰਵਾਨਗੀ-ਪਰਖ.ਭਾਰਤ⸡ dolor sit amet +Lorem ipsum ⸠다국어도메인이용환경테스트.한국⸡ dolor sit amet +Lorem ipsum ⸠מבחן-קבלה-אוניברסלי.קום⸡ dolor sit amet +Lorem ipsum ⸠どこでもつかえる.みんな⸡ dolor sit amet +Lorem ipsum ⸠ಸಾರ್ವತ್ರಿಕ-ಸ್ವೀಕಾರಾರ್ಹತೆ-ಪರೀಕ್ಷೆ.ಭಾರತ⸡ dolor sit amet +Lorem ipsum ⸠ユニバーサルアクセプタンス.クラウド⸡ dolor sit amet +Lorem ipsum ⸠ສາກົນ-ການຍອມຮັບ-ທົດລອງ.ລາວ⸡ dolor sit amet +Lorem ipsum ⸠സാർവത്രിക-സ്വീകാര്യതാ-പരിശോധന.ഭാരതം⸡ dolor sit amet +Lorem ipsum ⸠ଯୁନିଭରସାଲ-ଏକସେପ୍ଟନ୍ସ-ଟେଷ୍ଟ.ଭାରତ⸡ dolor sit amet +Lorem ipsum ⸠විශ්ව-සම්මුති-පිරික්සුම.ලංකා⸡ dolor sit amet +Lorem ipsum ⸠பொது-ஏற்பு-சோதனை.சிங்கப்பூர்⸡ dolor sit amet +Lorem ipsum ⸠యూనివర్సల్-ఆమోదం-పరీక్ష.భారత్⸡ dolor sit amet +Lorem ipsum ⸠ยูเอทดสอบ.ไทย⸡ dolor sit amet +Lorem ipsum ⸠普遍适用测试.我爱你⸡ dolor sit amet +Lorem ipsum ⸠普遍適用測試.台灣⸡ dolor sit amet +Lorem ipsum ⸠ሁለንአቀፍ-ተቀባይነት-ሙከራ.com⸡ dolor sit amet +Lorem ipsum ⸠ការសាកល្បងទទួលយកជាអន្តរជាតិ.com⸡ dolor sit amet +Lorem ipsum ⸠အလုံးစုံလက်ခံမှုစမ်းသပ်ချက်.com⸡ dolor sit amet +Lorem ipsum ⸠ދުނިޔެ-ގަބޫލުކުރާ-ޓެސްޓު.com⸡ dolor sit amet +Lorem ipsum ⸠universal-acceptance-test.קום⸡ dolor sit amet +Lorem ipsum ⸠épreuve-acceptation-universelle.org⸡ dolor sit amet +Lorem ipsum ⸠ཡོངས་ཁྱབ་ངོས་ལེན་བརྟག་དཔྱད.com⸡ dolor sit amet +Lorem ipsum ⸠Universales-Akzeptanz-Test.vermögensberatung⸡ dolor sit amet +Lorem ipsum ⸠épreuve-acceptation-universelle.org⸡ dolor sit amet +Lorem ipsum ⸠普遍适用测试。我爱你⸡ dolor sit amet +Lorem ipsum موريتانيا.xn-----ctdbabcfhu9c2b9l1acccr4c dolor sit amet +Lorem ipsum ⸠تجربة-القبول-الشامل.xn--mgbah1a3hjkrd⸡ dolor sit amet +Lorem ipsum ⸠xn-----ctdbabcfhu9c2b9l1acccr4c.xn--mgbah1a3hjkrd⸡ dolor sit amet +Lorem ipsum ⸠universal-acceptance-test.icu/测试⸡ dolor sit amet +Lorem ipsum ⸠普遍适用测试.我爱你/测试⸡ dolor sit amet +Lorem ipsum ⸠تجربة-القبول-الشامل.موريتانيا/تجربة⸡ dolor sit amet +blah ⸠en.wikipedia.org/wiki/The_Lovemakers_(film)⸡ blah +blah ⸠example.com/?foo[1]=a&foo[2]=b⸡ blah +Lorem ipsum ⸠comoyo.com/play/S(123)⸡ dolor sit amet +Lorem ipsum ⸠example.com/knutsen_ludvigsen/ver(k)ste(d)/brilleslange.mp3⸡ dolor sit amet +Lorem ipsum ⸠example.com/Bob_Marley/Rastaman_Vibration/11_Jah_Live_(originally_issued_as_Island_Single_(WIP_6265))_(bonus_track⸡ dolor sit amet +Lorem ipsum ⸠business.timesonline.co.uk/article/0,,9065-2473189,00.html⸡ dolor sit amet +Lorem ipsum ⸠www.mail-archive.com/ruby-talk@ruby-lang.org/⸡ dolor sit amet +Lorem ipsum ⸠tools.ietf.org/html/rfc3986⸡ dolor sit amet +Lorem ipsum ⸠www.amazon.com/Testing-Equal-Sign-In-Path/ref=pd_bbs_sr_1?ie=UTF8&s=books&qid=1198861734&sr=8-1⸡ dolor sit amet +Lorem ipsum ⸠www.google.com/doku.php?id=gps:resource:scs:start⸡ dolor sit amet +Lorem ipsum ⸠maps.google.co.uk/maps?f=q&q=the+london+eye&ie=UTF8&ll=51.503373,-0.11939&spn=0.007052,0.012767&z=16&iwloc=A⸡ dolor sit amet +Lorem ipsum ⸠www.rubyonrails.com/foo.cgi?trailing_hyphen=value-⸡ dolor sit amet +Lorem ipsum ⸠www.rubyonrails.com/foo.cgi?trailing_forward_slash=value⸡ dolor sit amet diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkFormattingTest.txt b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkFormattingTest.txt new file mode 100644 index 000000000000..eb3ff5169136 --- /dev/null +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkFormattingTest.txt @@ -0,0 +1,278 @@ +# LinkFormattingTest.txt +# Date: 2026-05-20, 21:41:06 GMT +# © 2026 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# The usage and stability of these values is covered in https://www.unicode.org/reports/tr58/ +# +# ================================================ +# +# Format: This is not a file with semicolon-delimited fields. +# Instead, it consists of paired lines, separated by empty lines and/or comment lines. +# The first line of each pair is the source, fully escaped. +# The second line of each pair is the result, minimally escaped. +# +# Comments are lines that begin with a #. +# +# The fully-escaped field percent-escapes all code points based on https://url.spec.whatwg.org/#percent-encoded-bytes. +# This means all literal syntax characters in each Part and all code points above ASCII. +# It also percent-escapes the last character, if it is Link_Term=Soft. +# +# The minimally-escaped field is the more readable format described in UTS #58. +# +# Each pair also has a comment line for the internal structure of the URL. +# 𝑺 = the schema +# 𝑯 = the host (typically just a domain name) the internal structure is not broken down. +# 𝑷 = indicates one of the labels in the path. So /seg1/seg2 becomes 𝑷=seg1 𝑷=seg1 +# 𝑸 = each key in the query. So &θ=ικλ&μ=γξο becomes 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γξο. +# 𝑽 = a value for the preceding key +# 𝑭 = a fragment. So #reserved:~:text=Reserved,into:~:text=open,closed becomes 𝑭=reserved 𝑫=Reserved,into 𝑫=text=open,closed +# 𝑫 = a fragment-directive +# ================================================ +# + +# Selected test cases + +# Path only +# {𝑺=https:// 𝑯=example.com 𝑷=α} +https://example.com/%CE%B1 +https://example.com/α + +# Query only +# {𝑺=https:// 𝑯=example.com 𝑸=α} +https://example.com?%CE%B1 +https://example.com?α + +# Fragment only +# {𝑺=https:// 𝑯=example.com 𝑭=α} +https://example.com#%CE%B1 +https://example.com#α + +# All parts +# {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δεζ 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γξο 𝑭=πρς} +https://example.com/%CE%B1%CE%B2%CE%B3/%CE%B4%CE%B5%CE%B6?%CE%B8=%CE%B9%CE%BA%CE%BB&%CE%BC=%CE%B3%CE%BE%CE%BF#%CF%80%CF%81%CF%82 +https://example.com/αβγ/δεζ?θ=ικλ&μ=γξο#πρς + +# Escape soft at end of Path (with nothing following) +# {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δ.εζ.} +https://example.com/%CE%B1%CE%B2%CE%B3/%CE%B4.%CE%B5%CE%B6%2E +https://example.com/αβγ/δ.εζ%2E + +# Escape soft at end of Query (with nothing following) +# {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δ.εζ. 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γ.ξο.} +https://example.com/%CE%B1%CE%B2%CE%B3/%CE%B4.%CE%B5%CE%B6.?%CE%B8=%CE%B9%CE%BA%CE%BB&%CE%BC=%CE%B3.%CE%BE%CE%BF%2E +https://example.com/αβγ/δ.εζ.?θ=ικλ&μ=γ.ξο%2E + +# Escape soft at end of Fragment (with nothing following) +# {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δ.εζ 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γ.ξο 𝑭=π.ρς.} +https://example.com/%CE%B1%CE%B2%CE%B3/%CE%B4.%CE%B5%CE%B6?%CE%B8=%CE%B9%CE%BA%CE%BB&%CE%BC=%CE%B3.%CE%BE%CE%BF#%CF%80.%CF%81%CF%82%2E +https://example.com/αβγ/δ.εζ?θ=ικλ&μ=γ.ξο#π.ρς%2E + +# Escape ? in Path +# {𝑺=https:// 𝑯=example.com 𝑷=α?μπ} +https://example.com/%CE%B1%3F%CE%BC%CF%80 +https://example.com/α%3Fμπ + +# Escape #, =, & in Path/Query +# {𝑺=https:// 𝑯=example.com 𝑷=α#β 𝑸=γ 𝑽=δ#ε} +https://example.com/%CE%B1#%CE%B2?%CE%B3=%CE%B4#%CE%B5 +https://example.com/α%23β?γ=δ%23ε + +# Escape hard (' ') +# {𝑺=https:// 𝑯=example.com 𝑷=αβ γ 𝑷=δεζ 𝑸=θ 𝑽=ικ λ 𝑸= 𝑽=γξο 𝑭=πρ σ} +https://example.com/%CE%B1%CE%B2%20%CE%B3/%CE%B4%CE%B5%CE%B6?%CE%B8=%CE%B9%CE%BA%20%CE%BB&=%CE%B3%CE%BE%CE%BF#%CF%80%CF%81%20%CF%83 +https://example.com/αβ%20γ/δεζ?θ=ικ%20λ&=γξο#πρ%20σ + +# Escape soft ('.') unless followed by include +# {𝑺=https:// 𝑯=example.com 𝑷=αβγ. 𝑷=δεζ. 𝑸=θ 𝑽=ικ.λ 𝑸= 𝑽=γξο. 𝑭=πρς.} +https://example.com/%CE%B1%CE%B2%CE%B3./%CE%B4%CE%B5%CE%B6.?%CE%B8=%CE%B9%CE%BA.%CE%BB&=%CE%B3%CE%BE%CE%BF.#%CF%80%CF%81%CF%82%2E +https://example.com/αβγ./δεζ.?θ=ικ.λ&=γξο.#πρς%2E + +# Escape unmatched brackets +# {𝑺=https:// 𝑯=example.com 𝑷=α(β) 𝑸=γ(δ) 𝑭=ε(ζ))} +https://example.com/%CE%B1(%CE%B2)?%CE%B3(%CE%B4)#%CE%B5(%CE%B6)) +https://example.com/α(β)?γ(δ)#ε(ζ)%29 + +# {𝑺=https:// 𝑯=example.com 𝑷=α(β) 𝑸=γ(δ))} +https://example.com/%CE%B1(%CE%B2)?%CE%B3(%CE%B4)) +https://example.com/α(β)?γ(δ)%29 + +# {𝑺=https:// 𝑯=example.com 𝑷=α(β) 𝑭=ε(ζ))} +https://example.com/%CE%B1(%CE%B2)#%CE%B5(%CE%B6)) +https://example.com/α(β)#ε(ζ)%29 + +# {𝑺=https:// 𝑯=example.com 𝑷=α(β))} +https://example.com/%CE%B1(%CE%B2)) +https://example.com/α(β)%29 + +# {𝑺=https:// 𝑯=example.com 𝑸=γ(δ) 𝑭=ε(ζ))} +https://example.com?%CE%B3(%CE%B4)#%CE%B5(%CE%B6)) +https://example.com?γ(δ)#ε(ζ)%29 + +# {𝑺=https:// 𝑯=example.com 𝑸=γ(δ))} +https://example.com?%CE%B3(%CE%B4)) +https://example.com?γ(δ)%29 + +# {𝑺=https:// 𝑯=example.com 𝑭=ε(ζ))} +https://example.com#%CE%B5(%CE%B6)) +https://example.com#ε(ζ)%29 + +# No escape for matched brackets +# {𝑺=https:// 𝑯=example.com 𝑷=α(β) 𝑸=γ(δ) 𝑭=ε(ζ)} +https://example.com/%CE%B1(%CE%B2)?%CE%B3(%CE%B4)#%CE%B5(%CE%B6) +https://example.com/α(β)?γ(δ)#ε(ζ) + +# {𝑺=https:// 𝑯=example.com 𝑷=α(β) 𝑸=γ(δ)} +https://example.com/%CE%B1(%CE%B2)?%CE%B3(%CE%B4) +https://example.com/α(β)?γ(δ) + +# {𝑺=https:// 𝑯=example.com 𝑷=α(β) 𝑭=ε(ζ)} +https://example.com/%CE%B1(%CE%B2)#%CE%B5(%CE%B6) +https://example.com/α(β)#ε(ζ) + +# {𝑺=https:// 𝑯=example.com 𝑷=α(β)} +https://example.com/%CE%B1(%CE%B2) +https://example.com/α(β) + +# {𝑺=https:// 𝑯=example.com 𝑸=γ(δ) 𝑭=ε(ζ)} +https://example.com?%CE%B3(%CE%B4)#%CE%B5(%CE%B6) +https://example.com?γ(δ)#ε(ζ) + +# {𝑺=https:// 𝑯=example.com 𝑸=γ(δ)} +https://example.com?%CE%B3(%CE%B4) +https://example.com?γ(δ) + +# {𝑺=https:// 𝑯=example.com 𝑭=ε(ζ)} +https://example.com#%CE%B5(%CE%B6) +https://example.com#ε(ζ) + +# Path with escaped separator +# {𝑺=https:// 𝑯=example.com 𝑷=α 𝑷=β/γ} +https://example.com/%CE%B1/%CE%B2/%CE%B3 +https://example.com/α/β%2Fγ + +# Path with escapes (% is escaped only if followed by 2 hex digits [0-9A-fa-f) +# {𝑺=https:// 𝑯=example.com 𝑷=α 𝑷=β%41γ%ε%} +https://example.com/%CE%B1/%CE%B2%2541%CE%B3%%CE%B5% +https://example.com/α/β%2541γ%ε% + +# Query with escapes (& separates key-value pairs, so handle literal &) +# {𝑺=https:// 𝑯=example.com 𝑸=α& 𝑽=β 𝑸=γ 𝑽=&δ} +https://example.com?%CE%B1&=%CE%B2&%CE%B3=&%CE%B4 +https://example.com?α%26=β&γ=%26δ + +# Query with escapes (= separates keys and values, so handle literal =) +# {𝑺=https:// 𝑯=example.com 𝑸=α=β 𝑽=γ=δ} +https://example.com?%CE%B1=%CE%B2=%CE%B3=%CE%B4 +https://example.com?α%3Dβ=γ%3Dδ + +# Query with escapes (% is escaped only if followed by 2 hex digits [0-9A-fa-f) +# {𝑺=https:// 𝑯=example.com 𝑸=α%β 𝑽=γ%δ} +https://example.com?%CE%B1%%CE%B2=%CE%B3%%CE%B4 +https://example.com?α%β=γ%δ + +# Path and Query with quoted syntax characters +# {𝑺=https:// 𝑯=example.com 𝑷=α 𝑷=b/?#c 𝑸=αβ 𝑽=γ&ζ=#Ξ 𝑸=k 𝑽=v 𝑭=frag} +https://example.com/%CE%B1/b/%3F#c?%CE%B1%CE%B2=%CE%B3&%CE%B6=#%CE%9E&k=v#frag +https://example.com/α/b%2F%3F%23c?αβ=γ%26ζ%3D%23Ξ&k=v#frag + + +# Wikipedia test cases + +# {𝑺=https:// 𝑯=ru.wikinews.org 𝑷=wiki 𝑷=Категория:Вселенная} +https://ru.wikinews.org/wiki/%D0%9A%D0%B0%D1%82%D0%B5%D0%B3%D0%BE%D1%80%D0%B8%D1%8F:%D0%92%D1%81%D0%B5%D0%BB%D0%B5%D0%BD%D0%BD%D0%B0%D1%8F +https://ru.wikinews.org/wiki/Категория:Вселенная + +# {𝑺=https:// 𝑯=av.wikipedia.org 𝑷=wiki 𝑷=Ракь_(планета)} +https://av.wikipedia.org/wiki/%D0%A0%D0%B0%D0%BA%D1%8C_(%D0%BF%D0%BB%D0%B0%D0%BD%D0%B5%D1%82%D0%B0) +https://av.wikipedia.org/wiki/Ракь_(планета) + +# {𝑺=https:// 𝑯=bo.wikipedia.org 𝑷=wiki 𝑷=སའི་གོ་ལ།} +https://bo.wikipedia.org/wiki/%E0%BD%A6%E0%BD%A0%E0%BD%B2%E0%BC%8B%E0%BD%82%E0%BD%BC%E0%BC%8B%E0%BD%A3%E0%BC%8D +https://bo.wikipedia.org/wiki/སའི་གོ་ལ%E0%BC%8D + +# {𝑺=https:// 𝑯=fiu-vro.wikipedia.org 𝑷=wiki 𝑷=Maa_(hod'otäht)} +https://fiu-vro.wikipedia.org/wiki/Maa_(hod'ot%C3%A4ht) +https://fiu-vro.wikipedia.org/wiki/Maa_(hod'otäht) + +# {𝑺=https:// 𝑯=ty.wikipedia.org 𝑷=wiki 𝑷=’Afirita} +https://ty.wikipedia.org/wiki/%E2%80%99Afirita +https://ty.wikipedia.org/wiki/’Afirita + +# {𝑺=https:// 𝑯=ab.wikipedia.org 𝑷=wiki 𝑷=Вашингтон,_Џьорџь} +https://ab.wikipedia.org/wiki/%D0%92%D0%B0%D1%88%D0%B8%D0%BD%D0%B3%D1%82%D0%BE%D0%BD,_%D0%8F%D1%8C%D0%BE%D1%80%D1%9F%D1%8C +https://ab.wikipedia.org/wiki/Вашингтон,_Џьорџь + +# {𝑺=https:// 𝑯=mni.wikipedia.org 𝑷=wiki 𝑷=ꯅ꯭ꯌꯨ_ꯌꯣꯔ꯭ꯛ_ꯁꯤꯇꯤꯒꯤ_ꯌꯨ.ꯑꯦꯁ.} +https://mni.wikipedia.org/wiki/%EA%AF%85%EA%AF%AD%EA%AF%8C%EA%AF%A8_%EA%AF%8C%EA%AF%A3%EA%AF%94%EA%AF%AD%EA%AF%9B_%EA%AF%81%EA%AF%A4%EA%AF%87%EA%AF%A4%EA%AF%92%EA%AF%A4_%EA%AF%8C%EA%AF%A8.%EA%AF%91%EA%AF%A6%EA%AF%81%2E +https://mni.wikipedia.org/wiki/ꯅ꯭ꯌꯨ_ꯌꯣꯔ꯭ꯛ_ꯁꯤꯇꯤꯒꯤ_ꯌꯨ.ꯑꯦꯁ%2E + +# {𝑺=https:// 𝑯=azb.wikipedia.org 𝑷=wiki 𝑷=واشینقتن،_دی.سی.} +https://azb.wikipedia.org/wiki/%D9%88%D8%A7%D8%B4%DB%8C%D9%86%D9%82%D8%AA%D9%86%D8%8C_%D8%AF%DB%8C.%D8%B3%DB%8C%2E +https://azb.wikipedia.org/wiki/واشینقتن،_دی.سی%2E + +# {𝑺=https:// 𝑯=mad.wikipedia.org 𝑷=wiki 𝑷=Tasè’} +https://mad.wikipedia.org/wiki/Tas%C3%A8%E2%80%99 +https://mad.wikipedia.org/wiki/Tasè%E2%80%99 + +# {𝑺=https:// 𝑯=wuu.wikipedia.org 𝑷=wiki 𝑷=聖保羅(巴西)} +https://wuu.wikipedia.org/wiki/%E8%81%96%E4%BF%9D%E7%BE%85%EF%BC%88%E5%B7%B4%E8%A5%BF%EF%BC%89 +https://wuu.wikipedia.org/wiki/聖保羅(巴西) + +# {𝑺=https:// 𝑯=vep.wikipedia.org 𝑷=wiki 𝑷=Brüssel'} +https://vep.wikipedia.org/wiki/Br%C3%BCssel%27 +https://vep.wikipedia.org/wiki/Brüssel%27 + +# {𝑺=https:// 𝑯=tw.wikipedia.org 𝑷=wiki 𝑷=Wiase_Nyinaa_Wɛbsaet_(_World_Wide_Web;_WWW_)} +https://tw.wikipedia.org/wiki/Wiase_Nyinaa_W%C9%9Bbsaet_(_World_Wide_Web;_WWW_) +https://tw.wikipedia.org/wiki/Wiase_Nyinaa_Wɛbsaet_(_World_Wide_Web;_WWW_) + +# {𝑺=https:// 𝑯=ja.wikibooks.org 𝑷=wiki 𝑷=植物学 𝑷=植物とはどのような生き物か?} +https://ja.wikibooks.org/wiki/%E6%A4%8D%E7%89%A9%E5%AD%A6/%E6%A4%8D%E7%89%A9%E3%81%A8%E3%81%AF%E3%81%A9%E3%81%AE%E3%82%88%E3%81%86%E3%81%AA%E7%94%9F%E3%81%8D%E7%89%A9%E3%81%8B%EF%BC%9F +https://ja.wikibooks.org/wiki/植物学/植物とはどのような生き物か%EF%BC%9F + +# {𝑺=https:// 𝑯=bn.wikibooks.org 𝑷=wiki 𝑷=উইকিশৈশব:দেশসমূহ_(অ-হ) 𝑷=ইসরায়েল} +https://bn.wikibooks.org/wiki/%E0%A6%89%E0%A6%87%E0%A6%95%E0%A6%BF%E0%A6%B6%E0%A7%88%E0%A6%B6%E0%A6%AC:%E0%A6%A6%E0%A7%87%E0%A6%B6%E0%A6%B8%E0%A6%AE%E0%A7%82%E0%A6%B9_(%E0%A6%85-%E0%A6%B9)/%E0%A6%87%E0%A6%B8%E0%A6%B0%E0%A6%BE%E0%A6%AF%E0%A6%BC%E0%A7%87%E0%A6%B2 +https://bn.wikibooks.org/wiki/উইকিশৈশব:দেশসমূহ_(অ-হ)/ইসরায়েল + +# {𝑺=https:// 𝑯=haw.wikipedia.org 𝑷=wiki 𝑷=Puke_noi‘i_kū‘ikena} +https://haw.wikipedia.org/wiki/Puke_noi%E2%80%98i_k%C5%AB%E2%80%98ikena +https://haw.wikipedia.org/wiki/Puke_noi‘i_kū‘ikena + +# {𝑺=https:// 𝑯=new.wikipedia.org 𝑷=wiki 𝑷=विन्सेन्ट_भ्यान_ग:} +https://new.wikipedia.org/wiki/%E0%A4%B5%E0%A4%BF%E0%A4%A8%E0%A5%8D%E0%A4%B8%E0%A5%87%E0%A4%A8%E0%A5%8D%E0%A4%9F_%E0%A4%AD%E0%A5%8D%E0%A4%AF%E0%A4%BE%E0%A4%A8_%E0%A4%97%3A +https://new.wikipedia.org/wiki/विन्सेन्ट_भ्यान_ग%3A + +# {𝑺=https:// 𝑯=he.wikipedia.org 𝑷=wiki 𝑷=נאט"ו} +https://he.wikipedia.org/wiki/%D7%A0%D7%90%D7%98%22%D7%95 +https://he.wikipedia.org/wiki/נאט"ו + +# {𝑺=https:// 𝑯=ks.wikipedia.org 𝑷=wiki 𝑷=شارٕک۔} +https://ks.wikipedia.org/wiki/%D8%B4%D8%A7%D8%B1%D9%95%DA%A9%DB%94 +https://ks.wikipedia.org/wiki/شارٕک%DB%94 + +# {𝑺=https:// 𝑯=zh.wikipedia.org 𝑷=wiki 𝑷=联合国教育、科学及文化组织} +https://zh.wikipedia.org/wiki/%E8%81%94%E5%90%88%E5%9B%BD%E6%95%99%E8%82%B2%E3%80%81%E7%A7%91%E5%AD%A6%E5%8F%8A%E6%96%87%E5%8C%96%E7%BB%84%E7%BB%87 +https://zh.wikipedia.org/wiki/联合国教育、科学及文化组织 + +# {𝑺=https:// 𝑯=am.wikipedia.org 𝑷=wiki 𝑷=«የሰብዓዊ_መብት_አቀፋዊ_መግለጽ»} +https://am.wikipedia.org/wiki/%C2%AB%E1%8B%A8%E1%88%B0%E1%89%A5%E1%8B%93%E1%8B%8A_%E1%88%98%E1%89%A5%E1%89%B5_%E1%8A%A0%E1%89%80%E1%8D%8B%E1%8B%8A_%E1%88%98%E1%8C%8D%E1%88%88%E1%8C%BD%C2%BB +https://am.wikipedia.org/wiki/«የሰብዓዊ_መብት_አቀፋዊ_መግለጽ%C2%BB + +# {𝑺=https:// 𝑯=bo.wikipedia.org 𝑷=wiki 𝑷=༼མ་ཧ་བ་ར་ཏ།༽} +https://bo.wikipedia.org/wiki/%E0%BC%BC%E0%BD%98%E0%BC%8B%E0%BD%A7%E0%BC%8B%E0%BD%96%E0%BC%8B%E0%BD%A2%E0%BC%8B%E0%BD%8F%E0%BC%8D%E0%BC%BD +https://bo.wikipedia.org/wiki/༼མ་ཧ་བ་ར་ཏ།༽ + +# {𝑺=https:// 𝑯=am.wikipedia.org 𝑷=wiki 𝑷=አፈ፡ታሪክ} +https://am.wikipedia.org/wiki/%E1%8A%A0%E1%8D%88%E1%8D%A1%E1%89%B3%E1%88%AA%E1%8A%AD +https://am.wikipedia.org/wiki/አፈ፡ታሪክ + +# {𝑺=https:// 𝑯=it.wikibooks.org 𝑷=wiki 𝑷=Questo_è_l'ebraismo!} +https://it.wikibooks.org/wiki/Questo_%C3%A8_l'ebraismo%21 +https://it.wikibooks.org/wiki/Questo_è_l'ebraismo%21 + +# {𝑺=https:// 𝑯=my.wikipedia.org 𝑷=wiki 𝑷=ခို၊_ချိုးနှင့်_အလားတူငှက်များ} +https://my.wikipedia.org/wiki/%E1%80%81%E1%80%AD%E1%80%AF%E1%81%8A_%E1%80%81%E1%80%BB%E1%80%AD%E1%80%AF%E1%80%B8%E1%80%94%E1%80%BE%E1%80%84%E1%80%B7%E1%80%BA_%E1%80%A1%E1%80%9C%E1%80%AC%E1%80%B8%E1%80%90%E1%80%B0%E1%80%84%E1%80%BE%E1%80%80%E1%80%BA%E1%80%99%E1%80%BB%E1%80%AC%E1%80%B8 +https://my.wikipedia.org/wiki/ခို၊_ချိုးနှင့်_အလားတူငှက်များ + From 1bdbf54b697cc0b0d7470c7e08d148e9d1d26910 Mon Sep 17 00:00:00 2001 From: Mark Davis Date: Sat, 27 Jun 2026 17:13:30 +0200 Subject: [PATCH 2/8] ICU-23431 Moved test files, fleshing out tests WIP --- .../com/ibm/icu/dev/test/links/LinkTest.java | 86 ++++++++++++++++--- .../links/LinkDetectionTest.txt | 0 .../links/LinkFormattingTest.txt | 0 3 files changed, 76 insertions(+), 10 deletions(-) rename icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/{testdata => unicode}/links/LinkDetectionTest.txt (100%) rename icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/{testdata => unicode}/links/LinkFormattingTest.txt (100%) diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java index 063dcd429a90..3b9101a843ed 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java @@ -1,8 +1,12 @@ package com.ibm.icu.dev.test.links; import com.ibm.icu.dev.test.CoreTestFmwk; +import com.ibm.icu.dev.test.TestUtil; +import com.ibm.icu.impl.locale.XCldrStub.Splitter; import com.ibm.icu.util.LinkUtilities; import com.ibm.icu.util.LinkUtilities.Extent; +import java.io.BufferedReader; +import java.io.IOException; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -39,15 +43,77 @@ public void testSimpleScanBackEmail() { assertEquals(source, expected, actual); } - // @Test - // public void testAgainstFiles() { - // // for later - // try (Stream lines = Files.lines(Paths.get("file.txt"))) { - // lines.forEach(dosomething); - // } catch (IOException e) { - // e.printStackTrace(); - // } - // - // } + @Test + public void testAgainstFiles() { + // # All parts + // # {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δεζ 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γξο 𝑭=πρς} + // + // https://example.com/%CE%B1%CE%B2%CE%B3/%CE%B4%CE%B5%CE%B6?%CE%B8=%CE%B9%CE%BA%CE%BB&%CE%BC=%CE%B3%CE%BE%CE%BF#%CF%80%CF%81%CF%82 + // https://example.com/αβγ/δεζ?θ=ικλ&μ=γξο#πρς + + String expectedPrefix = "https://example.com"; // for now, all the same + try (BufferedReader idnaTestFile = + TestUtil.getUtf8DataReader("unicode/links/LinkFormattingTest.txt"); ) { + String pqf = null; + boolean firstLine = true; + for (String line : (Iterable) idnaTestFile.lines()::iterator) { + if (line.startsWith("# {")) { + pqf = assemblePqf(line); + firstLine = true; + } else if (line.isBlank() || line.startsWith("#")) { + continue; + } else if (!line.startsWith(expectedPrefix)) { + throw new IllegalArgumentException("Expand the test to cover other S/H"); + } else if (firstLine) { + String actual = + expectedPrefix + + LinkUtilities.escapePathQueryFragment(pqf, Extent.MAXIMAL); + assertEquals("maximum " + pqf, line, actual); + firstLine = false; + } else { + String actual = + expectedPrefix + + LinkUtilities.escapePathQueryFragment(pqf, Extent.MINIMAL); + assertEquals("maximum " + pqf, line, actual); + firstLine = false; + } + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + private String assemblePqf(String line) { + line = line.substring(3, line.length() - 1); // subtract the # { and the } + StringBuilder sb = new StringBuilder(); + char query = '?'; + for (String item : Splitter.on(' ').splitToList(line)) { + int eq = item.indexOf('='); + String type = item.substring(0, eq); + String rest = item.substring(eq + 1); + // # {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δεζ 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γξο 𝑭=πρς} + switch (type) { + case "𝑺": + case "𝑯": + break; + case "𝑷": + sb.append('/').append(rest); + break; + case "𝑸": + sb.append(query).append(rest); + query = '&'; + break; + case "𝑽": + sb.append('=').append(rest); + break; + case "𝑭": + sb.append('#').append(rest); + break; + default: + throw new IllegalArgumentException("Unexpected line: " + item); + } + } + return sb.toString(); + } } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkDetectionTest.txt b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/unicode/links/LinkDetectionTest.txt similarity index 100% rename from icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkDetectionTest.txt rename to icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/unicode/links/LinkDetectionTest.txt diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkFormattingTest.txt b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/unicode/links/LinkFormattingTest.txt similarity index 100% rename from icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/testdata/links/LinkFormattingTest.txt rename to icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/unicode/links/LinkFormattingTest.txt From cb1ea2bf2463029ad8268a995e8ad1afa6e2c5a3 Mon Sep 17 00:00:00 2001 From: Mark Davis Date: Wed, 1 Jul 2026 21:41:21 +0300 Subject: [PATCH 3/8] ICU-23431 More progress but not done --- .../icu/impl/links/LinkHandlingUtilities.java | 18 +- .../java/com/ibm/icu/util/LinkUtilities.java | 2 +- .../com/ibm/icu/dev/test/links/LinkTest.java | 361 +++++++++++++++++- 3 files changed, 371 insertions(+), 10 deletions(-) diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java index e99420751518..c498a99a4fbe 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java @@ -83,7 +83,7 @@ private static String quote(String s) { /** TODO use real property */ private static int getOpening(int cp) { - return cp == '>' ? '<' : UCharacter.getIntPropertyValue(cp, UProperty.BIDI_PAIRED_BRACKET); + return cp == '>' ? '<' : UCharacter.getBidiPairedBracket(cp); } /** Defines the LinkTermination property TODO use real property */ @@ -159,7 +159,19 @@ public static int scanEmailBackwards(CharSequence source, int hardStart, int dom throw new IllegalArgumentException("Scanning must start after an '@' sign"); } int result = validEmailLocalPart.spanBack(source, domainStart - 1, SpanCondition.SIMPLE); - return result >= hardStart ? result : domainStart - 1; + if (result == domainStart - 1) { + return domainStart; + } else if (result < hardStart) { + result = hardStart; + } + String localPart = source.subSequence(result, domainStart - 1).toString(); + if (localPart.startsWith(".") || localPart.endsWith(".") || localPart.contains("..")) { + return domainStart; + } + if (source.toString().substring(0, result).endsWith("mailto:")) { + result -= "mailto:".length(); + } + return result; } private static String getGeneralCategory(int property, int codePoint, int nameChoice) { @@ -912,7 +924,7 @@ private static void appendPercentEscaped(StringBuilder output, String source) { byte[] bytes = source.getBytes(StandardCharsets.UTF_8); for (int i = 0; i < bytes.length; ++i) { output.append('%'); - output.append(Utility.hex(bytes[i], 2)); + output.append(Utility.hex(bytes[i] & 0xFF, 2)); } } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java index 74eda248f349..b944ab2f06c5 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java @@ -32,7 +32,7 @@ public static int scanPathQueryFragment(CharSequence source, int start, int limi * Lower level utility for finding the start of an email address in text. It assumes that the * limit position is immediately before an '@' + identified domain name. The purpose of this * routine is for fitting into algorithms that are already in use, just handling for the email - * `local-part`. + * `local-part`. It does not scan back through "mailto:". * * @param source the text to be scanned * @param start the position that is the earliest that should be considered in a backwards scan diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java index 3b9101a843ed..cdde9458efdb 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java @@ -3,24 +3,69 @@ import com.ibm.icu.dev.test.CoreTestFmwk; import com.ibm.icu.dev.test.TestUtil; import com.ibm.icu.impl.locale.XCldrStub.Splitter; +import com.ibm.icu.lang.UCharacter; +import com.ibm.icu.text.IDNA; +import com.ibm.icu.text.UnicodeSet; +import com.ibm.icu.text.UnicodeSet.SpanCondition; import com.ibm.icu.util.LinkUtilities; import com.ibm.icu.util.LinkUtilities.Extent; import java.io.BufferedReader; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +/** + * Test the APIs in LinkUtilties using some basic tests, plus the test data from UTS58 Note: the + * UTC58 tests check whole links, since that is easiest for implementers to write tests against.
+ * However, the Unicode testdata currently takes certain positions on edge cases that some client + * software might not agree with. A simple case is that it only recognizes TLDs that are valid, not + * just well formed. It mostly handles this by not using invalid TLDs. So it doesn't use + * "example.somefuturetld" as a test case. But it does take some positions that theoretically + * clients might disagree with, such as not recognizing "efg.com" in "abc..efg.com". This currently + * tests with code that duplicates that.
+ * There are also some edge cases + * + * + * + * + * + *
TextApproach
dav.is@dav.is/dav.is + * Note that a string may both be a valid local-part and a valid domain name. + * The UTS58 test data recognizes the start of a link; + * recognizing the email address (dav.is@dav.is), and skipping the PDF (/dav.is). + *
mailto:αβγ..δεζ@example.com + * A failing email local-part (αβγ..δεζ) fails as a whole (eg δεζ@example.com is not recognized) + *
+ */ @RunWith(JUnit4.class) public class LinkTest extends CoreTestFmwk { /** Constructor */ public LinkTest() {} + @Test + public void test1PairedBracket() { + // int actual = // UCharacter.getIntPropertyValue(')', UProperty.BIDI_PAIRED_BRACKET); + int actual = UCharacter.getBidiPairedBracket(')'); + assertEquals("", '(', actual); + } + @Test public void testSimpleScan() { - String source = "/αβγ?def#ghi "; - int expected = source.indexOf(" "); + checkScan("/α(β) on…", "α(β)"); + checkScan("/αβγ?def#ghi ", "ghi"); + // checkScan("⸠https://ja.wikipedia.org/wiki/フィンセント・ファン・ゴッホ", "ゴッホ"); + } + + private void checkScan(String toCheck, String expectedAfter) { + String source = toCheck; + int expected = source.indexOf(expectedAfter) + expectedAfter.length(); int actual = LinkUtilities.scanPathQueryFragment(source, 0, source.length()); assertEquals(source, expected, actual); } @@ -35,16 +80,187 @@ public void testSimpleEscape() { @Test public void testSimpleScanBackEmail() { - String source = "See αβγ@example.com"; - int start = 0; + checkEmail("See αβγ@", 0, "αβγ"); + checkEmail("See mailto:αβγ@", 0, "mailto:α"); + checkEmail("See mailto:αβγ.δεζ.@", 0, null); + checkEmail("See mailto:αβγ.δεζ@", 0, "mailto:α"); + } + + private void checkEmail(String source, int start, String localPartStart) { int end = source.indexOf('@') + 1; - int expected = source.indexOf("αβγ"); + int expected = localPartStart == null ? source.length() : source.indexOf(localPartStart); int actual = LinkUtilities.scanBackEmailLocalPart(source, start, end); assertEquals(source, expected, actual); } + @Test + public void testHackDomainName() { + String source = // "普遍适用测试。我爱你"; + "موريتانيا.xn-----ctdbabcfhu9c2b9l1acccr4c"; + IntRange domain = new IntRange(); + boolean actual = hackFindDomainName(source, 0, source.length(), false, domain); + assertEquals(source, true, actual); + assertTrue(source, domain.start == 0 && domain.end == source.length()); + } + + // Would like to use the following for JUnit, but can't + // @ParameterizedTest + // @MethodSource("testItemsProvider") + + Pattern SCHEME = Pattern.compile("https?://$"); + Pattern PORT = Pattern.compile(":\\d+"); + + final Set skip = Set.of("See http://-foo.example-.com. on…"); @Test - public void testAgainstFiles() { + public void testAgainstLinkDetectionTest() { + List failures = new ArrayList<>(); + try (BufferedReader idnaTestFile = + TestUtil.getUtf8DataReader("unicode/links/LinkDetectionTest.txt"); ) { + for (final String line : (Iterable) idnaTestFile.lines()::iterator) { + if (line.startsWith("#") || line.isBlank() || skip.contains(line)) { + continue; + } + StringBuilder actual = new StringBuilder(); + String rawLine2 = line.replace("⸠", "").replace("⸡", ""); + int start = 0; + IntRange domain = new IntRange(); + IntRange domainAtDomain = new IntRange(); + try { + while (hackFindDomainName(rawLine2, start, rawLine2.length(), false, domain)) { + boolean haveEmail = false; + // check for '@' before the domain name + // if there is one, we see if there is an email address. If not, + // we skip over the domain name + if (domain.start > 0 + && UCharacter.codePointBefore(rawLine2, domain.start) == '@') { + // handle email + // check backwards for email + int local_part_start = + LinkUtilities.scanBackEmailLocalPart( + rawLine2, start, domain.start); + if (local_part_start != domain.start) { + domain.start = local_part_start; + haveEmail = true; + } else { + // we skip over the domain name + actual.append(rawLine2.subSequence(start, domain.end)); + start = domain.end; + continue; + } + + } else { // we could have chanced on a case where the domain name is also + // a local_part (like john.uk@foo.com) + // For that case we check if there is an @ sign *after* the end of the + // domain name. + // If so: + // - we scan forwards (after the @) to see if there is a domain name, + // AND + // - we scan backwards (before the @) to see if there is local_part, + // AND + // - we scan backwards before that to ensure that there is no https?:// + // if both are true, we have an email address. + // if only the first is true, we skip over the 2nd domain name; + // ie, we skip over " @example.com" + if (domain.end < rawLine2.length() + && UCharacter.codePointAt(rawLine2, domain.end) == '@') { + if (hackFindDomainName( + rawLine2, + domain.end + 1, + rawLine2.length(), + true, + domainAtDomain)) { + domain.start = + LinkUtilities.scanBackEmailLocalPart( + rawLine2, start, domain.end + 1); + boolean failing = domain.start == domain.end + 1; + if (!failing) { + // slash is a valid email character, so check + // two characters after + Matcher schemeMatcher = SCHEME.matcher(rawLine2); + schemeMatcher.region(start, domain.start + 2); + failing = schemeMatcher.find(); + } + if (failing) { + // failed to find local part, or found scheme + // we failed, so copy up to here so we don't rescan + actual.append( + rawLine2.subSequence(start, domainAtDomain.end)); + start = domainAtDomain.end; + continue; + } + if (UCharacter.codePointBefore(rawLine2, domainAtDomain.end) + == '.') { + // backup; might be end of sentence + domainAtDomain.end--; + } + domain.end = domainAtDomain.end; + haveEmail = true; + } + } + } + // we only check for PQF if we don't have an email address + if (!haveEmail) { + // handle PQF + // check backwards for scheme "[a-z]://" + Matcher schemeMatcher = SCHEME.matcher(rawLine2); + schemeMatcher.region(start, domain.start); + if (schemeMatcher.find()) { + domain.start = schemeMatcher.start(); + } else { + domain.start = domain.start; + } + + Matcher portMatcher = PORT.matcher(rawLine2); + portMatcher.region(domain.end, rawLine2.length()); + if (portMatcher.lookingAt()) { + domain.end = portMatcher.end(); + } + + // check forwards for PQF + int pqfEnd = + LinkUtilities.scanPathQueryFragment( + rawLine2, domain.end, rawLine2.length()); + if (pqfEnd == domain.end) { + // special case dot at end of domain name and no PQF + if (UCharacter.codePointBefore(rawLine2, domain.end) == '.') { + // backup; might be end of sentence + pqfEnd--; + } + } + domain.end = pqfEnd; + } + if (start > domain.start) { + int debug = 0; + } + actual.append(rawLine2.subSequence(start, domain.start)) + .append("⸠") + .append(rawLine2.subSequence(domain.start, domain.end)) + .append("⸡"); + start = domain.end; + } + actual.append(rawLine2.substring(start)); + + assertEquals("", line, actual.toString()); + System.out.println("OK\t" + line); + } catch (AssertionError e) { + failures.add("FAIL " + e.getMessage()); + System.out.println("FAIL\t" + line + "\t " + actual.toString()); + } + } + if (!failures.isEmpty()) { + fail( + "Test finished with " + + failures.size() + + " failures:\n" + + String.join("\n", failures)); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testAgainstLinkFormattingTest() { // # All parts // # {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δεζ 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γξο 𝑭=πρς} // @@ -116,4 +332,137 @@ private String assemblePqf(String line) { } return sb.toString(); } + + class IntRange { + int start; + int end; + + @Override + public String toString() { + return start + ";" + end; + } + } + + Pattern DOT = Pattern.compile("[.。]"); + Pattern DOT2 = Pattern.compile("[.。][.。]"); + /** + * The UTS58 tests take whole domain names, because that is the easiest for clients to test. We + * have to hack it because ICU IDN support doesn't provide enough API. Returns true if found, + * with the range in result. + * + * @param mustBeAtStart + * TODO check with isValidDomain + */ + boolean hackFindDomainName( + CharSequence source, int start, int limit, boolean mustBeAtStart, IntRange result) { + if (mustBeAtStart && start < limit) { + int firstCodePoint = UCharacter.codePointAt(source, start); + if (!HACK_DOMAIN_SET.contains(firstCodePoint)) { + return false; + } + } + Matcher dot = DOT.matcher(source); + Matcher dot2 = DOT2.matcher(source); + while (start < limit) { + int endNot = HACK_DOMAIN_SET.span(source, start, SpanCondition.NOT_CONTAINED); + if (endNot > limit) { + return false; + } + int end = HACK_DOMAIN_SET.span(source, endNot, SpanCondition.SIMPLE); + if (end > limit) { + end = limit; + } + if (end == endNot) { + return false; + } + // Check that the possible domain name (endNot..end) is of the right form + // otherwise continue after end. + // Must not start with [.。], and must find a [.。] before the end + dot.region(endNot, end); + boolean found = dot.find(); + if (!found || dot.start() == endNot || dot.end() == end) { + start = end; + continue; + } + // Must not find [.。][.。] + dot2.region(endNot, end); + if (dot2.find()) { + start = end; + continue; + } + result.start = endNot; + result.end = end; + return true; + } + return false; + } + + private final UnicodeSet HACK_DOMAIN_SET = + new UnicodeSet( + "[。.\\-0-9a-z·ß-öø-ÿāăą ćĉċčďđēĕėęěĝğġģĥħĩ" + + "īĭį ıĵķĸĺļľłńņňŋōŏőœŕŗřśŝ şšţťŧũūŭůűųŵŷźżžƀƃƅƈƌ ƍƒƕƙ-ƛƞơƣƥƨƪƫƭưƴƶƹ-ƻƽ-ǃ ǎǐ" + + "ǒǔǖǘǚǜǝǟǡǣǥǧǩǫǭǯǰǵǹ ǻǽǿȁȃȅȇȉȋȍȏȑȓȕȗșțȝȟȡȣ ȥȧȩȫȭȯȱȳ-ȹȼȿɀɂɇɉɋɍɏ-ʯ ʹ-ˁˆ-ˑˬˮ̀-̿͂͆-͎͐-ͯͱͳ͵ ͷ-" + + "͹ ͻ-ͽ΀-΃΋΍ ΐ΢ ά-ώϗϙϛϝϟϡϣϥϧϩϫϭϯϳϸϻϼа-џ ѡѣѥѧѩѫѭѯѱѳѵѷѹѻѽѿҁ҃-҇ҋ ҍҏґғҕҗҙқҝҟҡңҥҧҩҫ" + + "ҭүұҳҵ ҷҹһҽҿӂӄӆӈӊӌӎӏӑӓӕӗәӛӝӟ ӡӣӥӧөӫӭӯӱӳӵӷӹӻӽӿԁԃԅԇԉ ԋԍԏԑԓԕԗԙԛԝԟԡԣԥԧԩԫԭԯ԰՗-ՙ ՠ-ֆֈ" + + "֋֌֐-ׇֽֿׁׂׅׄ-׿ؐ-ؚ ؠ-ؿف-٩ٮ-ٴٹ-ۓە-ۜ۟-۪ۨ-ۿ܎ ܐ-ߵ߻-߽ ࠀ-࠯࠿-࡝࡟-ࢇ ࢉ-࢏࢒-ࣣ࣡-ॗ ॠ-ॣ०-९ॱ-৛৞ ৠ-ৱৼ" + + "৾-ਲ਴ ਵ਷-੘ ੜ੝੟-ੵ੷-૯૲-୛୞-୯ ୱ୸-௯௻-౶ ಀ-ಃಅ-ൎ൐-ൗ ൟ-൯ൺ-ෳ෵-าิ-฾ เ-๎๐-๙๜-າິ-" + + "໛ ໞ-ༀ་༘༙༠-༩༹༵༷༾-གང-ཌཎ-ད ན-བམ-ཛཝ-ཨཪ-ིེུ-ྀྂ-྄྆-ྒྔ-ྜྞ-ྡྣ-ྦྨ-ྫྭ-ྸྺ-྽࿆࿍࿛-၉ ၐ-ႝ჆჈-჌჎-ჺ ჽ-ჿሀ-፟፽-ᎏ᎚-᏷᏾᏿ ᐁ-" + + "ᙬᙯ-ᙿᚁ-ᚚ᚝-ᛪ ᛱ-᜴᜷-ឳា-៓ ៗៜ-៯៺-៿ ᠐-᤿᥁-᥃ ᥆-᧙᧛-᧝ ᨀ-᨝ ᨠ-᪟ ᪧ᪮-᪽ᪿ-᭍ ᭐-᭙᭫-᭳ᮀ-᯻ " + + "ᰀ-᰺ ᱀-ᱽᲊ-᲏᲻᲼᳈-᳔᳒-ᴫ ᴯᴻᵎᵫ-ᵷᵹ-ᶚ᷀-᷿ḁḃḅḇḉḋḍḏḑ ḓḕḗḙḛḝḟḡḣḥḧḩḫḭḯḱḳḵḷḹḻ ḽḿṁṃṅṇṉṋṍṏṑṓṕṗ" + + "ṙṛṝṟṡṣṥ ṧṩṫṭṯṱṳṵṷṹṻṽṿẁẃẅẇẉẋẍẏ ẑẓẕ-ẙẜẝẟạảấầẩẫậắằẳẵặẹ ẻẽếềểễệỉịọỏốồổỗộớờởỡợ ụủứừử" + + "ữựỳỵỷỹỻỽỿ-ἇἐ-἗἞-ἧ ἰ-ἷὀ-὇὎-὘὚὜὞ ὠ-ὧὰὲὴὶὸὺὼ὾὿ ᾰᾱ᾵ ᾶ῅ ῆῐ-ῒ῔-ῗ῜ ῠ-ῢῤ-ῧ" + + "῰῱῵ ῶ῿ ‌‍⁥⁲⁳₏₝-₟⃂-⃏⃱-⃿ ⅎↄ↌-↏␪-␿⑋-⑟⭴⭵ ⰰ-ⱟⱡⱥⱦⱨⱪⱬⱱⱳⱴⱶ-ⱻⲁⲃⲅⲇⲉⲋ ⲍⲏⲑⲓⲕⲗ" + + "ⲙⲛⲝⲟⲡⲣⲥⲧⲩⲫⲭⲯⲱⲳⲵ ⲷⲹⲻⲽⲿⳁⳃⳅⳇⳉⳋⳍⳏⳑⳓⳕⳗⳙⳛⳝⳟ ⳡⳣⳤⳬⳮ-⳱ⳳ-⳸ ⴀ-⵮⵱-ⷿ ⸯ⹞-⹿⺚⻴-⻿⿖-⿯ 々-〇〪-〭" + + "〼぀-゚ ゝゞァ-ヾ㄀-㄰㆏ ㆠ-ㆿ㇦-㇮ ㇰ-ㇿ㈟ 㐀-䶿一-꒏꓇-ꓽ ꔀ-ꘌꘐ-꘿ ꙁꙃꙅꙇꙉꙋꙍꙏꙑꙓꙕꙗꙙꙛꙝꙟꙡꙣ" + + "ꙥꙧꙩ ꙫꙭ-꙯ꙴ-꙽ꙿꚁꚃꚅꚇꚉꚋꚍꚏꚑꚓꚕꚗꚙ ꚛꚞ-ꛥ꛰꛱꛸-꛿ ꜗ-ꜟꜣꜥꜧꜩꜫꜭꜯ-ꜱꜳꜵꜷꜹꜻꜽꜿꝁꝃ ꝅꝇꝉꝋꝍꝏꝑꝓꝕꝗꝙꝛꝝꝟꝡꝣꝥꝧꝩꝫꝭ ꝯꝱ-ꝸꝺꝼꝿ" + + "ꞁꞃꞅꞇꞈꞌꞎꞏꞑꞓ-ꞕꞗꞙ ꞛꞝꞟꞡꞣꞥꞧꞩꞯꞵꞷꞹꞻꞽꞿꟁꟃꟈꟊꟍ꟏ ꟑꟓꟕꟗꟙꟛ꟝-꟰ ꟶꟷꟺ-ꠧ꠬-꠯꠺-ꡳ꡸-꣍ ꣐-ꣷꣻꣽ-꤭ꤰ-꥞꥽-꧀꧎-꧝ " + + "ꧠ-꩛ ꩠ-ꩶꩺ-ꫝꫠ-ꫯꫲ-ꭚꭠ-ꭨ꭬-꭯ ꯀ-ꯪ꯬-힯퟇-퟊퟼-퟿ 﨎﨏﨑﨓﨔﨟﨡﨣﨤﨧-﨩﩮﩯﫚-﫿﬇-﬒" + + "﬘-﬜ﬞ﬷﬽﬿﭂﭅︚-︯﹓﹧﹬-﹯ ﹳ﹵﻽﻾＀﾿-￁￈￉￐￑￘￙￝-￟￧￯-￸ 𐀀-𐃿𐄃-𐄆𐄴-𐄶𐆏𐆝" + + "-𐆟𐆡-𐇏𐇽-𐋠𐋼-𐌟𐌤-𐍀 𐍂-𐍉𐍋-𐎞 𐎠-𐏏𐏖-𐏿 𐐨-𐒯𐓔-𐕮𐕻𐖋𐖓𐖖-𐞀𐞆𐞱𐞻-𐡖 𐡠-𐡶𐢀-𐢦𐢰-" + + "𐣺 𐤀-𐤕𐤜-𐤞 𐤠-𐤾 𐥀-𐦻 " + + "𐦾𐦿𐧐𐧑 𐨀-𐨿𐩉-𐩏𐩙-𐩼 " + + "𐪀-𐪜𐪠-𐫇 𐫉-𐫪𐫷-𐬸 𐭀-" + + "𐭗 𐭠-𐭷 𐮀-𐮘𐮝-𐮨𐮰" + + "-𐱿𐲳-𐳹 𐴀-𐵏𐵦-𐵭 𐵯-" + + "𐶍𐶐-𐹟𐹿-𐺬𐺮-𐻏𐻙-𐼜" + + " 𐼧-𐽐𐽚-𐾅𐾊-𐿄𐿌-𑁆𑁎-𑁑 𑁦-𑂺𑃂-𑃌𑃎-𑄿 𑅄-𑅳𑅶-𑇄𑇉-𑇌𑇎-𑇚𑇜𑇠𑇵-𑈷𑈾-𑊨𑊪-𑏓𑏖𑏙-𑑊 𑑐-𑑙𑑜𑑞-" + + "𑓅 𑓇-𑗀𑗘-𑙀𑙄-𑙟𑙭-𑚸𑚺-𑜹 𑝀-𑠺𑠼-𑢟 𑣀-𑣩𑣳-𑥃𑥇-𑧡 𑧣-𑨾𑩇-𑪙𑪝𑪣-𑫿𑬊-𑯠𑯢-𑱀𑱆-𑱙𑱭-𑱯 " + + "𑱲-𑻶𑻹-𑽂 𑽐-𑾿𑿲-𑿾 𒀀-𒏿𒑯𒑵-𒿰𒿳-𓐯𓑀-𖩭 𖩰-𖫴𖫶-𖬶 𖭀-𖭃𖭆-𖭚𖭢-𖵬 𖵰-𖸿 𖹠-𖹿𖺛-𖺟𖺹-𖿡 " + + "𖿣-𖿳𖿷-𛲛𛲝𛲞𛲤-𜯿𜳽-𜳿𜺴-𜺹𜻑-𜻟𜻱-𜽏𜿄-𜿿𝃶-𝃿𝄧𝄨𝇫-𝇿𝉆-𝊿𝋔-𝋟𝋴-𝋿𝍗-𝍟𝍹-𝏿𝑕𝒝𝒠𝒡𝒣𝒤" + + "𝒧𝒨𝒭𝒺𝒼𝓄𝔆𝔋𝔌𝔕𝔝𝔺𝔿𝕅𝕇-𝕉𝕑𝚦𝚧𝟌𝟍𝨀-𝨶𝨻-𝩬𝩵𝪄𝪌-𞀯𞁮-𞅎𞅐-𞋾𞌀-𞗾𞘀-𞣆𞣐-𞣿 𞤢-𞥝𞥠-𞱰𞲵-𞴀𞴾-𞷿𞸄𞸠𞸣𞸥𞸦𞸨𞸳𞸸𞸺𞸼-𞹁𞹃-𞹆𞹈𞹊𞹌𞹐𞹓𞹕𞹖𞹘𞹚𞹜𞹞𞹠𞹣𞹥𞹦𞹫𞹳𞹸𞹽𞹿𞺊𞺜" + + "-𞺠𞺤𞺪𞺼-𞻯𞻲-𞿿🀬-🀯🂔-🂟🂯🂰🃀🃐🃶-🃿🆮-🇥🈃-🈏🈼-🈿🉉-🉏🉒-🉟🉦-🋿🛙-🛛🛭-🛯🛽-🛿🟚-🟟🟬-🟯🟱-" + + "🟿🠌-🠏🡈-🡏🡚-🡟🢈-🢏🢮🢯🢼-🢿🣂-🣏🣙-🣿🩘-🩟🩮🩯🩽-🩿🪋-🪍🫇🫉-🫌🫝🫞🫫-🫮🫹-🫿🮓🯻-🿽 𠀀-𯟿𯨞-𯿽" + + " 𰀀-𿿽񀀀-񏿽񐀀-񟿽񠀀-񯿽񰀀-񿿽򀀀-򏿽򐀀-򟿽򠀀-򯿽򰀀-򿿽󀀀-󏿽󐀀-󟿽󠀀󠀂-󠀟󠂀-󠃿󠇰-󯿽]") + .closeOver(UnicodeSet.CASE_INSENSITIVE) + .freeze(); + + static boolean isValidDomain(String domainName) { + if (domainName == null || domainName.isEmpty()) { + return false; + } + + // Initialize the UTS#46 standard IDNA instance + IDNA idna = IDNA.getUTS46Instance( + IDNA.NONTRANSITIONAL_TO_ASCII | + IDNA.NONTRANSITIONAL_TO_UNICODE | + IDNA.CHECK_BIDI | + IDNA.CHECK_CONTEXTJ | + IDNA.CHECK_CONTEXTO | + IDNA.USE_STD3_RULES + ); + + StringBuilder dest = new StringBuilder(); + IDNA.Info info = new IDNA.Info(); + + try { + // Perform name-to-ASCII conversion/validation + idna.nameToASCII(domainName, dest, info); + } catch (IllegalArgumentException e) { + // Fails on severe structural issues + return false; + } + + // Return true only if no errors were encountered during validation + return !info.hasErrors(); + } } From cfd2d4a77c48fd23ec0d92f51893b2c61e6e2562 Mon Sep 17 00:00:00 2001 From: Mark Davis Date: Fri, 3 Jul 2026 12:42:32 +0300 Subject: [PATCH 4/8] ICU-23431 Fixed remaining test failures --- .../icu/impl/links/LinkHandlingUtilities.java | 18 +- .../com/ibm/icu/impl/locale/XCldrStub.java | 14 +- .../java/com/ibm/icu/util/LinkUtilities.java | 10 +- .../com/ibm/icu/dev/test/links/LinkTest.java | 257 +++++++++++++----- 4 files changed, 229 insertions(+), 70 deletions(-) diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java index c498a99a4fbe..7e1188ecb616 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java @@ -3,6 +3,7 @@ import com.ibm.icu.impl.IDNA2003; import com.ibm.icu.impl.UnicodeMap; import com.ibm.icu.impl.Utility; +import com.ibm.icu.impl.links.LinkHandlingUtilities.UrlInternals; import com.ibm.icu.impl.locale.XCldrStub; import com.ibm.icu.impl.locale.XCldrStub.Splitter; import com.ibm.icu.lang.UCharacter; @@ -17,6 +18,7 @@ import java.nio.charset.StandardCharsets; import java.util.EnumMap; import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -916,7 +918,7 @@ private static void appendPercentEscaped(StringBuilder output, int cp) { byte[] bytes = Character.toString(cp).getBytes(StandardCharsets.UTF_8); for (int i = 0; i < bytes.length; ++i) { output.append('%'); - output.append(Utility.hex(bytes[i], 2)); + output.append(Utility.hex(bytes[i] & 0xFF, 2)); } } @@ -934,7 +936,7 @@ private static String escape(String source, String cpToEscape) { } /** percent-escape all the toEscape characters. */ - private static String escape(String source, UnicodeSet toEscape) { + public static String escape(String source, UnicodeSet toEscape) { StringBuilder result = new StringBuilder(); int lastEnd = 0; while (true) { @@ -1139,4 +1141,16 @@ private static String toUnicode2(String x) { throw new ICUException(e); } } + + public static Map>> getInternals(String source) { + UrlInternals ui = UrlInternals.from(source.toString()); + Map>> result = new LinkedHashMap<>(); + for (Part p : Part.values()) { + List> list = ui.get(p); + if (list != null) { + result.put(p.toString(), list); + } + } + return result; + } } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/impl/locale/XCldrStub.java b/icu4j/main/core/src/main/java/com/ibm/icu/impl/locale/XCldrStub.java index c57c6228ef7d..437f9300aaea 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/impl/locale/XCldrStub.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/impl/locale/XCldrStub.java @@ -136,6 +136,15 @@ public boolean equals(Object obj) { public int hashCode() { return map.hashCode(); } + + @Override + public String toString() { + return "[" + + asMap().entrySet().stream() + .map(Entry::toString) + .collect(Collectors.joining(", ")) + + "]"; + } } public static class Multimaps { @@ -183,6 +192,7 @@ public Entry next() { Entry> e = it1.next(); entry.key = e.getKey(); it2 = e.getValue().iterator(); + entry.value = it2.next(); } return entry; } @@ -199,8 +209,8 @@ public void remove() { } private static class ReusableEntry implements Entry { - K key; - V value; + private K key; + private V value; @Override public K getKey() { diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java index b944ab2f06c5..dd51149a8f01 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java @@ -73,8 +73,8 @@ public static String escapePathQueryFragment(String source, Extent extent) { } } - // NOTE for reviewers: These are only temporary, until ICU supplies \p{Link_Term=hard} and - // \p{Idn_Status≠disallowed} + // NOTE for reviewers: These are only temporary, until ICU supplies \p{Link_Term=X} and + // \p{Idn_Status=X} /** * Returns a frozen set of Unicode characters that are guaranteed to never be part of a URL or @@ -82,7 +82,10 @@ public static String escapePathQueryFragment(String source, Extent extent) { * email addresses can never span these characters. For example, a span of characters between * safe characters that doesn't have a sequence of domain-character + . + domain-character can * be skipped in processing. + * + * @internal */ + @Deprecated public static UnicodeSet getSafeCharacters() { return null; } @@ -92,7 +95,10 @@ public static UnicodeSet getSafeCharacters() { * (pre-mapping) This allows implementations to make various optimizations because URLs and * email addresses must contain a sequence of domain-character + . + domain-character. It is the * same as the set of IDNA Mapping Table character with values ≠ disallowed + * + * @internal */ + @Deprecated public static UnicodeSet getDomainCharacters() { return null; } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java index cdde9458efdb..881d7981510e 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java @@ -2,7 +2,7 @@ import com.ibm.icu.dev.test.CoreTestFmwk; import com.ibm.icu.dev.test.TestUtil; -import com.ibm.icu.impl.locale.XCldrStub.Splitter; +import com.ibm.icu.impl.links.LinkHandlingUtilities; import com.ibm.icu.lang.UCharacter; import com.ibm.icu.text.IDNA; import com.ibm.icu.text.UnicodeSet; @@ -11,6 +11,7 @@ import com.ibm.icu.util.LinkUtilities.Extent; import java.io.BufferedReader; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -46,6 +47,24 @@ @RunWith(JUnit4.class) public class LinkTest extends CoreTestFmwk { + private static final boolean SHOW_LINK_DETECTION = false; + + /** + * There are 2 format test cases that need to be fixed in UTS58 In each of these, the = in a 𝑽 + * doesn't need to be quoted, because it doesn't initiate a different component. + */ + static final Set SKIP_FORMAT_TEST = + Set.of( + "# {𝑺=https:// 𝑯=example.com 𝑸=α=β 𝑽=γ=δ}", + "# {𝑺=https:// 𝑯=example.com 𝑷=α 𝑷=b/?#c 𝑸=αβ 𝑽=γ&ζ=#Ξ 𝑸=k 𝑽=v 𝑭=frag}"); + + /** + * There is 1 detection test case that need to be removed in UTS58 UTS58 checks for valid domain + * names, and for neutrality it should avoid listing such cases + */ + static final Set SKIP_DETECTION_TEST = + Set.of("Lorem ipsum موريتانيا.xn-----ctdbabcfhu9c2b9l1acccr4c dolor sit amet"); + /** Constructor */ public LinkTest() {} @@ -72,10 +91,37 @@ private void checkScan(String toCheck, String expectedAfter) { @Test public void testSimpleEscape() { - String source = "/%CE%B1%CE%B2%CE%B3/%CE%B4.%CE%B5%CE%B6%2E"; - String expected = "/αβγ/δ.εζ%2E"; - String actual = LinkUtilities.escapePathQueryFragment(source, Extent.MINIMAL); - assertEquals(source, expected, actual); + List> tests = // source, minimal escaping, maximal escaping + List.of( + List.of( + "https://example.com/αβγ/δεζ?θ=ικλ&μ=γξο#πρς", + "https://example.com/αβγ/δεζ?θ=ικλ&μ=γξο#πρς", + "https://example.com/%CE%B1%CE%B2%CE%B3/%CE%B4%CE%B5%CE%B6?%CE%B8=%CE%B9%CE%BA%CE%BB&%CE%BC=%CE%B3%CE%BE%CE%BF#%CF%80%CF%81%CF%82"), + List.of("https://example.com/α/β%2Fγ", "https://example.com/α/β%2Fγ", ""), + List.of( + "https://example.com/α%23β?γ=δ%23ε", + "https://example.com/α%23β?γ=δ%23ε", ""), + List.of( + "ex.com/%CE%B1%CE%B2%CE%B3/%CE%B4.%CE%B5%CE%B6%2E", + "ex.com/αβγ/δ.εζ%2E", ""), + List.of( + "https://example.com/α%3Fμπ", + "", "https://example.com/%CE%B1%3F%CE%BC%CF%80")); + for (List test : tests) { + String source = test.get(0); + String expectedMin = test.get(1); + String expectedMax = test.get(2); + if (!expectedMin.isEmpty()) { + String actual = LinkUtilities.escapePathQueryFragment(source, Extent.MINIMAL); + assertEquals( + "MIN " + LinkHandlingUtilities.getInternals(source), expectedMin, actual); + } + if (!expectedMax.isEmpty()) { + String actual = LinkUtilities.escapePathQueryFragment(source, Extent.MAXIMAL); + assertEquals( + "MAX " + LinkHandlingUtilities.getInternals(source), expectedMax, actual); + } + } } @Test @@ -92,14 +138,15 @@ private void checkEmail(String source, int start, String localPartStart) { int actual = LinkUtilities.scanBackEmailLocalPart(source, start, end); assertEquals(source, expected, actual); } + @Test public void testHackDomainName() { - String source = // "普遍适用测试。我爱你"; - "موريتانيا.xn-----ctdbabcfhu9c2b9l1acccr4c"; + String source = // "普遍适用测试。我爱你"; + "موريتانيا.xn-----ctdbabcfhu9c2b9l1acccr4c"; IntRange domain = new IntRange(); - boolean actual = hackFindDomainName(source, 0, source.length(), false, domain); - assertEquals(source, true, actual); - assertTrue(source, domain.start == 0 && domain.end == source.length()); + boolean actual = hackFindDomainName(source, 0, source.length(), false, domain); + assertEquals(source, true, actual); + assertTrue(source, domain.start == 0 && domain.end == source.length()); } // Would like to use the following for JUnit, but can't @@ -109,15 +156,13 @@ public void testHackDomainName() { Pattern SCHEME = Pattern.compile("https?://$"); Pattern PORT = Pattern.compile(":\\d+"); - final Set skip = Set.of("See http://-foo.example-.com. on…"); - @Test public void testAgainstLinkDetectionTest() { List failures = new ArrayList<>(); try (BufferedReader idnaTestFile = TestUtil.getUtf8DataReader("unicode/links/LinkDetectionTest.txt"); ) { for (final String line : (Iterable) idnaTestFile.lines()::iterator) { - if (line.startsWith("#") || line.isBlank() || skip.contains(line)) { + if (line.startsWith("#") || line.isBlank() || SKIP_DETECTION_TEST.contains(line)) { continue; } StringBuilder actual = new StringBuilder(); @@ -155,8 +200,8 @@ public void testAgainstLinkDetectionTest() { // If so: // - we scan forwards (after the @) to see if there is a domain name, // AND - // - we scan backwards (before the @) to see if there is local_part, - // AND + // - we scan backwards (before the @) to see if there is local_part, + // AND // - we scan backwards before that to ensure that there is no https?:// // if both are true, we have an email address. // if only the first is true, we skip over the 2nd domain name; @@ -241,10 +286,11 @@ public void testAgainstLinkDetectionTest() { actual.append(rawLine2.substring(start)); assertEquals("", line, actual.toString()); - System.out.println("OK\t" + line); + if (SHOW_LINK_DETECTION) System.out.println("OK\t" + line); } catch (AssertionError e) { failures.add("FAIL " + e.getMessage()); - System.out.println("FAIL\t" + line + "\t " + actual.toString()); + if (SHOW_LINK_DETECTION) + System.out.println("FAIL\t" + line + "\t " + actual.toString()); } } if (!failures.isEmpty()) { @@ -255,7 +301,7 @@ public void testAgainstLinkDetectionTest() { + String.join("\n", failures)); } } catch (IOException e) { - e.printStackTrace(); + throw new UncheckedIOException(e); } } @@ -267,72 +313,150 @@ public void testAgainstLinkFormattingTest() { // https://example.com/%CE%B1%CE%B2%CE%B3/%CE%B4%CE%B5%CE%B6?%CE%B8=%CE%B9%CE%BA%CE%BB&%CE%BC=%CE%B3%CE%BE%CE%BF#%CF%80%CF%81%CF%82 // https://example.com/αβγ/δεζ?θ=ικλ&μ=γξο#πρς - String expectedPrefix = "https://example.com"; // for now, all the same - + List failures = new ArrayList<>(); + // the test case is broken onto 3 lines for ease of reading + // so we assemble each one before testing. + List testCase = new ArrayList<>(); try (BufferedReader idnaTestFile = TestUtil.getUtf8DataReader("unicode/links/LinkFormattingTest.txt"); ) { + String parts = null; String pqf = null; - boolean firstLine = true; + StringBuilder schemeHost = new StringBuilder(); + String minimalPqf = null; + String maximalPqf = null; for (String line : (Iterable) idnaTestFile.lines()::iterator) { - if (line.startsWith("# {")) { - pqf = assemblePqf(line); - firstLine = true; - } else if (line.isBlank() || line.startsWith("#")) { - continue; - } else if (!line.startsWith(expectedPrefix)) { - throw new IllegalArgumentException("Expand the test to cover other S/H"); - } else if (firstLine) { - String actual = - expectedPrefix - + LinkUtilities.escapePathQueryFragment(pqf, Extent.MAXIMAL); - assertEquals("maximum " + pqf, line, actual); - firstLine = false; - } else { - String actual = - expectedPrefix - + LinkUtilities.escapePathQueryFragment(pqf, Extent.MINIMAL); - assertEquals("maximum " + pqf, line, actual); - firstLine = false; + try { + if (line.startsWith("# {")) { + testCase.clear(); + testCase.add(line); + } else if (line.isBlank() || line.startsWith("#")) { + continue; + } else if (testCase.size() == 1) { + testCase.add(line); + } else { // we have assembled the first two lines, so we can process + String partsLine = testCase.get(0); + if (SKIP_FORMAT_TEST.contains(partsLine)) { + continue; + } + parts = + partsLine.substring( + 3, partsLine.length() - 1); // just used in reporting + pqf = assemblePqf(partsLine, schemeHost); + String maximalLine = testCase.get(1); + String minimalLine = line; + System.out.println(partsLine); + minimalPqf = LinkUtilities.escapePathQueryFragment(pqf, Extent.MINIMAL); + assertEquals( + Extent.MINIMAL + " " + parts, minimalLine, schemeHost + minimalPqf); + maximalPqf = LinkUtilities.escapePathQueryFragment(pqf, Extent.MAXIMAL); + assertEquals( + Extent.MAXIMAL + " " + parts, maximalLine, schemeHost + maximalPqf); + } + System.out.println("OK " + line); + } catch (AssertionError e) { + failures.add("FAIL " + e.getMessage()); + System.out.println("FAIL\t" + line); } } } catch (IOException e) { - e.printStackTrace(); + throw new UncheckedIOException(e); + } + + if (!failures.isEmpty()) { + fail( + String.join( + " ", + "Test finished with", + failures.size() + "", + "failures:\n" + String.join("\n", failures))); } } - private String assemblePqf(String line) { + @Test + public void testParts() { + String source = + "# {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δεζ 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γξο 𝑭=πρς}"; + List> expected = new ArrayList<>(); + expected.add(List.of("𝑺", "https://")); + expected.add(List.of("𝑯", "example.com")); + expected.add(List.of("𝑷", "αβγ")); + expected.add(List.of("𝑷", "δεζ")); + expected.add(List.of("𝑸", "θ")); + expected.add(List.of("𝑽", "ικλ")); + expected.add(List.of("𝑸", "μ")); + expected.add(List.of("𝑽", "γξο")); + expected.add(List.of("𝑭", "πρς")); + List> actual = getParts(source); + assertEquals(source, expected, actual); + } + + Pattern UrlTypes = Pattern.compile("([𝑺𝑯𝑷𝑸𝑽𝑭])=([^𝑺𝑯𝑷𝑸𝑽𝑭]+)"); + + private List> getParts(String line) { + // # {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δεζ 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γξο 𝑭=πρς} line = line.substring(3, line.length() - 1); // subtract the # { and the } + Matcher urlType = UrlTypes.matcher(line); + List> result = new ArrayList<>(); + while (urlType.find()) { + String key = urlType.group(1); + String value = urlType.group(2); + if (value.endsWith(" ")) { + value = value.substring(0, value.length() - 1); + } + result.add(List.of(key, value)); + } + return List.copyOf(result); + } + + /** + * When composing a URL, the following have to be escaped within each part. They are the + * characters that could initiate a different component + */ + final UnicodeSet PATH_ESCAPE = new UnicodeSet("[/?#]").freeze(); + + final UnicodeSet QUERY_ESCAPE = new UnicodeSet("[\\&=#]").freeze(); + final UnicodeSet VALUE_ESCAPE = new UnicodeSet("[\\&#]").freeze(); + + private String assemblePqf(String line, StringBuilder schemeHost) { StringBuilder sb = new StringBuilder(); + schemeHost.setLength(0); + List> parts = getParts(line); char query = '?'; - for (String item : Splitter.on(' ').splitToList(line)) { - int eq = item.indexOf('='); - String type = item.substring(0, eq); - String rest = item.substring(eq + 1); - // # {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δεζ 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γξο 𝑭=πρς} + for (List entry : parts) { + String type = entry.get(0); + String value = entry.get(1); + // # {𝑺=https:// 𝑯=example.com 𝑷=αβγ 𝑷=δεζ 𝑸=θ 𝑽=ικλ 𝑸=μ 𝑽=γξο 𝑭=πρς} switch (type) { case "𝑺": case "𝑯": + if (value != null) { + schemeHost.append(value); + } break; case "𝑷": - sb.append('/').append(rest); + sb.append('/').append(escape(value, PATH_ESCAPE)); break; case "𝑸": - sb.append(query).append(rest); + sb.append(query).append(escape(value, QUERY_ESCAPE)); query = '&'; break; case "𝑽": - sb.append('=').append(rest); + sb.append('=').append(escape(value, VALUE_ESCAPE)); break; case "𝑭": - sb.append('#').append(rest); + sb.append('#').append(value); break; default: - throw new IllegalArgumentException("Unexpected line: " + item); + throw new IllegalArgumentException("Unexpected line: " + line); } } return sb.toString(); } + private Object escape(String value, UnicodeSet unicodeSet) { + return LinkHandlingUtilities.escape(value, unicodeSet); + } + class IntRange { int start; int end; @@ -345,13 +469,13 @@ public String toString() { Pattern DOT = Pattern.compile("[.。]"); Pattern DOT2 = Pattern.compile("[.。][.。]"); + /** * The UTS58 tests take whole domain names, because that is the easiest for clients to test. We * have to hack it because ICU IDN support doesn't provide enough API. Returns true if found, * with the range in result. * - * @param mustBeAtStart - * TODO check with isValidDomain + * @param mustBeAtStart TODO check with isValidDomain */ boolean hackFindDomainName( CharSequence source, int start, int limit, boolean mustBeAtStart, IntRange result) { @@ -373,7 +497,7 @@ boolean hackFindDomainName( end = limit; } if (end == endNot) { - return false; + return false; } // Check that the possible domain name (endNot..end) is of the right form // otherwise continue after end. @@ -382,7 +506,7 @@ boolean hackFindDomainName( boolean found = dot.find(); if (!found || dot.start() == endNot || dot.end() == end) { start = end; - continue; + continue; } // Must not find [.。][.。] dot2.region(endNot, end); @@ -390,6 +514,11 @@ boolean hackFindDomainName( start = end; continue; } + // and a final test for valid + if (!isValidDomain(source.subSequence(endNot, end).toString())) { + start = end; + continue; + } result.start = endNot; result.end = end; return true; @@ -442,14 +571,14 @@ static boolean isValidDomain(String domainName) { } // Initialize the UTS#46 standard IDNA instance - IDNA idna = IDNA.getUTS46Instance( - IDNA.NONTRANSITIONAL_TO_ASCII | - IDNA.NONTRANSITIONAL_TO_UNICODE | - IDNA.CHECK_BIDI | - IDNA.CHECK_CONTEXTJ | - IDNA.CHECK_CONTEXTO | - IDNA.USE_STD3_RULES - ); + IDNA idna = + IDNA.getUTS46Instance( + IDNA.NONTRANSITIONAL_TO_ASCII + | IDNA.NONTRANSITIONAL_TO_UNICODE + | IDNA.CHECK_BIDI + | IDNA.CHECK_CONTEXTJ + | IDNA.CHECK_CONTEXTO + | IDNA.USE_STD3_RULES); StringBuilder dest = new StringBuilder(); IDNA.Info info = new IDNA.Info(); From 6bd77efa3d84052d26a8d6e5ad475a74abfd7553 Mon Sep 17 00:00:00 2001 From: Mark Davis Date: Fri, 3 Jul 2026 13:07:52 +0300 Subject: [PATCH 5/8] ICU-23431 Copyright --- .../java/com/ibm/icu/impl/links/LinkHandlingUtilities.java | 3 ++- .../core/src/main/java/com/ibm/icu/util/LinkUtilities.java | 2 ++ .../src/test/java/com/ibm/icu/dev/test/links/LinkTest.java | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java index 7e1188ecb616..a7fe0fd4378a 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java @@ -1,9 +1,10 @@ +// © 2026 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html package com.ibm.icu.impl.links; import com.ibm.icu.impl.IDNA2003; import com.ibm.icu.impl.UnicodeMap; import com.ibm.icu.impl.Utility; -import com.ibm.icu.impl.links.LinkHandlingUtilities.UrlInternals; import com.ibm.icu.impl.locale.XCldrStub; import com.ibm.icu.impl.locale.XCldrStub.Splitter; import com.ibm.icu.lang.UCharacter; diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java index dd51149a8f01..af98b4c2fca5 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/util/LinkUtilities.java @@ -1,3 +1,5 @@ +// © 2026 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html package com.ibm.icu.util; import com.ibm.icu.impl.links.LinkHandlingUtilities; diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java index 881d7981510e..7a22b6e1612e 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java @@ -1,3 +1,5 @@ +// © 2026 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html package com.ibm.icu.dev.test.links; import com.ibm.icu.dev.test.CoreTestFmwk; From 7e7032b60e7c85cf49a6c8272d174bc5275c373d Mon Sep 17 00:00:00 2001 From: Mark Davis Date: Sat, 4 Jul 2026 16:30:02 +0300 Subject: [PATCH 6/8] ICU-23431 Cleanp unused code from port --- .../icu/impl/links/LinkHandlingUtilities.java | 306 ------------------ .../com/ibm/icu/dev/test/links/LinkTest.java | 30 +- 2 files changed, 21 insertions(+), 315 deletions(-) diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java index a7fe0fd4378a..72ceec174386 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java @@ -2,23 +2,16 @@ // License & terms of use: http://www.unicode.org/copyright.html package com.ibm.icu.impl.links; -import com.ibm.icu.impl.IDNA2003; import com.ibm.icu.impl.UnicodeMap; import com.ibm.icu.impl.Utility; import com.ibm.icu.impl.locale.XCldrStub; import com.ibm.icu.impl.locale.XCldrStub.Splitter; import com.ibm.icu.lang.UCharacter; -import com.ibm.icu.lang.UProperty; -import com.ibm.icu.lang.UProperty.NameChoice; -import com.ibm.icu.text.StringPrepParseException; import com.ibm.icu.text.UnicodeSet; -import com.ibm.icu.text.UnicodeSet.EntryRange; import com.ibm.icu.text.UnicodeSet.SpanCondition; -import com.ibm.icu.util.ICUException; import com.ibm.icu.util.Output; import java.nio.charset.StandardCharsets; import java.util.EnumMap; -import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -100,15 +93,10 @@ private enum LinkTermination { public final UnicodeSet base; - public UnicodeSet getBase() { - return base; - } - private LinkTermination(String uset) { if (uset == null) { // only called with Include, the "none of the above" option this.base = SOFAR.complement().freeze(); } else { - java.text.ParsePosition parsePosition = new java.text.ParsePosition(0); this.base = new UnicodeSet(uset).freeze(); SOFAR.addAll(this.base); } @@ -139,9 +127,6 @@ private LinkTermination(String uset) { } PROPERTY_MAP.freeze(); } - - private static final Set NON_MISSING = - XCldrStub.setsDifference(EnumSet.allOf(LinkTermination.class), Set.of(Hard)); } // https://www.rfc-editor.org/rfc/rfc5322.html#section-3.2.3 has the full list for ASCII part @@ -265,18 +250,6 @@ static Part fromInitiator(int cp) { return null; } - /** - * Unescape a part. But don't unescape interior characters or terminators because they are - * content! For example "a/b%2Fc" as a path should not be turned into a/b/c, because that - * b/c is a path-part. - * - * @param substring - * @return - */ - private String unescape(String substring) { - return LinkHandlingUtilities.unescape(substring, extraQuoted); - } - private String fullEscape(List> source) { if (source.isEmpty()) { return ""; @@ -310,16 +283,6 @@ private String fullEscape(List> source) { } return result.toString(); } - - /** Append an unescaped part, escaping as necessary */ - private void appendPart(StringBuilder toAppendTo, String partValue) { - if (!partValue.isBlank()) { - if (initiator != 0) { - toAppendTo.appendCodePoint(initiator); - } - toAppendTo.append(partValue); - } - } } public static class UrlInternals { @@ -662,116 +625,6 @@ private static List splitList(Part part, String separator, String partSt .collect(Collectors.toList()); } - // private static final UnicodeSet idnMapped = - // IUP.getSet("Idn_Status=" + Idn_Status_Values.mapped); - // - // private static final UnicodeSet idnValid = IUP.getSet("Idn_Status=" + - // Idn_Status_Values.valid); - // static final Pattern protocol = Pattern.compile("(https?://|mailto:)"); - // public static final UnicodeSet validHost = - // new UnicodeSet(idnValid) - // .addAll(idnMapped) - // .removeAll(new UnicodeSet("[:Block=Basic_Latin:]")) - // .addAll(new UnicodeSet("[-a-zA-Z0-9..。]")) // the three dots are - // different: - // .freeze(); - // public static final UnicodeSet validHostNoDot = - // new UnicodeSet(validHost).removeAll("..。").freeze(); - - /** Status of link found. Note that if the link is imputed, https or mailto will be returned. */ - enum LinkStatus { - bogus, - http, - https, - mailto, - tld - } - - /** Immutable class for returning the start/end of link that is found, plus some information */ - private static final class LinkFound { - public final int start; - public final int limit; - public final LinkStatus linkStatus; - - public LinkFound(int start, int limit, LinkStatus linkStatus) { - this.start = start; - this.limit = limit; - this.linkStatus = linkStatus; - } - - public String substring(String source) { - return source.substring(start, limit); - } - } - - // OLDER Code, leaving here for now, for comparison - // /** - // * Parses a restricted set of URLs, for testing the PathQueryFragment portion. That is, it - // is of - // * the form <host><domain_name>?
- // * The host is currently just http, https, and mailto.
- // * The domain_name is just approximated for testing. - // * - // * @return null if we run out of string, otherwise a LinkFound. If what is found is - // malformed in - // * some way, indicate with LinkFound.bogus. - // */ - // private static LinkFound parseLink(String source, int startCodePointOffset) { - // Matcher findStartMatcher = protocol.matcher(source); - // - // findStartMatcher.region(startCodePointOffset, source.length()); - // if (!findStartMatcher.find()) { - // return null; // we are at the end - // } - // // if we found something, and there was a character before it, - // // and that character was not soft, then exit - // int start = findStartMatcher.start(); - // int protocolEnd = findStartMatcher.end(); - // if (start != 0) { - // LinkTermination lt = - // LinkTermination.PROPERTY_MAP.get(UCharacter.codePointBefore(source, - // start)); - // if (lt == LinkTermination.Include) { - // return new LinkFound(start, protocolEnd, LinkStatus.bogus); - // } - // } - // - // String protocolValue = findStartMatcher.group(1); - // if (protocolValue.equals("mailto")) { - // return parseRestOfMailto(source, start, protocolEnd); - // } - // - // // dumb search for end of host, doesn't handle .. or edge cases, but this does not - // have to - // // be production-quality - // int hostLimit = findEndOfDomain(source, protocolEnd); - // if (protocolEnd == hostLimit) { - // return new LinkFound(start, protocolEnd, LinkStatus.bogus); - // } - // int limit = parsePathQueryFragment(source, hostLimit); - // return new LinkFound( - // start, limit, LinkStatus.valueOf(source.substring(start, protocolEnd))); - // } - // - // private static LinkFound parseRestOfMailto(String source, int start, int hostStart) { - // // basic implementation at first - // int atPosition = source.indexOf('@', hostStart); - // if (atPosition == -1) { - // return new LinkFound(start, hostStart, LinkStatus.bogus); - // } - // // TBD we could be in the middle of a quoted string, check for that later - // - // // see if what is in front of the @ looks ok - // - // int limit = parsePathQueryFragment(source, atPosition + 1); - // return new LinkFound(start, limit, LinkStatus.mailto); - // } - // - // // Simple implementation for testing, since the spec doesn't define the content - // private static int findEndOfDomain(String source, int protocolLimit) { - // return validHost.span(source, protocolLimit, SpanCondition.CONTAINED); - // } - /** * Set lastSafe to 0 — this marks the last code point that is definitely included in the * linkification.
@@ -931,11 +784,6 @@ private static void appendPercentEscaped(StringBuilder output, String source) { } } - /** percent-escape single char or string. */ - private static String escape(String source, String cpToEscape) { - return escape(source, new UnicodeSet().add(cpToEscape)); - } - /** percent-escape all the toEscape characters. */ public static String escape(String source, UnicodeSet toEscape) { StringBuilder result = new StringBuilder(); @@ -989,160 +837,6 @@ private static String percentUnescape(String escapedSource) { return new String(temp, StandardCharsets.UTF_8); } - /** - * The wikipedia languages are not all BCP47. Convert the ones that are not. See:
- * https://meta.wikimedia.org/wiki/Special_language_codes
- * https://meta.wikimedia.org/wiki/List_of_Wikipedias#Nonstandard_language_codes - * - * @param languageCode - * @return - */ - private static String fixWiki(String languageCode) { - switch (languageCode) { - case "als": - return "gsw"; - case "roa-rup": - return "rup"; - case "bat-smg": - return "sgs"; - case "simple": - return "en"; - case "fiu-vro": - return "vro"; - case "zh-classical": - return "lzh"; - case "zh-min-nan": - return "nan"; - case "zh-yue": - return "yue"; - case "cbk-zam": - return "cbk"; - case "map-bms": - return "map"; - case "nrm": - return "nrf"; - case "roa-tara": - return "nap"; - default: - return languageCode; - } - } - - /** Hard-coded list of wikilanguages, for testing */ - // private static Set WIKI_LANGUAGES = - // of( - // SPLIT_COMMA.splitToList( - // - // "en,ceb,de,fr,sv,nl,ru,es,it,pl,arz,zh,ja,uk,vi,war,ar,pt,fa,ca,id,sr,ko,no,tr,ce,fi,cs,hu,tt,ro,sh,eu,zh-min-nan,ms,he,eo,hy,da,bg,uz,cy,simple,sk,et,be,azb,el,kk,min,hr,lt,gl,ur,az,sl,lld,ka,nn,ta,th,hi,bn,mk,zh-yue,la,ast,lv,af,tg,my,te,sq,mr,mg,bs,oc,be-tarask,ku,br,sw,ml,nds,ky,lmo,jv,pnb,ckb,new,ht,vec,pms,lb,ba,su,ga,is,szl,cv,pa,fy,io,ha,tl,an,mzn,wuu,diq,vo,ig,yo,sco,kn,ne,als,gu,ia,avk,crh,bar,ban,scn,bpy,mn,qu,nv,si,xmf,frr,ps,os,or,tum,sd,bcl,bat-smg,sah,cdo,gd,bug,glk,yi,ilo,am,li,nap,gor,as,fo,mai,hsb,map-bms,shn,zh-classical,eml,ace,ie,wa,sa,hyw,sat,zu,sn,mhr,lij,hif,km,bjn,mrj,mni,dag,ary,hak,pam,rue,roa-tara,ug,zgh,bh,nso,co,tly,so,vls,nds-nl,mi,se,myv,rw,kaa,sc,bo,kw,vep,mt,tk,mdf,kab,gv,gan,fiu-vro,ff,zea,ab,skr,smn,ks,gn,frp,pcd,udm,kv,csb,ay,nrm,lo,ang,fur,olo,lfn,lez,ln,pap,nah,mwl,tw,stq,rm,ext,lad,gom,dty,av,tyv,koi,dsb,lg,cbk-zam,dv,ksh,za,bxr,blk,gag,pfl,bew,szy,haw,tay,pag,pi,awa,tcy,krc,inh,gpe,xh,kge,fon,atj,to,pdc,mnw,arc,shi,om,tn,dga,ki,nia,jam,kbp,wo,xal,nov,kbd,anp,nqo,bi,kg,roa-rup,tpi,tet,guw,jbo,mad,fj,lbe,kcg,pcm,cu,ty,trv,dtp,sm,ami,st,iba,srn,btm,alt,ltg,gcr,ny,kus,mos,ss,chr,ee,ts,got,bbc,gur,bm,pih,ve,rmy,fat,chy,rn,igl,ik,guc,ch,ady,pnt,iu,ann,rsk,pwn,dz,ti,sg,din,tdd,kl,bdr,nr,cr")); - - /** Show termination values, for generating property files */ - private void showLinkTermination() { - for (LinkTermination lt : LinkTermination.values()) { - UnicodeSet value = LinkTermination.PROPERTY_MAP.getSet(lt); - String name = lt.toString(); - System.out.println("\n#\tLink_Termination=" + name); - if (lt == LinkTermination.Include) { - System.out.println("# " + "(All code points without other values)"); - continue; - } else { - System.out.println("# draft = " + lt.base); - } - if (lt == LinkTermination.Hard) { - value.removeAll(new UnicodeSet("[\\p{Cn}\\p{Cs}]")); - System.out.println("# (not listing Unassigned or Surrogates)"); - } - System.out.println(); - for (EntryRange range : value.ranges()) { - final String rangeString = - Utility.hex(range.codepoint) - + (range.codepoint == range.codepointEnd - ? "" - : ".." + Utility.hex(range.codepointEnd)); - System.out.println( - rangeString - + ";" - + " ".repeat(15 - rangeString.length()) - + lt - + "\t# " - + "(" - + getGeneralCategory( - UProperty.GENERAL_CATEGORY, - range.codepoint, - NameChoice.SHORT) - + ") " - + quote(UCharacter.getExtendedName(range.codepoint)) - + (range.codepoint == range.codepointEnd - ? "" - : ".." - + "(" - + getGeneralCategory( - UProperty.GENERAL_CATEGORY, - range.codepointEnd, - NameChoice.SHORT) - + ") " - + quote( - UCharacter.getExtendedName( - range.codepointEnd)))); - } - System.out.println(); - } - } - - /** Show paired openers, for generating property files */ - private void showLinkPairedOpeners() { - UnicodeSet value = LinkTermination.PROPERTY_MAP.getSet(LinkTermination.Close); - - System.out.println("\n#\tLink_Paired_Opener"); - System.out.println( - "# draft = BidiPairedBracket + (“>” GREATER-THAN SIGN 🡆 “<” LESS-THAN SIGN)"); - System.out.println(); - - for (String cpString : value) { - int cp = cpString.codePointAt(0); - String hex = Utility.hex(cp); - final int value2 = getOpening(cp); - System.out.println( - hex - + ";" - + " ".repeat(7 - hex.length()) - + Utility.hex(value2) - + "\t#" - + " “" - + quote(Character.toString(cp)) - + "” " - + UCharacter.getExtendedName(cp) - + " 🡆 " - + " “" - + quote(Character.toString(value2)) - + "” " - + UCharacter.getExtendedName(value2)); - } - } - - /** - * Regex to scan for possible TLDs. The result needs to be checked that there is no - * validHostNoDot before and after - */ - // private static final Pattern TLD_SCANNER; - // - // private static final SortedSet TLDS; - // - private static final String DOTSET_STRING = "[.。]"; - - private static final UnicodeSet DOTSET = new UnicodeSet("[.。]").freeze(); - private static final Splitter SPLIT_LABELS = Splitter.on(Pattern.compile("[.。]")); - - private static String toUnicode2(String x) { - try { - if (x.startsWith("XN--")) { - x = IDNA2003.convertIDNToUnicode(x, 0).toString(); - } - return x; - } catch (StringPrepParseException e) { - throw new ICUException(e); - } - } - public static Map>> getInternals(String source) { UrlInternals ui = UrlInternals.from(source.toString()); Map>> result = new LinkedHashMap<>(); diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java index 7a22b6e1612e..2244e0b11cef 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java @@ -49,7 +49,7 @@ @RunWith(JUnit4.class) public class LinkTest extends CoreTestFmwk { - private static final boolean SHOW_LINK_DETECTION = false; + private static final boolean VERBOSE = false; /** * There are 2 format test cases that need to be fixed in UTS58 In each of these, the = in a 𝑽 @@ -288,11 +288,14 @@ public void testAgainstLinkDetectionTest() { actual.append(rawLine2.substring(start)); assertEquals("", line, actual.toString()); - if (SHOW_LINK_DETECTION) System.out.println("OK\t" + line); + if (isVerbose()) { + System.out.println("OK\t" + line); + } } catch (AssertionError e) { failures.add("FAIL " + e.getMessage()); - if (SHOW_LINK_DETECTION) + if (isVerbose()) { System.out.println("FAIL\t" + line + "\t " + actual.toString()); + } } } if (!failures.isEmpty()) { @@ -346,18 +349,27 @@ public void testAgainstLinkFormattingTest() { pqf = assemblePqf(partsLine, schemeHost); String maximalLine = testCase.get(1); String minimalLine = line; - System.out.println(partsLine); - minimalPqf = LinkUtilities.escapePathQueryFragment(pqf, Extent.MINIMAL); - assertEquals( - Extent.MINIMAL + " " + parts, minimalLine, schemeHost + minimalPqf); + + // Check explicit test cases + maximalPqf = LinkUtilities.escapePathQueryFragment(pqf, Extent.MAXIMAL); assertEquals( Extent.MAXIMAL + " " + parts, maximalLine, schemeHost + maximalPqf); + + minimalPqf = LinkUtilities.escapePathQueryFragment(pqf, Extent.MINIMAL); + assertEquals( + Extent.MINIMAL + " " + parts, minimalLine, schemeHost + minimalPqf); + + // TODO Look at roundtripping both of these. + } + if (isVerbose()) { + System.out.println("OK " + line); } - System.out.println("OK " + line); } catch (AssertionError e) { failures.add("FAIL " + e.getMessage()); - System.out.println("FAIL\t" + line); + if (isVerbose()) { + System.out.println("FAIL\t" + line); + } } } } catch (IOException e) { From 67c6da768d09ccd0a03bc5ba1fadc3059617c741 Mon Sep 17 00:00:00 2001 From: Mark Davis Date: Sat, 4 Jul 2026 17:23:56 +0300 Subject: [PATCH 7/8] ICU-23431 Cleanup --- .../icu/impl/links/LinkHandlingUtilities.java | 16 +--------------- .../com/ibm/icu/dev/test/links/LinkTest.java | 6 ------ 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java index 72ceec174386..d67a9bb9be4b 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/impl/links/LinkHandlingUtilities.java @@ -73,10 +73,6 @@ private WHATWG_PERCENT_ENCODED(WHATWG_PERCENT_ENCODED base, String uset) { } } - private static String quote(String s) { - return s; // later, escape for HTML - } - /** TODO use real property */ private static int getOpening(int cp) { return cp == '>' ? '<' : UCharacter.getBidiPairedBracket(cp); @@ -162,11 +158,6 @@ public static int scanEmailBackwards(CharSequence source, int hardStart, int dom return result; } - private static String getGeneralCategory(int property, int codePoint, int nameChoice) { - return UCharacter.getPropertyValueName( - property, UCharacter.getIntPropertyValue(codePoint, property), nameChoice); - } - private enum Structure { single, listP("/", "𝑷"), @@ -219,7 +210,6 @@ private enum Part { final int initiator; final UnicodeSet terminators; final UnicodeSet clearStack; - final UnicodeSet extraQuoted; final UnicodeSet whatwg_quoted; private Part( @@ -233,11 +223,7 @@ private Part( this.initiator = initiator; this.terminators = new UnicodeSet(terminators).freeze(); this.clearStack = new UnicodeSet(clearStack).freeze(); - this.extraQuoted = - new UnicodeSet(extraQuoted) - .addAll(this.clearStack) - .addAll(this.terminators) - .freeze(); + new UnicodeSet(extraQuoted).addAll(this.clearStack).addAll(this.terminators).freeze(); this.whatwg_quoted = whatwg_quoted == null ? UnicodeSet.EMPTY : whatwg_quoted.set; } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java index 2244e0b11cef..3f96cff2ee7f 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java @@ -48,9 +48,6 @@ */ @RunWith(JUnit4.class) public class LinkTest extends CoreTestFmwk { - - private static final boolean VERBOSE = false; - /** * There are 2 format test cases that need to be fixed in UTS58 In each of these, the = in a 𝑽 * doesn't need to be quoted, because it doesn't initiate a different component. @@ -276,9 +273,6 @@ public void testAgainstLinkDetectionTest() { } domain.end = pqfEnd; } - if (start > domain.start) { - int debug = 0; - } actual.append(rawLine2.subSequence(start, domain.start)) .append("⸠") .append(rawLine2.subSequence(domain.start, domain.end)) From 24790ac83c9e96ece257245c1a5b7aededf09c25 Mon Sep 17 00:00:00 2001 From: Mark Davis Date: Sat, 4 Jul 2026 19:51:46 +0300 Subject: [PATCH 8/8] ICU-23431 Cleanup --- .../core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java index 3f96cff2ee7f..32599bfe3222 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/links/LinkTest.java @@ -250,8 +250,6 @@ public void testAgainstLinkDetectionTest() { schemeMatcher.region(start, domain.start); if (schemeMatcher.find()) { domain.start = schemeMatcher.start(); - } else { - domain.start = domain.start; } Matcher portMatcher = PORT.matcher(rawLine2);