Fix sorting self import alias and importing unicode (#17)

Author: mtfishman

Project.toml

@@ -1,6 +1,6 @@
 name = "ITensorFormatter"
 uuid = "b6bf39f1-c9d3-4bad-aad8-593d802f65fd"
-version = "0.2.10"
+version = "0.2.11"
 authors = ["ITensor developers <support@itensor.org> and contributors"]
 
 [workspace]

src/ITensorFormatter.jl

@@ -47,9 +47,9 @@ const JULIAFORMATTER_OPTIONS = (
     trailing_zero = true,
 )
 
-is_using_or_import(x) = kind(x) === K"using" || kind(x) === K"import"
+is_using_or_import(x::SyntaxNode) = kind(x) === K"using" || kind(x) === K"import"
 
-function find_using_or_import(x)
+function find_using_or_import(x::SyntaxNode)
     if is_using_or_import(x)
         return x.parent
     elseif iszero(length(children(x)))
@@ -63,46 +63,62 @@ function find_using_or_import(x)
     end
 end
 
-char_range(x) = x.position:(x.position + span(x) - 1)
+# JuliaSyntax nodes report (position, span) in *bytes*.
+# Julia strings must be sliced using *valid string indices* (start bytes of UTF-8 chars).
+#
+# Convert a node's byte range into a safe `UnitRange` of valid string indices.
+# (O(1): fix endpoints with `thisind`.)
+function node_char_range(n::SyntaxNode, src::AbstractString)
+    startb = n.position
+    nspan = span(n)
+    nspan <= 0 && return startb:(startb - 1) # empty range
+    stopb = min(startb + nspan - 1, ncodeunits(src))
+    return thisind(src, startb):thisind(src, stopb)
+end
 
-function organize_import_blocks_string(s)
-    jst = parseall(SyntaxNode, s)
+function organize_import_blocks_string(s::AbstractString)
+    jst = parseall(SyntaxNode, String(s))
     return organize_import_blocks(jst)
 end
-organize_import_blocks_file(f) = organize_import_blocks_string(read(f, String))
 
-# Sort symbols, but keep the module self-reference first if present
-function sort_with_self_first(syms, self)
-    self′ = pop!(syms, self, nothing)
-    sorted = sort!(collect(syms))
-    if self′ !== nothing
-        pushfirst!(sorted, self)
-    end
-    return sorted
+function organize_import_blocks_file(f::AbstractString)
+    return organize_import_blocks_string(read(f, String))
+end
+
+# Sort symbols, but keep the module self-reference (bare and all aliases) first if present
+function sort_with_self_first(syms::Vector{String}, self::String)
+    syms = unique(syms)
+    selfs = self in syms ? [self] : String[]
+    self_aliases = sort!(filter(s -> startswith(s, self * " as "), syms))
+    rest = sort!(setdiff(syms, [selfs; self_aliases]))
+    return [selfs; self_aliases; rest]
 end
 
 # Organize a single block of adjacent import/using statements
-function organize_import_block(siblings, node_text)
+function organize_import_block(siblings::AbstractVector{<:SyntaxNode}, node_text)
     using_mods = Set{String}()
-    using_syms = Dict{String, Set{String}}()
+    using_syms = Dict{String, Vector{String}}()
     import_mods = Set{String}()
-    import_syms = Dict{String, Set{String}}()
+    import_syms = Dict{String, Vector{String}}()
 
     for s in siblings
         isusing = kind(s) === K"using"
         for a in children(s)
             if kind(a) === K":"
                 a_args = children(a)
-                mod = node_text(a_args[1])
-                set = get!(Set, isusing ? using_syms : import_syms, mod)
+                mod = String(node_text(a_args[1]))
+                set = get!(() -> String[], isusing ? using_syms : import_syms, mod)
                 for i in 2:length(a_args)
                     push!(set, String(node_text(a_args[i])))
                 end
             elseif kind(a) === K"." || kind(a) === K"importpath"
                 push!(isusing ? using_mods : import_mods, String(node_text(a)))
             elseif !isusing && kind(a) === K"as"
                 a_args = children(a)
-                push!(import_mods, node_text(a_args[1]) * " as " * node_text(a_args[end]))
+                push!(
+                    import_mods,
+                    String(node_text(a_args[1])) * " as " * String(node_text(a_args[end]))
+                )
             else
                 error("Unexpected syntax in using/import statement.")
             end
@@ -124,21 +140,26 @@ function organize_import_block(siblings, node_text)
         push!(using_lines, "using " * m * ": " * join(sort_with_self_first(s, m), ", "))
     end
     io = IOBuffer()
-    join(io, sort!(import_lines), "\n")
-    length(import_lines) > 0 && length(using_lines) > 0 && print(io, "\n")
-    join(io, sort!(using_lines), "\n")
+    if !isempty(import_lines)
+        print(io, join(sort!(import_lines), "\n"))
+    end
+    if !isempty(import_lines) && !isempty(using_lines)
+        print(io, "\n")
+    end
+    if !isempty(using_lines)
+        print(io, join(sort!(using_lines), "\n"))
+    end
     return String(take!(io))
 end
 
-function organize_import_blocks(input)
-    src = JuliaSyntax.sourcetext(input)
+function organize_import_blocks(input::SyntaxNode)
+    # Keep a stable copy for slicing: node positions/spans refer to this text.
+    src0 = JuliaSyntax.sourcetext(input)
     x = find_using_or_import(input)
-    isnothing(x) && return src
-
+    isnothing(x) && return src0
     child_nodes = children(x)
-
     # Find all groups of adjacent import/using statements
-    groups = Vector{Any}[]
+    groups = Vector{SyntaxNode}[]
     i = 1
     while i <= length(child_nodes)
         if is_using_or_import(child_nodes[i])
@@ -151,18 +172,25 @@ function organize_import_blocks(input)
             i += 1
         end
     end
-
-    # Extract the source text of a node, trimming whitespace
-    node_text(n) = strip(src[char_range(n)])
-
+    # Extract the source text of a node, trimming whitespace (Unicode-safe).
+    # Always slice from src0 (stable offsets), not the rewritten `src`.
+    node_text(n::SyntaxNode) = strip(src0[node_char_range(n, src0)])
+    # Rewritten output source.
+    src = src0
     # Process each group from right to left to preserve positions
     for siblings in reverse(groups)
         formatted = organize_import_block(siblings, node_text)
-        first_pos = first(char_range(siblings[1]))
-        last_pos = last(char_range(siblings[end]))
-        src = src[1:(first_pos - 1)] * chomp(formatted) * src[(last_pos + 1):end]
+        # Compute splice bounds using src0 (node offsets), then splice into src.
+        first_pos = first(node_char_range(first(siblings), src0))
+        last_pos = last(node_char_range(last(siblings), src0))
+        # Unicode-safe splice boundaries (never do ±1 on raw integer indices)
+        before =
+            first_pos == firstindex(src) ? "" :
+            src[firstindex(src):prevind(src, first_pos)]
+        after_start = nextind(src, last_pos)
+        after = after_start > lastindex(src) ? "" : src[after_start:end]
+        src = before * chomp(formatted) * after
     end
-
     return src
 end
 

test/test_basics.jl

@@ -128,6 +128,50 @@ organize(s) = ITensorFormatter.organize_import_blocks_string(s)
         @test result ==
             "using Baz: baz\nusing Foo: foo\nx = 1\nusing Alpha: a\nusing Zebra: a, z\n"
     end
+
+    @testset "multiple self-imports and aliases" begin
+        # Bare self and aliases, plus others
+        result = organize("using Foo: bar, Foo as Q, Foo, Foo as P, baz, A")
+        @test result == "using Foo: Foo, Foo as P, Foo as Q, A, bar, baz"
+
+        # Only aliases, no bare self
+        result = organize("using Foo: Foo as Q, Foo as P, baz, A")
+        @test result == "using Foo: Foo as P, Foo as Q, A, baz"
+
+        # Only bare self
+        result = organize("using Foo: Foo, baz, A")
+        @test result == "using Foo: Foo, A, baz"
+
+        # Duplicates should be removed
+        result = organize("using Foo: Foo, Foo as P, Foo as P, Foo, A, A")
+        @test result == "using Foo: Foo, Foo as P, A"
+    end
+
+    @testset "unicode and non-ASCII symbols" begin
+        # Single unicode symbol
+        result = organize("using Foo: ⊗")
+        @test result == "using Foo: ⊗"
+
+        # Multiple unicode and ASCII symbols
+        result = organize("using Foo: ⊗, x, y, X")
+        @test result == "using Foo: X, x, y, ⊗"
+
+        # Multiple unicode symbols, ASCII symbols, and self imports
+        result = organize("using Foo: ⊗, Foo as B, x, ×, Foo, y, Foo as A, X")
+        @test result == "using Foo: Foo, Foo as A, Foo as B, X, x, y, ×, ⊗"
+
+        # Unicode self-reference
+        result = organize("using ⊗: ⊗, x, y")
+        @test result == "using ⊗: ⊗, x, y"
+
+        # Unicode self-reference with alias
+        result = organize("using ⊗: ⊗ as T, ⊗, x, y")
+        @test result == "using ⊗: ⊗, ⊗ as T, x, y"
+
+        # Unicode and ASCII, with alias
+        result = organize("using Foo: ⊗ as T, x, ⊗, y")
+        @test result == "using Foo: x, y, ⊗, ⊗ as T"
+    end
 end
 
 @testset "main" begin