[codex] Stabilize Harbor-backed test runs (#235)

quinnj · web-flow · commit 4261c3d4fbbd · 2026-04-23T13:59:33.000-06:00
* fix(mysql): stabilize Harbor-backed test runs Switch the test suite to a Harbor-managed MySQL container so local and CI testing only assumes Docker availability. Also restore a stable Connector/C SSL default, fix option reads, close load-time prepared statements, and advance multi-result cleanup via nextresult() so the suite stays green on current MariaDB Connector/C releases. Fixes #234 * chore(julia): raise minimum version to 1.10 Julia 1.6 CI cannot resolve Harbor, and Julia 1.10 is the current LTS line as of April 23, 2026. Bump the package compat floor and replace the 1.6 CI lane with 1.10 so the matrix matches the versions we intend to support with the Harbor-backed test harness. * fix(results): restore guarded multi-result advance * fix(results): check more results before free * test(results): use direct connection for multistmt * fix(api): read string option values correctly * test(harbor): require released log wait fix Require Harbor 1.0.3 for tests and switch the MySQL container readiness check back to the intended log-pattern wait now that the stderr capture fix is released.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - "1.6"
+          - "1.10"
           - 1 # automatically expands to the latest stable 1.x release of Julia
           - nightly
         os:
@@ -21,7 +21,7 @@ jobs:
           - x64
     steps:
       - uses: actions/checkout@v5
-      - run: docker compose up -d
+      - run: docker info
       - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
@@ -50,4 +50,4 @@ jobs:
       - run: julia --project=docs docs/make.jl
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
+          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
diff --git a/Project.toml b/Project.toml
@@ -17,14 +17,17 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 [compat]
 DBInterface = "2.5"
 DecFP = "0.4.9, 0.4.10, 1"
+Harbor = "1.0.3"
 MariaDB_Connector_C_jll = "3.1.12"
 OpenSSL_jll = "3"
 Parsers = "0.3, 1, 2"
 Tables = "1"
-julia = "1.6"
+julia = "1.10"
 
 [extras]
+Harbor = "af79dbb9-1a80-47ad-8928-192a4af69376"
+Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test"]
+test = ["Harbor", "Sockets", "Test"]
diff --git a/README.md b/README.md
@@ -16,12 +16,10 @@ Package for interfacing with MySQL databases from Julia via the MariaDB C connec
 [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://mysql.juliadatabases.org/stable)
 [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://mysql.juliadatabases.org/dev)
 
-## Contributing
-
-The tests require a MySQL DB to be running, which is provided by Docker:
-
-```sh
-docker compose up -d
-julia --project -e 'using Pkg; Pkg.test()'
-docker compose down
-```
+## Contributing
+
+The test suite manages its own temporary MySQL container via Harbor.jl. The only prerequisite is a working Docker daemon:
+
+```sh
+julia --project -e 'using Pkg; Pkg.test()'
+```
diff --git a/docker-compose.yml b/docker-compose.yml
diff --git a/src/MySQL.jl b/src/MySQL.jl
@@ -128,7 +128,7 @@ function setoptions!(mysql;
         ssl_crl::Union{AbstractString, Nothing}=nothing,
         ssl_crlpath::Union{AbstractString, Nothing}=nothing,
         passphrase::Union{AbstractString, Nothing}=nothing,
-        ssl_verify_server_cert::Union{Bool, Nothing}=nothing,
+        ssl_verify_server_cert::Union{Bool, Nothing}=false,
         ssl_enforce::Union{Bool, Nothing}=nothing,
         ssl_mode::Union{API.mysql_ssl_mode, Nothing}=nothing,
         default_auth::Union{AbstractString, Nothing}=nothing,
@@ -279,7 +279,7 @@ Connect to a MySQL database with provided `host`, `user`, and `passwd` positiona
   * `ssl_cipher::AbstractString`: Defines a list of permitted ciphers or cipher suites to use for TLS, like `"DHE-RSA-AES256-SHA"`
   * `ssl_crl::AbstractString`: Defines a path to a PEM file that should contain one or more revoked X509 certificates to use for TLS. This option requires that you use the absolute path, not a relative path.
   * `ssl_crlpath::AbstractString`: Defines a path to a directory that contains one or more PEM files that should each contain one revoked X509 certificate to use for TLS. This option requires that you use the absolute path, not a relative path. The directory specified by this option needs to be run through the openssl rehash command.
-  * `ssl_verify_server_cert::Bool`: Enables (or disables) server certificate verification.
+  * `ssl_verify_server_cert::Bool=false`: Enables (or disables) server certificate verification.
   * `ssl_enforce::Bool`: Whether to force TLS
   * `default_auth::AbstractString`: Default authentication client-side plugin to use.
   * `connection_handler::AbstractString`: Specify the name of a connection handler plugin.
diff --git a/src/api/capi.jl b/src/api/capi.jl
@@ -431,13 +431,21 @@ Zero for success. Nonzero if an error occurred; this occurs for option values th
 """=#
 function getoption(mysql::MYSQL, option::mysql_option)
     if option in CUINTOPTS
-        return @checksuccess mysql mysql_get_option_Cuint(mysql.ptr, option, Ref{Cuint}())
+        ref = Ref{Cuint}()
+        @checksuccess mysql mysql_get_option_Cuint(mysql.ptr, Int(option), ref)
+        return ref[]
     elseif option in CULONGOPTS
-        return @checksuccess mysql mysql_get_option_Culong(mysql.ptr, option, Ref{Culong}())
+        ref = Ref{Culong}()
+        @checksuccess mysql mysql_get_option_Culong(mysql.ptr, Int(option), ref)
+        return ref[]
     elseif option in BOOLOPTS
-        return @checksuccess mysql mysql_get_option_Bool(mysql.ptr, option, Ref{Bool}())
+        ref = Ref{Bool}()
+        @checksuccess mysql mysql_get_option_Bool(mysql.ptr, Int(option), ref)
+        return ref[]
     else
-        return @checksuccess mysql mysql_get_option_String(mysql.ptr, option, Ref{String}())
+        ref = Ref{Ptr{UInt8}}(C_NULL)
+        @checksuccess mysql mysql_get_option_String(mysql.ptr, Int(option), ref)
+        return ref[] == C_NULL ? nothing : unsafe_string(ref[])
     end
 end
 
diff --git a/src/api/ccalls.jl b/src/api/ccalls.jl
@@ -236,6 +236,13 @@ function mysql_get_option_Bool(mysql::Ptr{Cvoid}, option::Integer, arg::Ref{Bool
                 mysql, option, arg)
 end
 
+function mysql_get_option_String(mysql::Ptr{Cvoid}, option::Integer, arg::Ref{Ptr{UInt8}})
+    return @c(:mysql_get_option,
+                Cint,
+                (Ptr{Cvoid}, Cint, Ref{Ptr{UInt8}}),
+                mysql, option, arg)
+end
+
 function mysql_get_option_Cvoid(mysql::Ptr{Cvoid}, option::Integer, arg::Ptr{Cvoid})
     return @c(:mysql_get_option,
                 Cint,
diff --git a/src/execute.jl b/src/execute.jl
@@ -203,8 +203,9 @@ Base.IteratorSize(::Type{<:TextCursors}) = Base.SizeUnknown()
 function Base.iterate(cursor::TextCursors{buffered}, first=true) where {buffered}
     cursor.cursor.result.ptr == C_NULL && return nothing
     if !first
+        has_more_results = API.moreresults(cursor.cursor.conn.mysql)
         finalize(cursor.cursor.result)
-        if API.moreresults(cursor.cursor.conn.mysql)
+        if has_more_results
             @assert API.nextresult(cursor.cursor.conn.mysql) !== nothing
             cursor.cursor.result = buffered ? API.storeresult(cursor.cursor.conn.mysql) : API.useresult(cursor.cursor.conn.mysql)
             if buffered
@@ -222,4 +223,4 @@ function Base.iterate(cursor::TextCursors{buffered}, first=true) where {buffered
 end
 
 DBInterface.executemultiple(conn::Connection, sql::AbstractString, params=(); kw...) =
-    TextCursors(DBInterface.execute(conn, sql, params; kw...))
+    TextCursors(DBInterface.execute(conn, sql, params; kw...))
diff --git a/src/load.jl b/src/load.jl
@@ -101,10 +101,14 @@ function load(itr, conn::Connection, name::AbstractString="mysql_"*Random.randst
     DBInterface.transaction(conn) do
         params = chop(repeat("?,", length(sch.names)))
         stmt = DBInterface.prepare(conn, "INSERT INTO $name ($(join(sch.names .|> string .|> quoteid,", "))) VALUES ($params)")
-        for (i, row) in enumerate(rows)
-            i > limit && break
-            debug && @info "inserting row $i; $(Tables.Row(row))"
-            DBInterface.execute(stmt, Tables.Row(row))
+        try
+            for (i, row) in enumerate(rows)
+                i > limit && break
+                debug && @info "inserting row $i; $(Tables.Row(row))"
+                DBInterface.execute(stmt, Tables.Row(row))
+            end
+        finally
+            DBInterface.close!(stmt)
         end
     end
 
diff --git a/test/my.ini b/test/my.ini
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -1,18 +1,142 @@
-using Test, MySQL, DBInterface, Tables, Dates, DecFP
+using Test, MySQL, DBInterface, Tables, Dates, DecFP, Harbor, Sockets
 
-conn = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306)
+const MYSQL_IMAGE_REF = get(ENV, "MYSQL_IMAGE", "mysql:8")
+const MYSQL_TEST_USER = "root"
+const MYSQL_TEST_PASSWORD = ""
+
+struct MySQLTestConfig
+    host::String
+    port::Int
+    user::String
+    password::String
+end
+
+const TEST_CONFIG = Ref{Union{Nothing, MySQLTestConfig}}(nothing)
+const TEST_OPTION_FILE = Ref{Union{Nothing, String}}(nothing)
+
+function parse_image_ref(ref::String)
+    slash = findlast('/', ref)
+    colon = findlast(':', ref)
+    if colon !== nothing && (slash === nothing || colon > slash)
+        return String(ref[begin:prevind(ref, colon)]), String(ref[nextind(ref, colon):end])
+    end
+    return ref, "latest"
+end
+
+function docker_available()
+    Sys.which("docker") === nothing && return false
+    try
+        run(pipeline(`docker info`, stdout=devnull, stderr=devnull))
+        return true
+    catch
+        return false
+    end
+end
+
+function pick_port()
+    server = Sockets.listen(Sockets.IPv4(0), 0)
+    _, port = Sockets.getsockname(server)
+    port = Int(port)
+    close(server)
+    return port
+end
+
+test_config() = something(TEST_CONFIG[])
+test_host() = test_config().host
+test_port() = test_config().port
+test_user() = test_config().user
+test_password() = test_config().password
+test_option_file() = something(TEST_OPTION_FILE[])
+
+connect_mysql(; kw...) = DBInterface.connect(MySQL.Connection, test_host(), test_user(), test_password(); port=test_port(), kw...)
+
+function wait_for_connection(cfg::MySQLTestConfig; timeout::Float64=90.0)
+    start_time = time()
+    last_err = nothing
+    while time() - start_time < timeout
+        try
+            return DBInterface.connect(MySQL.Connection, cfg.host, cfg.user, cfg.password; port=cfg.port, connect_timeout=2)
+        catch err
+            last_err = err
+            sleep(0.5)
+        end
+    end
+    last_err === nothing && error("MySQL did not become ready")
+    error("MySQL did not become ready: $(sprint(showerror, last_err))")
+end
+
+function write_option_file(dir::AbstractString, cfg::MySQLTestConfig)
+    path = joinpath(dir, "my.ini")
+    open(path, "w") do io
+        print(io, """
+[client]
+host=$(cfg.host)
+user=$(cfg.user)
+port=$(cfg.port)
+connect_timeout=30
+report-data-truncation=true
+password = $(repr(cfg.password))
+""")
+    end
+    return path
+end
+
+function with_mysql(f::Function)
+    image, tag = parse_image_ref(MYSQL_IMAGE_REF)
+    host_port = pick_port()
+    env = Dict("MYSQL_ALLOW_EMPTY_PASSWORD" => "yes")
+    return Harbor.with_container(
+        image;
+        tag=tag,
+        ports=Dict(3306 => host_port),
+        environment=env,
+        wait_strategy=(pattern="ready for connections",),
+        wait_timeout=120.0,
+    ) do _
+        cfg = MySQLTestConfig("127.0.0.1", host_port, MYSQL_TEST_USER, MYSQL_TEST_PASSWORD)
+        conn = wait_for_connection(cfg)
+        DBInterface.close!(conn)
+        return f(cfg)
+    end
+end
+
+@testset "MySQL" begin
+
+let mysql = MySQL.API.init()
+    MySQL.setoptions!(mysql)
+    @test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_SSL_VERIFY_SERVER_CERT) == false
+    MySQL.setoptions!(mysql; ssl_verify_server_cert=true)
+    @test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_SSL_VERIFY_SERVER_CERT) == true
+    MySQL.setoptions!(mysql; connect_timeout=7)
+    @test Int(MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_CONNECT_TIMEOUT)) == 7
+    MySQL.setoptions!(mysql; max_allowed_packet=1024)
+    @test Int(MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_MAX_ALLOWED_PACKET)) == 1024
+    MySQL.setoptions!(mysql; bind="127.0.0.1")
+    @test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_BIND) == "127.0.0.1"
+end
+
+if !docker_available()
+    @info "Docker not available; skipping MySQL integration tests."
+    @test true
+else
+    with_mysql() do cfg
+        TEST_CONFIG[] = cfg
+        mktempdir() do dir
+            TEST_OPTION_FILE[] = write_option_file(dir, cfg)
+
+conn = connect_mysql()
 DBInterface.close!(conn)
 
 # https://github.com/JuliaDatabases/MySQL.jl/issues/170
-conn = DBInterface.connect(MySQL.Connection, "mysql://127.0.0.1", "root"; port=3306)
+conn = DBInterface.connect(MySQL.Connection, string("mysql://", test_host()), test_user(); port=test_port())
 DBInterface.close!(conn)
 
 # AbstractString as a connection parameter or an option
-conn = DBInterface.connect(MySQL.Connection, SubString("127.0.0.1"), SubString("root"), SubString(""); port=3306, charset_name=SubString("utf8mb4"))
+conn = DBInterface.connect(MySQL.Connection, SubString(test_host()), SubString(test_user()), SubString(test_password()); port=test_port(), charset_name=SubString("utf8mb4"))
 DBInterface.close!(conn)
 
 # load host/user + options from file
-conn = DBInterface.connect(MySQL.Connection, "", ""; option_file=joinpath(dirname(pathof(MySQL)), "../test/", "my.ini"))
+conn = DBInterface.connect(MySQL.Connection, "", ""; port=0, option_file=test_option_file())
 @test isopen(conn)
 
 DBInterface.execute(conn, "DROP DATABASE if exists mysqltest")
@@ -310,7 +434,8 @@ res = DBInterface.execute(stmt) |> columntable
 res = DBInterface.execute(stmt)
 res = DBInterface.execute(stmt)
 
-results = DBInterface.executemultiple(conn, "select * from Employee; select DeptNo, OfficeNo from Employee where OfficeNo IS NOT NULL")
+multi_conn = connect_mysql(db="mysqltest")
+results = DBInterface.executemultiple(multi_conn, "select * from Employee; select DeptNo, OfficeNo from Employee where OfficeNo IS NOT NULL")
 state = iterate(results)
 @test state !== nothing
 res, st = state
@@ -325,6 +450,7 @@ res, st = state
 @test length(res) == 4
 ret = columntable(res)
 @test length(ret[1]) == 4
+DBInterface.close!(multi_conn)
 
 # multiple-queries not supported by mysql w/ prepared statements
 @test_throws MySQL.API.StmtError DBInterface.prepare(conn, "select * from Employee; select DeptNo, OfficeNo from Employee where OfficeNo IS NOT NULL")
@@ -372,15 +498,15 @@ ret = columntable(res)
 @test_throws ArgumentError MySQL.load(ct, conn, "test194")
 
 @testset "transactions" begin
-    conn = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306)
+    conn = connect_mysql()
     try
         DBInterface.execute(conn, "DROP DATABASE if exists mysqltest")
         DBInterface.execute(conn, "CREATE DATABASE mysqltest")
         DBInterface.execute(conn, "use mysqltest")
         DBInterface.execute(conn, "DROP TABLE IF EXISTS TransactionTest")
         DBInterface.execute(conn, "CREATE TABLE TransactionTest (a int)")
 
-        conn2 = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306)
+        conn2 = connect_mysql()
         DBInterface.execute(conn2, "use mysqltest")
 
         try
@@ -414,4 +540,8 @@ ret = columntable(res)
     finally
         DBInterface.close!(conn)
     end
+end
+        end
+    end
+end
 end
