Skip to content

Commit 50d2505

Browse files
committed
fix(mysql): stabilize Harbor-backed test runs
Switch the test suite to a Harbor-managed MySQL container so local and CI testing only assumes Docker availability. Also restore a stable Connector/C SSL default, fix option reads, close load-time prepared statements, and advance multi-result cleanup via nextresult() so the suite stays green on current MariaDB Connector/C releases. Fixes #234
1 parent bfdcb57 commit 50d2505

10 files changed

Lines changed: 179 additions & 82 deletions

File tree

.github/workflows/ci.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
- x64
2222
steps:
2323
- uses: actions/checkout@v5
24-
- run: docker compose up -d
24+
- run: docker info
2525
- uses: julia-actions/setup-julia@v2
2626
with:
2727
version: ${{ matrix.version }}
@@ -50,4 +50,4 @@ jobs:
5050
- run: julia --project=docs docs/make.jl
5151
env:
5252
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
53-
DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
53+
DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}

Project.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ Tables = "1"
2424
julia = "1.6"
2525

2626
[extras]
27+
Harbor = "af79dbb9-1a80-47ad-8928-192a4af69376"
28+
Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
2729
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
2830

2931
[targets]
30-
test = ["Test"]
32+
test = ["Harbor", "Sockets", "Test"]

README.md

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,10 @@ Package for interfacing with MySQL databases from Julia via the MariaDB C connec
1616
[![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://mysql.juliadatabases.org/stable)
1717
[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://mysql.juliadatabases.org/dev)
1818

19-
## Contributing
20-
21-
The tests require a MySQL DB to be running, which is provided by Docker:
22-
23-
```sh
24-
docker compose up -d
25-
julia --project -e 'using Pkg; Pkg.test()'
26-
docker compose down
27-
```
19+
## Contributing
20+
21+
The test suite manages its own temporary MySQL container via Harbor.jl. The only prerequisite is a working Docker daemon:
22+
23+
```sh
24+
julia --project -e 'using Pkg; Pkg.test()'
25+
```

docker-compose.yml

Lines changed: 0 additions & 29 deletions
This file was deleted.

src/MySQL.jl

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,12 @@ function clear!(conn, result::API.MYSQL_RES)
5555
if conn.mysql.ptr != C_NULL && result.ptr != C_NULL
5656
while true
5757
if API.fetchrow(conn.mysql, result) == C_NULL
58-
if API.moreresults(conn.mysql)
59-
finalize(result)
60-
@assert API.nextresult(conn.mysql) !== nothing
61-
result = API.useresult(conn.mysql)
62-
else
58+
nxt = API.nextresult(conn.mysql)
59+
if nxt === nothing
6360
break
6461
end
62+
finalize(result)
63+
result = API.useresult(conn.mysql)
6564
end
6665
end
6766
finalize(result)
@@ -128,7 +127,7 @@ function setoptions!(mysql;
128127
ssl_crl::Union{AbstractString, Nothing}=nothing,
129128
ssl_crlpath::Union{AbstractString, Nothing}=nothing,
130129
passphrase::Union{AbstractString, Nothing}=nothing,
131-
ssl_verify_server_cert::Union{Bool, Nothing}=nothing,
130+
ssl_verify_server_cert::Union{Bool, Nothing}=false,
132131
ssl_enforce::Union{Bool, Nothing}=nothing,
133132
ssl_mode::Union{API.mysql_ssl_mode, Nothing}=nothing,
134133
default_auth::Union{AbstractString, Nothing}=nothing,
@@ -279,7 +278,7 @@ Connect to a MySQL database with provided `host`, `user`, and `passwd` positiona
279278
* `ssl_cipher::AbstractString`: Defines a list of permitted ciphers or cipher suites to use for TLS, like `"DHE-RSA-AES256-SHA"`
280279
* `ssl_crl::AbstractString`: Defines a path to a PEM file that should contain one or more revoked X509 certificates to use for TLS. This option requires that you use the absolute path, not a relative path.
281280
* `ssl_crlpath::AbstractString`: Defines a path to a directory that contains one or more PEM files that should each contain one revoked X509 certificate to use for TLS. This option requires that you use the absolute path, not a relative path. The directory specified by this option needs to be run through the openssl rehash command.
282-
* `ssl_verify_server_cert::Bool`: Enables (or disables) server certificate verification.
281+
* `ssl_verify_server_cert::Bool=false`: Enables (or disables) server certificate verification.
283282
* `ssl_enforce::Bool`: Whether to force TLS
284283
* `default_auth::AbstractString`: Default authentication client-side plugin to use.
285284
* `connection_handler::AbstractString`: Specify the name of a connection handler plugin.

src/api/capi.jl

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -431,13 +431,21 @@ Zero for success. Nonzero if an error occurred; this occurs for option values th
431431
"""=#
432432
function getoption(mysql::MYSQL, option::mysql_option)
433433
if option in CUINTOPTS
434-
return @checksuccess mysql mysql_get_option_Cuint(mysql.ptr, option, Ref{Cuint}())
434+
ref = Ref{Cuint}()
435+
@checksuccess mysql mysql_get_option_Cuint(mysql.ptr, Int(option), ref)
436+
return ref[]
435437
elseif option in CULONGOPTS
436-
return @checksuccess mysql mysql_get_option_Culong(mysql.ptr, option, Ref{Culong}())
438+
ref = Ref{Culong}()
439+
@checksuccess mysql mysql_get_option_Culong(mysql.ptr, Int(option), ref)
440+
return ref[]
437441
elseif option in BOOLOPTS
438-
return @checksuccess mysql mysql_get_option_Bool(mysql.ptr, option, Ref{Bool}())
442+
ref = Ref{Bool}()
443+
@checksuccess mysql mysql_get_option_Bool(mysql.ptr, Int(option), ref)
444+
return ref[]
439445
else
440-
return @checksuccess mysql mysql_get_option_String(mysql.ptr, option, Ref{String}())
446+
ref = Ref{String}()
447+
@checksuccess mysql mysql_get_option_String(mysql.ptr, Int(option), ref)
448+
return ref[]
441449
end
442450
end
443451

src/execute.jl

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -204,22 +204,21 @@ function Base.iterate(cursor::TextCursors{buffered}, first=true) where {buffered
204204
cursor.cursor.result.ptr == C_NULL && return nothing
205205
if !first
206206
finalize(cursor.cursor.result)
207-
if API.moreresults(cursor.cursor.conn.mysql)
208-
@assert API.nextresult(cursor.cursor.conn.mysql) !== nothing
209-
cursor.cursor.result = buffered ? API.storeresult(cursor.cursor.conn.mysql) : API.useresult(cursor.cursor.conn.mysql)
210-
if buffered
211-
cursor.cursor.nrows = API.numrows(cursor.cursor.result)
212-
end
213-
cursor.cursor.nfields = API.numfields(cursor.cursor.result)
214-
fields = API.fetchfields(cursor.cursor.result, cursor.cursor.nfields)
215-
cursor.cursor.names = [ccall(:jl_symbol_n, Ref{Symbol}, (Cstring, Csize_t), x.name, x.name_length) for x in fields]
216-
cursor.cursor.types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), cursor.cursor.mysql_date_and_time) for x in fields]
217-
else
207+
nxt = API.nextresult(cursor.cursor.conn.mysql)
208+
if nxt === nothing
218209
return nothing
219210
end
211+
cursor.cursor.result = buffered ? API.storeresult(cursor.cursor.conn.mysql) : API.useresult(cursor.cursor.conn.mysql)
212+
if buffered
213+
cursor.cursor.nrows = API.numrows(cursor.cursor.result)
214+
end
215+
cursor.cursor.nfields = API.numfields(cursor.cursor.result)
216+
fields = API.fetchfields(cursor.cursor.result, cursor.cursor.nfields)
217+
cursor.cursor.names = [ccall(:jl_symbol_n, Ref{Symbol}, (Cstring, Csize_t), x.name, x.name_length) for x in fields]
218+
cursor.cursor.types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), cursor.cursor.mysql_date_and_time) for x in fields]
220219
end
221220
return cursor.cursor, false
222221
end
223222

224223
DBInterface.executemultiple(conn::Connection, sql::AbstractString, params=(); kw...) =
225-
TextCursors(DBInterface.execute(conn, sql, params; kw...))
224+
TextCursors(DBInterface.execute(conn, sql, params; kw...))

src/load.jl

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,14 @@ function load(itr, conn::Connection, name::AbstractString="mysql_"*Random.randst
101101
DBInterface.transaction(conn) do
102102
params = chop(repeat("?,", length(sch.names)))
103103
stmt = DBInterface.prepare(conn, "INSERT INTO $name ($(join(sch.names .|> string .|> quoteid,", "))) VALUES ($params)")
104-
for (i, row) in enumerate(rows)
105-
i > limit && break
106-
debug && @info "inserting row $i; $(Tables.Row(row))"
107-
DBInterface.execute(stmt, Tables.Row(row))
104+
try
105+
for (i, row) in enumerate(rows)
106+
i > limit && break
107+
debug && @info "inserting row $i; $(Tables.Row(row))"
108+
DBInterface.execute(stmt, Tables.Row(row))
109+
end
110+
finally
111+
DBInterface.close!(stmt)
108112
end
109113
end
110114

test/my.ini

Lines changed: 0 additions & 7 deletions
This file was deleted.

test/runtests.jl

Lines changed: 130 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,137 @@
1-
using Test, MySQL, DBInterface, Tables, Dates, DecFP
1+
using Test, MySQL, DBInterface, Tables, Dates, DecFP, Harbor, Sockets
22

3-
conn = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306)
3+
const MYSQL_IMAGE_REF = get(ENV, "MYSQL_IMAGE", "mysql:8")
4+
const MYSQL_TEST_USER = "root"
5+
const MYSQL_TEST_PASSWORD = ""
6+
7+
struct MySQLTestConfig
8+
host::String
9+
port::Int
10+
user::String
11+
password::String
12+
end
13+
14+
const TEST_CONFIG = Ref{Union{Nothing, MySQLTestConfig}}(nothing)
15+
const TEST_OPTION_FILE = Ref{Union{Nothing, String}}(nothing)
16+
17+
function parse_image_ref(ref::String)
18+
slash = findlast('/', ref)
19+
colon = findlast(':', ref)
20+
if colon !== nothing && (slash === nothing || colon > slash)
21+
return String(ref[begin:prevind(ref, colon)]), String(ref[nextind(ref, colon):end])
22+
end
23+
return ref, "latest"
24+
end
25+
26+
function docker_available()
27+
Sys.which("docker") === nothing && return false
28+
try
29+
run(pipeline(`docker info`, stdout=devnull, stderr=devnull))
30+
return true
31+
catch
32+
return false
33+
end
34+
end
35+
36+
function pick_port()
37+
server = Sockets.listen(Sockets.IPv4(0), 0)
38+
_, port = Sockets.getsockname(server)
39+
port = Int(port)
40+
close(server)
41+
return port
42+
end
43+
44+
test_config() = something(TEST_CONFIG[])
45+
test_host() = test_config().host
46+
test_port() = test_config().port
47+
test_user() = test_config().user
48+
test_password() = test_config().password
49+
test_option_file() = something(TEST_OPTION_FILE[])
50+
51+
connect_mysql(; kw...) = DBInterface.connect(MySQL.Connection, test_host(), test_user(), test_password(); port=test_port(), kw...)
52+
53+
function wait_for_connection(cfg::MySQLTestConfig; timeout::Float64=90.0)
54+
start_time = time()
55+
last_err = nothing
56+
while time() - start_time < timeout
57+
try
58+
return DBInterface.connect(MySQL.Connection, cfg.host, cfg.user, cfg.password; port=cfg.port, connect_timeout=2)
59+
catch err
60+
last_err = err
61+
sleep(0.5)
62+
end
63+
end
64+
last_err === nothing && error("MySQL did not become ready")
65+
error("MySQL did not become ready: $(sprint(showerror, last_err))")
66+
end
67+
68+
function write_option_file(dir::AbstractString, cfg::MySQLTestConfig)
69+
path = joinpath(dir, "my.ini")
70+
open(path, "w") do io
71+
print(io, """
72+
[client]
73+
host=$(cfg.host)
74+
user=$(cfg.user)
75+
port=$(cfg.port)
76+
connect_timeout=30
77+
report-data-truncation=true
78+
password = $(repr(cfg.password))
79+
""")
80+
end
81+
return path
82+
end
83+
84+
function with_mysql(f::Function)
85+
image, tag = parse_image_ref(MYSQL_IMAGE_REF)
86+
host_port = pick_port()
87+
env = Dict("MYSQL_ALLOW_EMPTY_PASSWORD" => "yes")
88+
return Harbor.with_container(
89+
image;
90+
tag=tag,
91+
ports=Dict(3306 => host_port),
92+
environment=env,
93+
# Harbor's port wait is enough here because we also poll for a real client connection below.
94+
wait_strategy=(port=3306,),
95+
wait_timeout=120.0,
96+
) do _
97+
cfg = MySQLTestConfig("127.0.0.1", host_port, MYSQL_TEST_USER, MYSQL_TEST_PASSWORD)
98+
conn = wait_for_connection(cfg)
99+
DBInterface.close!(conn)
100+
return f(cfg)
101+
end
102+
end
103+
104+
@testset "MySQL" begin
105+
106+
let mysql = MySQL.API.init()
107+
MySQL.setoptions!(mysql)
108+
@test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_SSL_VERIFY_SERVER_CERT) == false
109+
MySQL.setoptions!(mysql; ssl_verify_server_cert=true)
110+
@test MySQL.API.getoption(mysql, MySQL.API.MYSQL_OPT_SSL_VERIFY_SERVER_CERT) == true
111+
end
112+
113+
if !docker_available()
114+
@info "Docker not available; skipping MySQL integration tests."
115+
@test true
116+
else
117+
with_mysql() do cfg
118+
TEST_CONFIG[] = cfg
119+
mktempdir() do dir
120+
TEST_OPTION_FILE[] = write_option_file(dir, cfg)
121+
122+
conn = connect_mysql()
4123
DBInterface.close!(conn)
5124

6125
# https://github.com/JuliaDatabases/MySQL.jl/issues/170
7-
conn = DBInterface.connect(MySQL.Connection, "mysql://127.0.0.1", "root"; port=3306)
126+
conn = DBInterface.connect(MySQL.Connection, string("mysql://", test_host()), test_user(); port=test_port())
8127
DBInterface.close!(conn)
9128

10129
# AbstractString as a connection parameter or an option
11-
conn = DBInterface.connect(MySQL.Connection, SubString("127.0.0.1"), SubString("root"), SubString(""); port=3306, charset_name=SubString("utf8mb4"))
130+
conn = DBInterface.connect(MySQL.Connection, SubString(test_host()), SubString(test_user()), SubString(test_password()); port=test_port(), charset_name=SubString("utf8mb4"))
12131
DBInterface.close!(conn)
13132

14133
# load host/user + options from file
15-
conn = DBInterface.connect(MySQL.Connection, "", ""; option_file=joinpath(dirname(pathof(MySQL)), "../test/", "my.ini"))
134+
conn = DBInterface.connect(MySQL.Connection, "", ""; port=0, option_file=test_option_file())
16135
@test isopen(conn)
17136

18137
DBInterface.execute(conn, "DROP DATABASE if exists mysqltest")
@@ -372,15 +491,15 @@ ret = columntable(res)
372491
@test_throws ArgumentError MySQL.load(ct, conn, "test194")
373492

374493
@testset "transactions" begin
375-
conn = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306)
494+
conn = connect_mysql()
376495
try
377496
DBInterface.execute(conn, "DROP DATABASE if exists mysqltest")
378497
DBInterface.execute(conn, "CREATE DATABASE mysqltest")
379498
DBInterface.execute(conn, "use mysqltest")
380499
DBInterface.execute(conn, "DROP TABLE IF EXISTS TransactionTest")
381500
DBInterface.execute(conn, "CREATE TABLE TransactionTest (a int)")
382501

383-
conn2 = DBInterface.connect(MySQL.Connection, "127.0.0.1", "root", ""; port=3306)
502+
conn2 = connect_mysql()
384503
DBInterface.execute(conn2, "use mysqltest")
385504

386505
try
@@ -414,4 +533,8 @@ ret = columntable(res)
414533
finally
415534
DBInterface.close!(conn)
416535
end
536+
end
537+
end
538+
end
539+
end
417540
end

0 commit comments

Comments
 (0)