Skip to content

Commit 9cb9e8d

Browse files
Ericson2314Ericson2314
authored
Merge pull request #1700 from obsidiansystems/small-test-cleanup
Extract `QueueRunnerContext` from `QueueRunnerBuildOne`
2 parents 12f0568 + 8d9b3c3 commit 9cb9e8d

2 files changed

Lines changed: 109 additions & 79 deletions

File tree

‎subprojects/hydra-tests/lib/QueueRunnerBuildOne.pm‎

Lines changed: 8 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -2,47 +2,17 @@ use warnings;
22
use strict;
33

44
package QueueRunnerBuildOne;
5-
use IO::Socket::IP;
65
use IPC::Run;
76
use JSON::PP;
87
use LWP::UserAgent;
98
use HTTP::Request;
10-
use Hydra::Config;
9+
use QueueRunnerContext;
1110
our @ISA = qw(Exporter);
1211
our @EXPORT = qw(
1312
runBuild
1413
runBuilds
1514
);
1615

17-
sub _get_random_port {
18-
my ($min, $max) = @_;
19-
while (1) {
20-
my $port = $min + int(rand($max - $min + 1));
21-
my $sock = IO::Socket::IP->new(
22-
LocalAddr => '::',
23-
LocalPort => $port,
24-
Proto => 'tcp',
25-
ReuseAddr => 0,
26-
);
27-
if ($sock) {
28-
close($sock);
29-
return $port;
30-
}
31-
}
32-
}
33-
34-
sub _wait_for {
35-
my ($ua, $url, $check) = @_;
36-
for my $i (1..30) {
37-
my $resp = $ua->get($url);
38-
if ($resp->is_success) {
39-
return 1 if !$check || $check->($resp);
40-
}
41-
select(undef, undef, undef, 0.5);
42-
}
43-
return 0;
44-
}
45-
4616
sub _flush_stream {
4717
my ($label, $stream, $buf_ref, $final) = @_;
4818
return if $$buf_ref eq "";
@@ -70,52 +40,11 @@ sub runBuilds {
7040
ref $ctx eq 'HydraTestContext' or die "runBuilds requires a HydraTestContext as first argument\n";
7141
my @build_ids = map { $_->id } @builds;
7242

73-
my $grpc_port = _get_random_port(5000, 9999);
74-
my $http_port = _get_random_port(10000, 19999);
75-
76-
my $config_dir = $ENV{T2_HARNESS_TEMP_DIR}
77-
// $ctx->{central}{hydra_data};
78-
my $config_file = "$config_dir/config.toml";
79-
80-
# Read store settings from the Hydra config file.
81-
my $hydra_config_file = $ctx->{central}{hydra_config_file};
82-
my $hydra_config = ($hydra_config_file && -f $hydra_config_file)
83-
? Hydra::Config::loadConfig($hydra_config_file) : {};
84-
my $dest_store_uri = $hydra_config->{store_uri} // "";
85-
my $use_substitutes = $hydra_config->{'use-substitutes'} // "";
86-
87-
# Write the TOML config for the queue runner.
88-
{
89-
my $db_url = $ctx->{central}{hydra_database_url};
90-
open(my $fh, '>', $config_file) or die "Cannot write $config_file: $!\n";
91-
print $fh "dbUrl = \"$db_url\"\n";
92-
print $fh "hydraDataDir = \"$config_dir/data\"\n";
93-
print $fh "remoteStoreAddr = [\"$dest_store_uri\"]\n" if $dest_store_uri ne "";
94-
print $fh "useSubstitutes = true\n" if $use_substitutes eq "1";
95-
close($fh);
96-
}
43+
my ($qr_harness, $base_url, $grpc_port, $qr_out_ref, $qr_err_ref) = start_queue_runner($ctx,
44+
rust_log => "queue_runner=debug,info",
45+
);
9746

98-
my ($qr_in, $qr_out, $qr_err) = ("", "", "");
9947
my ($bl_in, $bl_out, $bl_err) = ("", "", "");
100-
101-
# Start the queue runner with central env applied.
102-
my $qr_harness;
103-
{
104-
local @ENV{keys %{$ctx->{central_env}}} = values %{$ctx->{central_env}};
105-
local $ENV{RUST_LOG} = "queue_runner=debug,info";
106-
local $ENV{NO_COLOR} = "1";
107-
$qr_harness = IPC::Run::start(
108-
["hydra-queue-runner",
109-
"--config-path", $config_file,
110-
"--rest-bind", "[::]:$http_port",
111-
"--grpc-bind", "[::]:$grpc_port",
112-
"--disable-queue-monitor-loop",
113-
],
114-
\$qr_in, \$qr_out, \$qr_err,
115-
);
116-
}
117-
118-
my $base_url = "http://[::1]:$http_port";
11948
my $ua = LWP::UserAgent->new(timeout => 2);
12049
my $bl_harness;
12150

@@ -124,7 +53,7 @@ sub runBuilds {
12453
local $SIG{ALRM} = sub { die "timeout\n" };
12554
alarm $timeout;
12655
# Wait for the REST server to become available.
127-
_wait_for($ua, "$base_url/status")
56+
wait_for_url($ua, "$base_url/status")
12857
or die "Timed out waiting for queue-runner REST server\n";
12958

13059
# Start the builder with its own store settings.
@@ -146,7 +75,7 @@ sub runBuilds {
14675
}
14776

14877
# Wait for the builder to register as a machine.
149-
_wait_for($ua, "$base_url/status/machines", sub {
78+
wait_for_url($ua, "$base_url/status/machines", sub {
15079
shift->decoded_content =~ /"hostname"/;
15180
}) or die "Timed out waiting for builder to register\n";
15281

@@ -167,7 +96,7 @@ sub runBuilds {
16796
$qr_harness->pump_nb;
16897
$bl_harness->pump_nb;
16998
# Flush accumulated output so logs are visible while waiting.
170-
_flush_harness("Queue runner", \$qr_out, \$qr_err);
99+
_flush_harness("Queue runner", $qr_out_ref, $qr_err_ref);
171100
_flush_harness("Builder", \$bl_out, \$bl_err);
172101
if (!$bl_harness->pumpable) {
173102
$bl_harness->finish;
@@ -201,7 +130,7 @@ sub runBuilds {
201130
_flush_harness("Builder", \$bl_out, \$bl_err, 1);
202131
}
203132
$qr_harness->kill_kill;
204-
_flush_harness("Queue runner", \$qr_out, \$qr_err, 1);
133+
_flush_harness("Queue runner", $qr_out_ref, $qr_err_ref, 1);
205134

206135
if (!$ok) {
207136
print STDERR "runBuilds failed: $err" if $err;

‎subprojects/hydra-tests/lib/QueueRunnerContext.pm‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
use warnings;
2+
use strict;
3+
4+
package QueueRunnerContext;
5+
use IO::Socket::IP;
6+
use IPC::Run;
7+
use LWP::UserAgent;
8+
use Hydra::Config;
9+
our @ISA = qw(Exporter);
10+
our @EXPORT = qw(
11+
get_random_port
12+
start_queue_runner
13+
wait_for_url
14+
);
15+
16+
sub get_random_port {
17+
my ($min, $max) = @_;
18+
while (1) {
19+
my $port = $min + int(rand($max - $min + 1));
20+
my $sock = IO::Socket::IP->new(
21+
LocalAddr => '::',
22+
LocalPort => $port,
23+
Proto => 'tcp',
24+
ReuseAddr => 0,
25+
);
26+
if ($sock) {
27+
close($sock);
28+
return $port;
29+
}
30+
}
31+
}
32+
33+
sub wait_for_url {
34+
my ($ua, $url, $check) = @_;
35+
for my $i (1..30) {
36+
my $resp = $ua->get($url);
37+
if ($resp->is_success) {
38+
return 1 if !$check || $check->($resp);
39+
}
40+
select(undef, undef, undef, 0.5);
41+
}
42+
return 0;
43+
}
44+
45+
# Start a queue runner process.
46+
# Returns ($harness, $http_url, $grpc_port, \$stdout_buf, \$stderr_buf).
47+
# Caller is responsible for calling $harness->kill_kill when done.
48+
sub start_queue_runner {
49+
my ($ctx, %opts) = @_;
50+
ref $ctx eq 'HydraTestContext' or die "start_queue_runner requires a HydraTestContext\n";
51+
52+
my $grpc_port = get_random_port(5000, 9999);
53+
my $http_port = get_random_port(10000, 19999);
54+
55+
my $config_dir = $ENV{T2_HARNESS_TEMP_DIR}
56+
// $ctx->{central}{hydra_data};
57+
my $config_file = "$config_dir/qr-config.toml";
58+
59+
# Read store settings from the Hydra config file.
60+
my $hydra_config_file = $ctx->{central}{hydra_config_file};
61+
my $hydra_config = ($hydra_config_file && -f $hydra_config_file)
62+
? Hydra::Config::loadConfig($hydra_config_file) : {};
63+
my $dest_store_uri = $hydra_config->{store_uri} // "";
64+
my $use_substitutes = $hydra_config->{'use-substitutes'} // "";
65+
66+
# Write the TOML config for the queue runner.
67+
{
68+
my $db_url = $ctx->{central}{hydra_database_url};
69+
open(my $fh, '>', $config_file) or die "Cannot write $config_file: $!\n";
70+
print $fh "dbUrl = \"$db_url\"\n";
71+
print $fh "hydraDataDir = \"$config_dir/data\"\n";
72+
print $fh "remoteStoreAddr = [\"$dest_store_uri\"]\n" if $dest_store_uri ne "";
73+
print $fh "useSubstitutes = true\n" if $use_substitutes eq "1";
74+
close($fh);
75+
}
76+
77+
my ($qr_in, $qr_out, $qr_err) = ("", "", "");
78+
79+
# Start the queue runner with central env applied.
80+
my $qr_harness;
81+
{
82+
local @ENV{keys %{$ctx->{central_env}}} = values %{$ctx->{central_env}};
83+
local $ENV{RUST_LOG} = $opts{rust_log} // "error";
84+
local $ENV{NO_COLOR} = "1";
85+
$qr_harness = IPC::Run::start(
86+
["hydra-queue-runner",
87+
"--config-path", $config_file,
88+
"--rest-bind", "[::]:$http_port",
89+
"--grpc-bind", "[::]:$grpc_port",
90+
"--disable-queue-monitor-loop",
91+
],
92+
\$qr_in, \$qr_out, \$qr_err,
93+
);
94+
}
95+
96+
my $base_url = "http://[::1]:$http_port";
97+
98+
return ($qr_harness, $base_url, $grpc_port, \$qr_out, \$qr_err);
99+
}
100+
101+
1;

0 commit comments

Comments
 (0)