test: Fix flaky full-stack tests by mocking service ports

Motivation:
Multiple full-stack tests were failing intermittently when run in parallel
due to "Address already in use" errors. They all defaulted to the same
base port (9526), causing conflicts in concurrent test environments.

Design Choices:
- Restored mock_service_ports in OpenQA::Test::Utils which was previously
  removed as dead code in 8d7b9a5. It mocks OpenQA::Utils::service_port
  via Test::MockModule to allocate independent free ports per service using
  Mojo::IOLoop::Server->generate_port, which asks the OS for a truly free
  port instead of blindly guessing.
- Applied mock_service_ports to t/05-scheduler-full.t, t/25-cache-service.t,
  t/33-developer_mode.t, t/43-scheduling-and-worker-scalability.t, and
  t/full-stack.t, replacing the prior setup_random_base_port approach.
- Each service gets an independently chosen free port (not a fixed-offset
  block), eliminating the risk that one occupied port breaks all services.
- Removed duplicate randomization logic and redundant comments across
  multiple test files.
- Cleaned up BEGIN blocks and imports in full-stack tests.

Benefits:
- Guarantees free ports via OS-level allocation, eliminating race conditions
  that setup_random_base_port could not prevent.
- Eliminates port conflicts during parallel test execution.
- Improves test stability and reliability for CI/CD.
- Centralizes logic for easier future maintenance and consistent
  port selection strategy.
- Cleaner and more maintainable test code.

Author: okurz

t/05-scheduler-full.t

@@ -27,16 +27,17 @@ use OpenQA::Scheduler::Client;
 use OpenQA::Scheduler::Model::Jobs;
 use OpenQA::WebAPI;
 use OpenQA::Worker::WebUIConnection;
-use OpenQA::Utils;
+use OpenQA::Utils qw(service_port);
 require OpenQA::Test::Database;
 use OpenQA::Test::Utils qw(
-  setup_mojo_app_with_default_worker_timeout
+  mock_service_ports setup_mojo_app_with_default_worker_timeout
   setup_fullstack_temp_dir create_user_for_workers
   create_webapi setup_share_dir create_websocket_server
   stop_service unstable_worker
   unresponsive_worker broken_worker rejective_worker
   wait_for simulate_load
 );
+mock_service_ports;
 use OpenQA::Test::TimeLimit '150';
 
 # treat this test like the fullstack test

t/25-cache-service.t

@@ -10,15 +10,16 @@ $ENV{MOJO_LOG_LEVEL} = 'info';
 my $tempdir;
 
 BEGIN {
-    use Mojo::File qw(path tempdir);
-
     $ENV{OPENQA_CACHE_SERVICE_QUIET} = $ENV{HARNESS_IS_VERBOSE} ? 0 : 1;
     $ENV{OPENQA_CACHE_ATTEMPTS} = 3;
     $ENV{OPENQA_CACHE_ATTEMPT_SLEEP_TIME} = 0;
     $ENV{OPENQA_RSYNC_RETRY_PERIOD} = 0;
     $ENV{OPENQA_RSYNC_RETRIES} = 1;
     $ENV{OPENQA_METRICS_DOWNLOAD_SIZE} = 1024;
-    $ENV{OPENQA_BASE_PORT} = 20000 + int rand 10000;
+}
+
+BEGIN {
+    use Mojo::File qw(path tempdir);
     $ENV{OPENQA_TEST_WAIT_INTERVAL} = 0.05;
 
     $tempdir = tempdir;
@@ -33,9 +34,6 @@ CACHEWORKERS = 10
 CACHELIMIT = 100');
 }
 
-use FindBin;
-use lib "$FindBin::Bin/lib", "$FindBin::Bin/../external/os-autoinst-common/lib";
-
 use Test::Warnings ':report_warnings';
 use Test::Output qw(stderr_like);
 use Test::Mojo;
@@ -46,15 +44,19 @@ use Mojo::IOLoop::Server;
 use POSIX '_exit';
 use Mojo::IOLoop::ReadWriteProcess qw(queue process);
 use Mojo::IOLoop::ReadWriteProcess::Session 'session';
+use FindBin;
+use lib "$FindBin::Bin/lib", "$FindBin::Bin/../external/os-autoinst-common/lib";
 use OpenQA::Test::Utils
-  qw(fake_asset_server cache_minion_worker cache_worker_service wait_for_or_bail_out perform_minion_jobs wait_for);
+  qw(mock_service_ports fake_asset_server cache_minion_worker cache_worker_service wait_for_or_bail_out perform_minion_jobs wait_for);
 use OpenQA::Test::TimeLimit '90';
 use Mojo::Util qw(md5_sum);
 use OpenQA::CacheService;
 use OpenQA::CacheService::Request;
 use OpenQA::CacheService::Client;
 use Time::Seconds;
 
+mock_service_ports;
+
 my $cachedir = $ENV{OPENQA_CACHE_DIR};
 my $port = Mojo::IOLoop::Server->generate_port;
 my $host = "http://localhost:$port";

t/33-developer_mode.t

@@ -33,9 +33,11 @@ use File::Path qw(make_path remove_tree);
 use Module::Load::Conditional 'can_load';
 use OpenQA::Utils qw(service_port);
 use OpenQA::Test::Utils qw(
+  mock_service_ports
   create_websocket_server create_scheduler create_live_view_handler setup_share_dir setup_fullstack_temp_dir
   start_worker stop_service
 );
+mock_service_ports;
 use OpenQA::Test::FullstackUtils;
 use OpenQA::SeleniumTest;
 
@@ -48,9 +50,13 @@ my $livehandler;
 my $scheduler;
 
 sub turn_down_stack {
-    stop_service($_) for ($worker, $ws, $livehandler, $scheduler);
+    stop_service $scheduler;
+    stop_service $livehandler;
+    stop_service $ws;
+    stop_service $worker;
 }
 
+
 driver_missing unless check_driver_modules;
 
 # setup directories

t/43-scheduling-and-worker-scalability.t

@@ -23,9 +23,10 @@ require OpenQA::Test::Database;
 use OpenQA::Jobs::Constants;
 use OpenQA::Log qw(setup_log);
 use OpenQA::Test::Utils qw(
-  setup_mojo_app_with_default_worker_timeout
+  mock_service_ports setup_mojo_app_with_default_worker_timeout
   create_user_for_workers create_webapi create_websocket_server
   stop_service setup_fullstack_temp_dir simulate_load);
+mock_service_ports;
 use OpenQA::Test::TimeLimit '20';
 use OpenQA::Utils 'testcasedir';
 

t/full-stack.t

@@ -49,7 +49,8 @@ use Module::Load::Conditional 'can_load';
 use OpenQA::Test::Utils
   qw(create_websocket_server create_live_view_handler setup_share_dir),
   qw(cache_minion_worker cache_worker_service setup_fullstack_temp_dir),
-  qw(start_worker stop_service wait_for_or_bail_out);
+  qw(mock_service_ports start_worker stop_service wait_for_or_bail_out);
+mock_service_ports;
 use OpenQA::Test::FullstackUtils;
 
 plan skip_all => 'set FULLSTACK=1 (be careful)' unless $ENV{FULLSTACK};

t/lib/OpenQA/Test/Utils.pm

@@ -50,6 +50,7 @@ BEGIN {
 
 our (@EXPORT, @EXPORT_OK);
 @EXPORT_OK = qw(
+  mock_service_ports
   setup_mojo_app_with_default_worker_timeout create_user_for_workers
   create_webapi create_websocket_server create_scheduler create_live_view_handler
   unresponsive_worker broken_worker rejective_worker setup_share_dir setup_fullstack_temp_dir run_gru_job
@@ -71,6 +72,18 @@ our (@EXPORT, @EXPORT_OK);
 #
 # Potentially this approach can also be used in production code.
 
+sub mock_service_ports {
+    my %ports;
+    Test::MockModule->new('OpenQA::Utils')->redefine(
+        service_port => sub {
+            my ($service) = @_;
+            my $port = $ports{$service} //= Mojo::IOLoop::Server->generate_port;
+            note("Mocking service port for $service to be $port");
+            return $port;
+        });
+    note('Used ports: ' . dumper(\%ports));
+}
+
 sub setup_mojo_app_with_default_worker_timeout ($class = 'Mojolicious') {
     my $app = $class->new(config => {global => {worker_timeout => DEFAULT_WORKER_TIMEOUT}}, log => undef);
     OpenQA::App->set_singleton($app);