Add additional repository support

M4rtinK · M4rtinK · commit a92fb9130b7d · 2021-01-26T17:34:30.000+01:00
Make it possible to add additional repo for a kickstart test run.

At the moment we assume the repository is a regular RPM repository,
as for example created by running createrepo_c on a directory
containing a bunch of RPMs.

The main aim of this functionality is to make it possible to add
additional packages to the RPM transaction Anaconda runs during a
regular RPM based installation. Also by adding packages with a higher
version number than those in the normal package repositories it
is possible to override them with the package from the additional repo.

This can be useful to test Anaconda and/or Initial Setup scratchbuilds.

To use this feature use the -a or --additional-repo flag to the appropriate
runner script and pass either a local filesystem path to a RPM
repository or to a remote RPM repository.

If a local path is detected a localhost web server on the runner will be used to
serve the repo folder to the VM running the test. If a remote path to a
RPM repository is passed, it will be passed directly to the VM.

In both cases the additional repositories are passed via the
inst.addrepo boot option as "inst.addrepo=KSTEST_ADDITIONAL_REPOSITORY,&lt;repo_path&gt;".
diff --git a/containers/runner/launch b/containers/runner/launch
@@ -34,6 +34,7 @@ Options:
  -t, --testtype TYPE                  Only run TYPE tests
  -s, --skip-testtypes TYPE[,TYPE..]   Don't run tests with given types
  -u, --updates PATH|URL               Set updates.img path or URL
+ -a, --additional-repo PATH|URL       Set additional repo path or URL
  -r, --retry                          Retry failed tests once, to guard against random
                                       infrastructure failures
  --daily-iso TOKEN_FILE               Download and use daily boot.iso instead of rawhide's
@@ -46,7 +47,7 @@ EOF
 }
 
 # parse options
-eval set -- "$(getopt -o j:p:t:s:u:rh --long jobs:,platform:,testtype:,skip-testtypes:,updates:,retry,daily-iso:,defaults:,run-args:,help -- "$@")"
+eval set -- "$(getopt -o j:p:t:s:u:a:rh --long jobs:,platform:,testtype:,skip-testtypes:,updates:additional-repo:,retry,daily-iso:,defaults:,run-args:,help -- "$@")"
 
 while true; do
     case "${1:-}" in
@@ -55,6 +56,7 @@ while true; do
         -t|--testtype) shift; TESTTYPE="$1" ;;
         -s|--skip-testtypes) shift; SKIP_TESTTYPES="$1" ;;
         -u|--updates) shift; UPDATES_IMAGE="$1" ;;
+        -a|--additional-repo) shift; ADDITIONAL_REPO="$1" ;;
         -r|--retry) TEST_RETRY=1 ;;
         --daily-iso) shift; DAILY_ISO_TOKEN="$1" ;;
         --defaults) shift; DEFAULTS_SH="$1" ;;
@@ -103,6 +105,15 @@ elif [ -n "${UPDATES_IMAGE:-}" ]; then
     UPDATES_IMG_ARGS="--env UPDATES_IMAGE=$UPDATES_IMAGE"
 fi
 
+# support both path and URL for additional repo
+if [ -e "${ADDITIONAL_REPO:-}" ]; then
+    # local folder; bind mount into container
+    ADDITIONAL_REPO_ARGS="-v $ADDITIONAL_REPO:/kstest_additional_repo:ro,Z --env ADDITIONAL_REPO=/kstest_additional_repo"
+elif [ -n "${ADDITIONAL_REPO:-}" ]; then
+    # URL, pass through
+    ADDITIONAL_REPO_ARGS="--env ADDITIONAL_REPO=$ADDITIONAL_REPO"
+fi
+
 if [ -n "${DEFAULTS_SH:-}" ]; then
     DEFAULTS_SH_ARGS="-v $DEFAULTS_SH:/home/kstest/.kstests.defaults.sh:ro,z"
 fi
@@ -126,6 +137,7 @@ fi
 set -x
 $CRUN run -it --rm --device=/dev/kvm --publish 127.0.0.1::16509 $PODMAN_SELINUX_FIX \
     --env KSTESTS_TEST="$KSTESTS_TEST" --env TESTTYPE="${TESTTYPE:-}" --env SKIP_TESTTYPES="${SKIP_TESTTYPES:-}" \
-    --env TEST_JOBS="$TEST_JOBS" --env PLATFORM="${PLATFORM:-}" --env TEST_RETRY="${TEST_RETRY:-}" ${UPDATES_IMG_ARGS:-} ${CONTAINER_RUN_ARGS:-} \
+    --env TEST_JOBS="$TEST_JOBS" --env PLATFORM="${PLATFORM:-}" --env TEST_RETRY="${TEST_RETRY:-}" \
+    ${UPDATES_IMG_ARGS:-} ${ADDITIONAL_REPO_ARGS:-} ${CONTAINER_RUN_ARGS:-} \
     ${VAR_TMP:-} -v "$PWD/data:/opt/kstest/data:z" -v "$BASEDIR:/kickstart-tests:ro,z" ${DEFAULTS_SH_ARGS:-} \
     $CONTAINER /kickstart-tests/containers/runner/run-kstest
diff --git a/containers/runner/run-kstest b/containers/runner/run-kstest
@@ -30,6 +30,11 @@ if [ -n "${UPDATES_IMAGE}" ]; then
   UPDATES_IMAGE_ARG="-u ${UPDATES_IMAGE}"
 fi
 
+ADDITIONAL_REPO_ARG=""
+if [ -n "${ADDITIONAL_REPO}" ]; then
+  ADDITIONAL_REPO_ARG="-a ${ADDITIONAL_REPO}"
+fi
+
 PLATFORM_ARG=""
 if [ -n "${PLATFORM}" ]; then
     PLATFORM_ARG="-p ${PLATFORM}"
@@ -74,7 +79,7 @@ fi
 TEST_LOG=/var/tmp/kstest.log
 pushd ${KSTESTS_DIR}
 set +e
-scripts/run_kickstart_tests.sh -k ${KSTESTS_KEEP} -i ${ISO_DIR}/${BOOT_ISO} ${UPDATES_IMAGE_ARG} ${PLATFORM_ARG} ${TESTTYPE_ARG} ${SKIP_TESTTYPES_ARG} ${RETRY_ARG} ${KSTESTS_TEST} 2>&1 | tee $TEST_LOG
+scripts/run_kickstart_tests.sh -k ${KSTESTS_KEEP} -i ${ISO_DIR}/${BOOT_ISO} ${UPDATES_IMAGE_ARG} ${ADDITIONAL_REPO_ARG} ${PLATFORM_ARG} ${TESTTYPE_ARG} ${SKIP_TESTTYPES_ARG} ${RETRY_ARG} ${KSTESTS_TEST} 2>&1 | tee $TEST_LOG
 RC=$?
 set -e
 popd
diff --git a/scripts/launcher/lib/conf/configuration.py b/scripts/launcher/lib/conf/configuration.py
@@ -83,6 +83,7 @@ def __init__(self):
         self._image_path = ""
         self._keep_option = KeepLevel.NOTHING
         self._updates_img_path = ""
+        self._additional_repo_path = ""
         self._append_host_id = False
         self._hung_task_timeout_secs = 1200
 
@@ -138,6 +139,14 @@ def updates_img_path(self):
     def updates_img_path(self, val):
         self._updates_img_path = val
 
+    @property
+    def additional_repo_path(self):
+        return self._additional_repo_path
+
+    @additional_repo_path.setter
+    def additional_repo_path(self, val):
+        self._additional_repo_path = val
+
     @property
     def script_path(self):
         path = os.path.join(__file__, "..")
diff --git a/scripts/launcher/lib/conf/runner_parser.py b/scripts/launcher/lib/conf/runner_parser.py
@@ -85,6 +85,9 @@ def _configure_parser(self):
         self._parser.add_argument("--updates", '-u', metavar="Path",
                                   type=str, dest="updates_path",
                                   help="Updates image path used in the test")
+        self._parser.add_argument("--additional_repo", '-a', metavar="Path",
+                                  type=str, dest="additional_repo_path",
+                                  help="Additionl repo path used in the test")
         self._parser.add_argument("--append-host-id", default=False, action="store_true",
                                   dest="append_host_id",
                                   help="append an id of the host running the test to the result")
@@ -115,6 +118,9 @@ def get_configuration(self):
         if ns.updates_path:
             conf.updates_img_path = ns.updates_path
 
+        if ns.additional_repo_path:
+            conf.additional_repo_path = ns.additional_repo_path
+
         if ns.append_host_id:
             conf.append_host_id = ns.append_host_id
 
diff --git a/scripts/launcher/run_one_test.py b/scripts/launcher/run_one_test.py
@@ -135,6 +135,9 @@ def _create_virtual_conf(self, log_path) -> VirtualConfiguration:
         if self._conf.updates_img_path:
             kernel_args += " inst.updates={}".format(self._conf.updates_img_path)
 
+        if self._conf.additional_repo_path:
+            kernel_args += " inst.addrepo=KSTEST_ADDITIONAL_REPO,{}".format(self._conf.additional_repo_path)
+
         if self._conf.hung_task_timeout_secs:
             kernel_args += " inst.kernel.hung_task_timeout_secs={}".format(
                 self._conf.hung_task_timeout_secs)
diff --git a/scripts/run_kickstart_tests.sh b/scripts/run_kickstart_tests.sh
@@ -61,10 +61,13 @@ KEEPIT=${KEEPIT:-0}
 # responsibility, this can break tests.
 UPDATES_IMG=""
 
+# Local or remote path to an additional repo that will be added for the installation transaction.
+ADDITIONAL_REPO=""
+
 TESTTYPE=""
 SKIP_TESTTYPES=""
 
-while getopts ":i:k:t:s:u:b:p:o:r" opt; do
+while getopts ":i:k:t:s:u:a:b:p:o:r" opt; do
     case $opt in
        i)
            # If this wasn't set from the environment, set it from the command line
@@ -97,6 +100,12 @@ while getopts ":i:k:t:s:u:b:p:o:r" opt; do
            # This may not be compatible with all the tests.
            UPDATES_IMG=$OPTARG
            ;;
+       a)
+           # Path to a RPM repo on a server or in local directory. This will be added as
+           # a kernel parameter inst.addrepo=KSTEST_ADDITIONAL_REPO,<server> to the VM boot options.
+           # This may not be compatible with all the tests.
+           ADDITIONAL_REPO=$OPTARG
+           ;;
        b)
            # Use additional boot options. Will be added to kernel_args from .sh file.
            BOOT_ARGS=$OPTARG
@@ -118,7 +127,7 @@ while getopts ":i:k:t:s:u:b:p:o:r" opt; do
            RETRY=--retry
            ;;
        *)
-           echo "Usage: run_kickstart_tests.sh [-i boot.iso] [-k 0|1|2] [-t test_type_to_run] [-s test_types_to_ignore] [-u link_to_updates.img] [-b additional_boot_options] [-p platform_name] [-o ksappend_overrides] [tests]"
+           echo "Usage: run_kickstart_tests.sh [-i boot.iso] [-k 0|1|2] [-t test_type_to_run] [-s test_types_to_ignore] [-u link_to_updates.img] [-a local_or_remote_repo_path] [-b additional_boot_options] [-p platform_name] [-o ksappend_overrides] [tests]"
            exit 1
            ;;
     esac
@@ -351,6 +360,22 @@ if [[ -n "$UPDATES_IMG" ]]; then
     UPDATES_ARG="-u ${UPDATES_IMG}"
 fi
 
+# set up additional package repo
+ADDITIONAL_REPO_ARG=""
+if [[ -n "$ADDITIONAL_REPO" ]]; then
+    if [ -e "$ADDITIONAL_REPO" ]; then
+        # set up a local web server for the local runner local
+        # folder containing the repo
+        python3 -m http.server --directory ${ADDITIONAL_REPO} 9999 &
+        # stop it when this script exits
+        trap "kill $!" EXIT INT QUIT PIPE
+        # SLIRP networking address as seem from QEMU guests
+        ADDITIONAL_REPO="http://10.0.2.2:9999/${ADDITIONAL_REPO}"
+    fi
+
+    ADDITIONAL_REPO_ARG="-a ${ADDITIONAL_REPO}"
+fi
+
 BOOT_ARG=""
 if [[ -n "$BOOT_ARGS" ]]; then
     BOOT_ARG="-b \"${BOOT_ARGS}\""
@@ -416,7 +441,7 @@ if [[ "$TEST_REMOTES" != "" ]]; then
                                                                -i ../install_images/${_IMAGE} \
                                                                -k ${KEEPIT} \
                                                                --append-host-id \
-                                                               ${RETRY} ${UPDATES_ARG} ${BOOT_ARG} {} ::: ${tests}
+                                                               ${RETRY} ${UPDATES_ARG} ${ADDITIONAL_REPO_ARG} ${BOOT_ARG} {} ::: ${tests}
     rc=$?
     cd -
 
@@ -448,7 +473,7 @@ else
                                                       -i ${IMAGE} \
                                                       -k ${KEEPIT} \
                                                       --append-host-id \
-                                                      ${RETRY} ${UPDATES_ARG} ${BOOT_ARG} {} ::: ${tests}
+                                                      ${RETRY} ${UPDATES_ARG} ${ADDITIONAL_REPO_ARG} ${BOOT_ARG} {} ::: ${tests}
     rc=$?
 fi
 
