Merge pull request #146 from okartau/run-qemu-test-on-builder

CI: various clean-ups and improvements

Author: mythi

Jenkinsfile

@@ -28,27 +28,27 @@ def global_sum_log = ""
 def added_commits = ""
 def builder_node = ""
 def builder_wspace = ""
-def slot_name = "builder-slot-"
+def slot_name = "ci-"
 // reasonable value: keep few recent, dont take risk to fill disk
 int num_build_dirs_to_keep = 4
 
 // Define global environment common for all docker sessions
 def script_env = """
     export WORKSPACE=\$PWD
     export HOME=\$JENKINS_HOME
-    export CURRENT_PROJECT=${current_project}
     export BUILD_CACHE_DIR=${env.PUBLISH_DIR}/bb-cache
     export GIT_PROXY_COMMAND=oe-git-proxy
     export CI_BUILD_ID=${ci_build_id}
     export GIT_COMMITTER_NAME="IOT Refkit CI"
     export GIT_COMMITTER_EMAIL='refkit-ci@yoctoproject.org'
     export TARGET_MACHINE=${target_machine}
+    export CI_LOG=bitbake-${target_machine}-${ci_build_id}.log
 """
 
 try {
     timestamps {
         node('rk-docker') {
-            ws("workspace/${slot_name}${env.EXECUTOR_NUMBER}") {
+            ws("workspace/${slot_name}${ci_build_id}") {
                 // remember node and workspace needed by workspace cleaner
                 builder_node = "${env.NODE_NAME}"
                 builder_wspace = "${env.WORKSPACE}"
@@ -102,11 +102,6 @@ try {
                     added_commits = readFile("added_commits")
                 }
             } // ws
-            // rotate workspace out of way, rename using CI_BUILD_ID.
-            // Older trees are deleted later in test_runs[], without keeping build waiting.
-            ws("workspace") {
-                sh "mv ${builder_wspace} ${builder_wspace}.ci-prev.${ci_build_id}"
-            }
         } // node
     } // timestamps
 
@@ -146,9 +141,12 @@ try {
                 } finally {
                     // read tests summary prepared by tester-exec.sh
                     // Here one tester adds it's summary piece to the global buffer.
-                    global_sum_log += readFile "results-summary-${test_device}.${img}.log"
-                    archiveArtifacts allowEmptyArchive: true,
-                                     artifacts: '*.log, *.xml'
+                    // Grab lock as we deal with global data from multiple workers
+                    lock(resource: "global_data") {
+                        global_sum_log += readFile "results-summary-${test_device}.${img}.log"
+                        archiveArtifacts allowEmptyArchive: true,
+                                         artifacts: '*.log, *.xml'
+                    }
                 }
                 // without locking we may lose tester result set(s)
                 // if testers run xunit step in nearly same time
@@ -220,12 +218,10 @@ def run_proxy_args() {
 
 def build_user_args() {
     dir(pwd([tmp:true])+"/.build_user_args") {
-         // get jenkins user uid/gid
-        sh "id -u > jenkins_uid"
+        // get jenkins user uid/gid
+        sh "id -u > jenkins_uid && id -g > jenkins_gid"
         jenkins_uid = readFile("jenkins_uid").trim()
-        sh "id -g > jenkins_gid"
         jenkins_gid = readFile("jenkins_gid").trim()
-        deleteDir()
     }
     return "--build-arg uid=${jenkins_uid} --build-arg gid=${jenkins_gid}"
 }
@@ -300,12 +296,17 @@ def step_xunit() {
 }
 
 // Delete older builder trees.
-// While majority/regular workspaces are named builder-slot-0, (for EXECUTOR=0),
-// Jenkins may create additional trees as builder-slot-0_X.
-// Wildcard covers all such workspaces, dont want some pattern filling disk independently.
+// While majority/regular workspaces are named ci-CI_BUILD_ID,
+// Jenkins may create additional trees as ci-build-CI_BUILD_ID_<NUM>
+// Regex with underscore should cover all such workspaces.
 def trim_build_dirs(slotname, num_to_keep) {
     sh """
-dirs=`find ${env.WORKSPACE} -mindepth 1 -maxdepth 1 -type d -name "${slotname}*.ci-prev.*" |sort -n |head -n -${num_to_keep} |tr '\n' ' '`
+# tmpdirs in separate pass
+dirs=`find . -mindepth 1 -maxdepth 1 -type d -regex ".*/${slotname}[0-9_-]*-build-[0-9_]*.*tmp\$" |sort -n |head -n -${num_to_keep} |tr '\n' ' '`
+if [ -n "\${dirs}" ]; then
+    ionice -c 3 rm -fr \$dirs
+fi
+dirs=`find . -mindepth 1 -maxdepth 1 -type d -regex ".*/${slotname}[0-9_-]*-build-[0-9_]*\$" |sort -n |head -n -${num_to_keep} |tr '\n' ' '`
 if [ -n "\${dirs}" ]; then
     ionice -c 3 rm -fr \$dirs
 fi

docker/build-common-util.sh

@@ -36,6 +36,11 @@ EOF
       # save sstate to workspace
       echo "SSTATE_DIR = \"${BUILD_CACHE_DIR}/sstate\"" >> conf/auto.conf
     fi
+    # lower compression levels in a PR build, to save build time
+    if [ -z ${CI_ARCHIVER_MODE+x} ]; then
+        echo "ZIP_COMPRESSION_LEVEL ?= \"-1\"" >> conf/auto.conf
+        echo "XZ_COMPRESSION_LEVEL ?= \"-0\"" >> conf/auto.conf
+    fi
 }
 
 auto_conf_archiver() {

docker/build-project.sh

@@ -103,8 +103,7 @@ bitbake -S none ${_bitbake_targets}
 
 if [ ! -z ${JOB_NAME+x} ]; then
   # CI run: save output to log file
-  LOG=$WORKSPACE/bitbake-${TARGET_MACHINE}-${CI_BUILD_ID}.log
-  bitbake ${_bitbake_targets} 2>&1 | tee -a $LOG
+  bitbake ${_bitbake_targets} 2>&1 | tee -a $WORKSPACE/$CI_LOG
 else
   bitbake ${_bitbake_targets}
 fi

docker/opensuse-42.1/Dockerfile

@@ -58,7 +58,7 @@ _DEPL=${_BRESULT}/deploy
 
 # create publishing destination structure and copy
 create_remote_dirs ${RSYNC_PUBLISH_DIR}/builds ${JOB_NAME}/${CI_BUILD_ID}
-[ -d ${_DEPL}/images ] && create_remote_dirs ${_RSYNC_DEST} images && rsync -avS --exclude=README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt ${_DEPL}/images/${TARGET_MACHINE} ${_RSYNC_DEST}/images/
+[ -d ${_DEPL}/images ] && create_remote_dirs ${_RSYNC_DEST} images && rsync -avS ${_DEPL}/images/${TARGET_MACHINE} ${_RSYNC_DEST}/images/
 [ -d ${_DEPL}/licenses ] && create_remote_dirs ${_RSYNC_DEST} licenses && rsync -az --ignore-existing ${_DEPL}/licenses ${_RSYNC_DEST}/
 [ -d ${_DEPL}/sources ] && create_remote_dirs ${_RSYNC_DEST} sources && rsync -av --ignore-existing ${_DEPL}/sources ${_RSYNC_DEST}/
 [ -d ${_DEPL}/tools ] && create_remote_dirs ${_RSYNC_DEST} tools && rsync -av --ignore-existing ${_DEPL}/tools ${_RSYNC_DEST}/
@@ -110,7 +110,7 @@ if [ -n "$(find ${_BRESULT}/log -maxdepth 1 -name 'isafw*' -print -quit)" ]; the
     fi
 fi
 
-LOG="$WORKSPACE/bitbake-${TARGET_MACHINE}-${CI_BUILD_ID}.log"
+LOG=$WORKSPACE/$CI_LOG
 if [ -f "${LOG}" ]; then
     xz -v -k ${LOG}
     rsync -avz ${LOG}* ${_RSYNC_DEST}/

docker/opensuse-tumbleweed/Dockerfile

@@ -72,11 +72,6 @@ testimg() {
   tar -xzf ${TEST_SUITE_FILE}
   tar -xzf ${TEST_CASES_FILE} -C iottest/
 
-  # Copy local WLAN settings to iottest over example file and chmod to readable
-  _WLANCONF=./iottest/oeqa/runtime/connectivity/wifi/files/config.ini
-  cp $HOME/.config.ini.wlan ${_WLANCONF}
-  chmod 644 ${_WLANCONF}
-
   FILENAME=${_IMG_NAME}-${MACHINE}-${CI_BUILD_ID}.wic
   set +e
   wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.bmap
@@ -114,6 +109,11 @@ testimg() {
   if [ "${DEVICE}" = "qemu" ]; then
     test_qemu
   else
+    # Copy local WLAN settings to iottest over example file and chmod to readable
+    _WLANCONF=./iottest/oeqa/runtime/connectivity/wifi/files/config.ini
+    cp $HOME/.config.ini.wlan ${_WLANCONF}
+    chmod 644 ${_WLANCONF}
+
     daft ${DEVICE} ${FILENAME} --record
     TEST_EXIT_CODE=$?
   fi