os-autoinst-obs-auto-submit

#!/bin/bash

set -euo pipefail

export LC_ALL=C
color=${color:-auto}
src_project=${src_project:-devel:openQA}
dst_project=${dst_project:-${src_project}:tested}
staging_project=${staging_project:-${src_project}:testing}
submit_target="${submit_target:-"openSUSE:Factory"}"
dry_run="${dry_run:-"0"}"
osc_poll_interval="${osc_poll_interval:-2}"
osc_build_start_poll_tries="${osc_build_start_poll_tries:-90}"
throttle_days=${throttle_days:-2}
throttle_days_leap_16=${throttle_days_leap_16:-7}
git_user=${git_user:-os-autoinst-obs-workflow}
XMLSTARLET=$(command -v xmlstarlet || true)
[[ -n $XMLSTARLET ]] || (echo "Need xmlstarlet" >&2 && exit 1)

prefix="${prefix:-""}"
[ "$dry_run" = "1" ] && prefix="echo"

# shellcheck source=/dev/null
. "$(dirname "${BASH_SOURCE[0]}")"/_common

git=${git:-"$prefix retry -e -- git"}
git_obs=${git_obs:-"$prefix retry -e -- git-obs"}
git_obs_no_retry=${git_obs_no_retry:-"$prefix git-obs"}

# Use fixed Backport version for now
submit_target_extra=${submit_target_extra:-"openSUSE:Backports:SLE-15-SP7:Update,openSUSE:Leap:16.0"}
IFS=',' read -ra targets <<< "$submit_target,$submit_target_extra"
failed_packages=()

# declare Git branches of submit targets which have already been migrated to Git
declare -A git_branches=(
    ['openSUSE:Leap:16.0']=leap-16.0
)

# allow specifying target-specific throttle variables
declare -A throttle_variables=(
    ['openSUSE:Leap:16.0']=throttle_days_leap_16
)

encode_variable() {
    #  https://stackoverflow.com/a/298258
    perl -MURI::Escape -e 'print uri_escape($ARGV[0])' "$1"
}

get_obs_sr_id() {
    local target=$1
    local dst_project=$2
    local package=$3
    local rc=0
    # The project in the request target will be different than the original
    # submit target for Backport requests
    [[ $target =~ Backports ]] && target="openSUSE:Maintenance"
    local states actions
    states=$(encode_variable "state/@name='declined' or state/@name='new' or state/@name='review'")
    actions=$(encode_variable "action/target/@project='$target' and action/source/@project='$dst_project' and action/source/@package='$package'")
    debug_osc api "/search/request/id?match=($states)%20and%20($actions)" | xmlstarlet sel -t -v "/collection/request/@id" || rc=$?
    [[ $rc == 1 ]] && return 0
    return "$rc"
}

reenable_buildtime_services() {
    sed -i -e 's,mode="buildtime" mode="disabled",mode="buildtime",' _service
}

wait_for_package_build() {
    local target=$1
    log-info "wait_for_package_build $target"
    target_escaped=$(echo -n "$target" | sed -e 's|\:|_|g') # avoid, e.g. "repoid 'openSUSE:Factory' is illegal"
    local osc_query="https://api.opensuse.org/public/build/$dst_project/_result?repository=$target_escaped&package=$package"
    local wip_states='\(unknown\|blocked\|scheduled\|dispatching\|building\|signing\|finished\)'
    local bad_states='\(failed\|unresolvable\|broken\)'
    local attempts="$osc_build_start_poll_tries"
    while ! curl -sS "$osc_query" | grep -e "$wip_states"; do
        if [[ $attempts -le 0 ]]; then
            log-warn "Re-build of $package has not been considered in time (or package has been re-built so fast that we've missed it)"
            break
        fi
        log-info "Waiting for re-build of $package to be considered (attempts left: $attempts)"
        sleep "$osc_poll_interval"
        attempts=$((attempts - 1))
    done
    while curl -sS "$osc_query" | grep -e "$wip_states"; do
        log-info "Waiting while $package is in progress"
        sleep "$osc_poll_interval"
    done
    if curl -sS "$osc_query" | grep -e "$bad_states"; then
        log-info "Building $package has failed, skipping submission"
        return 1
    fi
}

make_obs_submit_request() {
    # shellcheck disable=SC2206
    local package=$1 target=$2 version=$3 cmd=(debug_osc sr) req='' # $osc is supposed to be split, hence disabling SC2206
    req=$(get_obs_sr_id "$target" "$dst_project" "$package") || return $?
    # TODO: check if it's a different revision than HEAD
    [[ -n $req ]] && cmd+=(-s "$req")
    "${cmd[@]}" -m "Update to ${version}" "$target" || return $?
}

make_git_submit_request() {
    local package=$1 target=$2 version=$3 branch=$4 forked_repo='' fork_rc=0 pr_out='' pr_rc=''
    title="Update to $version"
    description="Automatic submission via https://github.com/os-autoinst/os-autoinst-scripts/blob/master/os-autoinst-obs-auto-submit"

    # ensure a directory to store Git repositories under has been created
    git_dir=../git-repos
    mkdir -p "$git_dir"

    # ensure the user we are authenticating with has a fork of the package we want to submit
    # note: For `gib-obs` commands to work one needs to create a config file, e.g. via:
    #       `git-obs login add --url=https://src.opensuse.org obs --set-as-default --user=… --token=…`
    #       For `git push` to work (done below) you need to deploy an SSH key on the Gitea instance.
    #       Checkout https://opensuse-commander.readthedocs.io/en/latest/git-obs-quickstart.html for details.
    fork_output=$($git_obs repo fork "pool/$package" 2>&1 > /dev/null) || fork_rc=$?
    forked_repo=$(echo "$fork_output" | perl -ne 'print for /Fork .*: (.*)/g')
    if [[ $fork_rc != 0 ]] || [[ -z $forked_repo ]]; then
        log-error "Unable to create fork, $git_obs exited with return code $fork_rc and output:\n$fork_output"
        [[ -e $HOME/.config/tea/config.yml ]] \
            || log-info "Be sure you have followed https://opensuse-commander.readthedocs.io/en/latest/git-obs-quickstart.html"
        return 1
    fi

    # enter a working copy of the fork and switch to the branch the submission should be based on
    $git_obs repo clone --no-ssh-strict-host-key-checking "$forked_repo" || return $?
    cd "$package"
    $git switch "$branch"

    # base our changes on the latest state of the target repository (which is added as remote "parent" by git-obs)
    $git fetch parent || return $?
    $git reset --hard "parent/$branch"

    # remove everything and replace it with the updated files we have just created (in the calling function)
    rm -v ./*
    cp -v ../../"$package"/* ./

    # commit and force push
    $git add --all
    $git commit --message "$title"
    $git push -f || return $?

    # create PR
    local sleep_time=3 remaining_attempts=4
    while [[ $pr_rc != 0 ]]; do
        pr_out=$($git_obs_no_retry pr create --title "$title" --description "$description" --target-branch="$branch" 2>&1) && return 0 || pr_rc=$?
        echo "$pr_out"
        # do just the force push if the PR already exists
        [[ $pr_out =~ pull\ request\ already\ exists ]] && return 0
        # otherwise try again with exponential backoff (can't use retry here as we need to filter out the non-error case when the PR already exists)
        remaining_attempts=$((remaining_attempts - 1))
        [[ $remaining_attempts -lt 1 ]] && break
        echo "Retrying up to $remaining_attempts more times after sleeping $sleep_time s …"
        sleep "$sleep_time"
        sleep_time=$((sleep_time * 2))
    done
    return "$pr_rc"
}

update_package() {
    log-info "update_package $*"
    local package=$1 version=
    xmlstarlet ed -L -i "/services/service" --type attr -n mode --value 'disabled' _service
    sed -i -e 's,mode="disabled" mode="disabled",mode="disabled",' _service
    reenable_buildtime_services
    # ignore those updates
    rm -f _service:*-test.changes
    cp -v .osc/*-test.changes . 2> /dev/null || :
    for file in _service:*; do
        # shellcheck disable=SC2001
        mv -v "$file" "$(echo "$file" | sed -e 's,.*:,,')"
    done
    version=$(sed -n 's/^Version:\s*//p' ./*.spec || sed -n 's/^version:\s*//p' ./*.obsinfo)
    rm -f ./*rpmlintrc _servicedata node_modules.sums package-lock.json
    debug_osc addremove
    sed -i '/rpmlintrc/d' ./*.spec || true
    local ci_args=""
    if [[ "$package" == "os-autoinst-distri-opensuse-deps" || "$package" =~ "container" ]]; then
        ci_args="--noservice"
    fi
    # We would get an empty changelog when there are no changes to files contained
    # in this package/container. So add some text.
    sed -i -e 's/^  \* $/  * Update to latest openQA version/' ./*.changes
    debug_osc status
    number_of_relevant_changed_files=$(debug_osc status | grep -v --count --perl-regexp '\.changes$')
    if [[ $number_of_relevant_changed_files -lt 1 ]]; then
        log-info "## Not updating package '$dst_project/$package' as only changelogs were modified"
        return 0
    fi
    debug_osc ci -m "Offline generation of ${version}" $ci_args
    rc=0
    wait_for_package_build "$submit_target"
    for target in "${targets[@]}"; do
        [[ $target == none ]] && continue # allow specifying submit_target_extra=none for easier testing
        has_pending_submission "$package" "$target" || continue
        log-info "## Ready to submit $package to $target ##"
        local submit_cmd=make_obs_submit_request
        local args=("$package" "$target" "$version")
        local git_branch=${git_branches[$target]:-}
        [[ $git_branch ]] && submit_cmd=make_git_submit_request args+=("$git_branch")
        "$submit_cmd" "${args[@]}" || rc=$? failed_packages+=("$package:$?")
    done
    return $rc
}

sync_changesrevision() {
    log-info "sync_changesrevision $*"
    local dir=$1
    local package=$2
    local target_rev=$3
    path="$dir/$package"
    $prefix xmlstarlet ed -L -u "//param[@name='changesrevision']" -v "$target_rev" "$path"/_servicedata
    if ! diff -u "$path"/.osc/_servicedata "$path"/_servicedata; then
        debug_osc up "$path"
        debug_osc cat "$submit_target" "$package" "$package".changes > "$path/$package".changes
        debug_osc ci -m "sync changesrevision with $submit_target" "$path"
        debug_osc up --server-side-source-service-files "$path"
    fi
}

get_spec_changes_in_requirements() {
    local package=$1
    local project=$2
    { diff -u "$package-factory.spec" "$project/$package/_service:obs_scm:$package.spec" || :; } | grep '^[+-]Requires'
}

generate_os-autoinst-distri-opensuse-deps_changelog() {
    local dir=$1
    local package=$2
    debug_osc cat "$submit_target" "$package" "$package".changes > "$package"-factory.changes
    {
        echo "-------------------------------------------------------------------"
        echo "$(LANG=c date -u) - Dominik Heidler <dheidler@suse.de>"
        echo
        get_spec_changes_in_requirements "$package" "$dir" \
            | sed -e 's/Requires:\s*/dependency /g' -e 's/^-/- Removed /g' -e 's/^+/- Added /g'
        echo
        cat "$package"-factory.changes
    } > "$dir/$package/$package".changes
}

handle_auto_submit() {
    log-info "handle_auto_submit $*"
    local package=$1
    debug_osc service wait "$src_project" "$package"
    debug_osc co --server-side-source-service-files "$src_project"/"$package"
    if [[ "$package" == "os-autoinst-distri-opensuse-deps" ]]; then
        debug_osc cat "$submit_target" "$package" "$package.spec" > "$package-factory.spec"
        if get_spec_changes_in_requirements "$package" "$src_project"; then
            # dependency added or removed
            generate_os-autoinst-distri-opensuse-deps_changelog "$src_project" "$package"
        else
            log-info "No dependency changes for $package"
            return
        fi
    else
        # skip submission if there is nothing new to submit
        target_rev=$(last_revision "$submit_target" "$package")
        if [[ "$target_rev" == "$(last_revision "$src_project" "$package")" ]]; then
            log-info "Latest revision already in $submit_target"
            return
        fi
    fi
    debug_osc service wait "$dst_project" "$package"
    debug_osc co "$dst_project"/"$package"
    rm "$dst_project"/"$package"/*
    cp -v "$src_project"/"$package"/* "$dst_project"/"$package"/
    (
        cd "$dst_project"/"$package"
        update_package "$package"
    )
}

usage() {
    cat << EOF
Usage: $0 [OPTIONS]

Prepare and submit os-autoinst packages to target projects like
openSUSE:Factory.

Can be configured additionally with environment variables.

Options:
 -h, --help         display this help
EOF
    exit "$1"
}

opts=$(getopt -o h -l help -n "$0" -- "$@") || usage 1
eval set -- "$opts"
while true; do
    case "$1" in
        -h | --help) usage 0 ;;
        --)
            shift
            break
            ;;
        *) break ;;
    esac
done

get_project_packages() {
    echo "${packages:-$(debug_osc ls "$dst_project" | grep -v '\-test$')}"
}

has_pending_submission() {
    local package=$1 target=$2
    local git_branch=${git_branches[$target]:-}
    local throttle_variable=${throttle_variables[$target]:-throttle_days}
    local throttle_days_value=${!throttle_variable:-0}
    local requestlist recent_pr_url
    [[ $throttle_days_value == 0 ]] && return 0
    log-info "has_pending_submission($package, $target)"

    # throttle number of submissions to allow only one PR or SR within a certain number of days
    if [[ $git_branch ]]; then
        # check for PR on Gitea
        recent_pr_url=$($git_obs -q api "repos/pool/$package/pulls?state=open&sort=recentupdate" 2> /dev/null \
            | jq --raw-output \
                --arg days "$throttle_days_value" \
                --arg target "$git_branch" \
                --arg login "$git_user" \
                '[.[] | select(.base.ref == $target) | select(.user.login == $login) | select(.updated_at | strptime("%Y-%m-%dT%H:%M:%S%z") | mktime > (now - ($days | tonumber) * 86400)) | .html_url] | first')
        if [[ $recent_pr_url != null ]]; then
            log-info "Skipping submission, there is the still pending PR '$recent_pr_url' by $git_user targeting $git_branch younger than $throttle_days_value days."
            return 1
        fi
    else
        # check for SR on OBS
        # note: Avoid using `grep --quiet` here to keep consuming input so osc does not run into "BrokenPipeError: [Errno 32] Broken pipe".
        requestlist=$(debug_osc request list --project "$dst_project" --package "$package" --type submit,maintenance_incident --state new,review --days "$throttle_days_value")
        if echo "$requestlist" | grep --fixed-strings "$target" > /dev/null; then
            log-info "Skipping submission, there is still a pending SR for package $package in project $target younger than $throttle_days_value days."
            echo "$requestlist"
            return 1
        fi
    fi
    return 0
}

submit() {
    log-info "submit()"
    # submit from staging project if specified
    local must_cleanup_staging=
    if [[ -n $staging_project && $staging_project != none && $staging_project != "$src_project" ]]; then
        src_project=$staging_project
        must_cleanup_staging=1
    fi
    if [[ -e job_post_skip_submission ]]; then
        log-info "Skipping submission, reason: "
        cat job_post_skip_submission
        return 0
    fi
    (
        cd "$TMPDIR"
        rc=0
        auto_submit_packages=$(get_project_packages)
        for package in $auto_submit_packages; do
            handle_auto_submit "$package" || rc=$?
            # delete package from staging project
            [[ $must_cleanup_staging ]] && debug_osc rdelete -m "Cleaning up $package from $staging_project for next submission" "$staging_project" "$package"
        done
        if [[ ${#failed_packages[@]} -gt 0 ]]; then
            log-warn "Failed packages:"
            for item in "${failed_packages[@]}"; do
                package=${item%:*}
                exit_status=${item#*:}
                log-warn "- $package (exit status: $exit_status)"
            done
        fi
        return "$rc"
    )
}

main() {
    local rc=0
    submit "$@" || rc=$?
    [[ -e job_post_skip_submission ]] || cleanup-obs-project devel:openQA:testing 'I am sure'
    exit "$rc"
}

TMPDIR=${TMPDIR:-$(mktemp -d -t os-autoinst-obs-auto-submit-XXXX)}
trap 'rm -rf "$TMPDIR"' EXIT

caller 0 > /dev/null || main "$@"