added a script to bisect regressions (#4286)

firewave · web-flow · commit 41d2d1b0b570 · 2023-02-07T21:59:52.000+01:00
diff --git a/tools/bisect/README.md b/tools/bisect/README.md
@@ -0,0 +1,91 @@
+# Bisecting
+
+NOTE: THIS IS WORK IN PROGRESS
+
+`bisect.sh` is a script to bisect issues.
+
+## Command
+
+```
+./bisect.sh <hash-good> <hash-bad> "<cppcheck-options>"
+```
+
+`hash-good` - the last known good commit hash - in case of daca it is the last tagged minor release (not patch release - i.e. 2.x)
+`hash-bad` - the known bad commit hash - in case of daca the one from the `head-info:` line
+`cppcheck-options` - the options for the Cppcheck invokation - in case of daca the ones from the `cppcheck-options:` line and the path to the folder/file to scan
+
+If possible use `main` as the function to test stuff with since it won't emit an `unusedFunction` warning.
+
+## Bisecting scan time regressions
+
+We use daca to track differences in scan time. An overview of regressions in scan time can be found at http://cppcheck1.osuosl.org:8000/time_gt.html.
+
+You need to download the archive as specified by the second line in the output and extract it.
+
+If the overall scan time regressed you need to specify the whole folder.
+
+If a timeout (potential hang) was introduced you can simply specify the file from `error: Internal error: Child process crashed with signal 15 [cppcheckError]`.
+
+
+## Bisecting result regressions
+
+Results regressions are being bisected based on the `--error-exitcode=` result.
+If nothing is found the result will be `0` and it is treated as a _good_ commit.
+If a finding occurs the result will be `1` which is treated as a _bad_ commit.
+
+### False positive
+
+Provide a code sample which will trigger the false postive.
+
+```cpp
+// cppcheck-suppress unusedFunction
+static void f()
+{
+    <code triggering FP>
+}
+```
+
+### False negative
+
+Provide a code sample which will trigger a `unmatchedSuppression`.
+
+```cpp
+// cppcheck-suppress unusedFunction
+static void f()
+{
+    // cppcheck-suppress unreadVariable
+    int i;
+}
+```
+
+## Notes
+
+### Compilation issues:
+
+- 2.5 and before can only be built with GCC<=10 because of missing includes caused by cleanups within the standard headers. You need to specify `CXX=g++-10`.
+- 1.88 and 1.89 cannot be compiled:
+```
+make: python: No such file or directory
+```
+- 1.39 to 1.49 (possibly more versions - 1.54 and up work) cannot be compiled:
+```
+lib/mathlib.cpp:70:42: error: invalid conversion from ‘char’ to ‘char**’ [-fpermissive]
+   70 |         return std::strtoul(str.c_str(), '\0', 16);
+      |                                          ^~~~
+      |                                          |
+      |                                          char
+```
+- some commits between 2.0 and 2.2 cannot be compiled:
+```
+cli/cppcheckexecutor.cpp:333:22: error: size of array ‘mytstack’ is not an integral constant-expression
+  333 | static char mytstack[MYSTACKSIZE]= {0}; // alternative stack for signal handler
+  |                      ^~~~~~~~~~~
+```
+RESOLVED: a hot-patch is applied before compilation.
+- some commits between 1.54 and 1.55 cannot be compiled:
+```
+lib/preprocessor.cpp:2103:5: error: ‘errorLogger’ was not declared in this scope; did you mean ‘_errorLogger’?
+2103 |     errorLogger->reportInfo(errmsg);
+|     ^~~~~~~~~~~
+|     _errorLogger
+```
diff --git a/tools/bisect/bisect.sh b/tools/bisect/bisect.sh
@@ -0,0 +1,96 @@
+#!/bin/sh
+
+# TODO: set -e
+set -x
+
+# TODO: check parameters
+hash_good=$1
+hash_bad=$2
+options=$3
+
+# TODO: verify "good" commit happened before "bad" commit
+
+hang=0
+expected=""
+
+script_dir="$(dirname "$(realpath "$0")")"
+
+# TODO: make configurable
+bisect_dir=~/.bisect
+
+mkdir -p "$bisect_dir" || exit 1
+
+cd "$bisect_dir" || exit 1
+
+if [ ! -d 'cppcheck' ]; then
+  git clone https://github.com/danmar/cppcheck.git || exit 1
+fi
+
+bisect_repo_dir="$bisect_dir/cppcheck"
+
+cd $bisect_repo_dir || exit 1
+
+git fetch --all --tags || exit 1
+
+# clean up in case we previously exited prematurely
+git restore . || exit 1
+git clean -df || exit 1
+
+# reset potentially unfinished bisect - also reverts to 'main' branch
+git bisect reset || exit 1
+
+# update `main` branch
+git pull || exit 1
+
+# TODO: filter addons, cfg and platforms based on the options
+# limit to paths which actually affect the analysis
+git bisect start -- Makefile 'addons/*.py' 'cfg/*.cfg' 'cli/*.cpp' 'cli/*.h' 'externals/**/*.cpp' 'externals/**/*.h' 'lib/*.cpp' 'lib/*.h' platforms tools/matchcompiler.py || exit 1
+
+git checkout "$hash_good" || exit 1
+
+if [ $hang -eq 1 ]; then
+  # TODO: exitcode overflow on 255
+  # get expected time from good commit
+  python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options"
+  elapsed_time=$?
+else
+  # verify the given commit is actually "good"
+  python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected"
+  # shellcheck disable=SC2181
+  if [ $? -ne 0 ]; then
+    echo "given good commit is not actually good"
+    exit 1
+  fi
+fi
+
+# mark commit as "good"
+git bisect good || exit 1
+
+git checkout "$hash_bad" || exit 1
+
+# verify the given commit is actually "bad"
+if [ $hang -eq 1 ]; then
+  python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time
+else
+  python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected"
+fi
+
+if [ $? -ne 1 ]; then
+  echo "given bad commit is not actually bad"
+  exit 1
+fi
+
+# mark commit as "bad"
+git bisect bad || exit 1
+
+# perform the actual bisect
+if [ $hang -eq 1 ]; then
+  git bisect run python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time || exit 1
+else
+  git bisect run python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected" || exit 1
+fi
+
+# show the bisect log
+git bisect log || exit 1
+
+git bisect reset || exit 1
diff --git a/tools/bisect/bisect_common.py b/tools/bisect/bisect_common.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+import subprocess
+import os
+import shutil
+
+EC_GOOD = 0 # tells bisect that the commit is "good"
+EC_BAD = 1 # tells bisect that the commit is "bad"
+EC_SKIP = 125 # tells bisect to skip this commit since it cannot be tested
+
+def build_cppcheck(bisect_path):
+    commit_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode('ascii').strip()
+    install_path = os.path.join(bisect_path, commit_hash)
+    cppcheck_path = os.path.join(install_path, 'cppcheck')
+    if os.path.exists(install_path):
+        print('binary for {} already exists'.format(commit_hash))
+        return cppcheck_path
+
+    bisect_repo_dir = os.path.join(bisect_path, 'cppcheck')
+
+    if os.path.exists(os.path.join(bisect_repo_dir, 'cppcheck')):
+        os.remove(os.path.join(bisect_repo_dir, 'cppcheck'))
+
+    print('patching cli/cppcheckexecutor.cpp')
+    subprocess.check_call(['sed', '-i', 's/SIGSTKSZ/32768/g', os.path.join(bisect_repo_dir, 'cli', 'cppcheckexecutor.cpp')])
+
+    # TODO: older versions do not build because of include changes in libstdc++ - check compiler version and try to use an earlier one
+    # TODO: make jobs configurable
+    # TODO: use "make install"?
+    # TODO: allow CXXFLAGS overrides to workaround compiling issues in older versions
+    print('building {}'.format(commit_hash))
+    subprocess.check_call(['make', '-C', bisect_repo_dir, '-j6', 'MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -w -pipe', '-s'])
+
+    # TODO: remove folder if installation failed
+    print('installing {}'.format(commit_hash))
+    os.mkdir(install_path)
+    if os.path.exists(os.path.join(bisect_repo_dir, 'cfg')):
+        shutil.copytree(os.path.join(bisect_repo_dir, 'cfg'), os.path.join(install_path, 'cfg'))
+    if os.path.exists(os.path.join(bisect_repo_dir, 'platforms')):
+        shutil.copytree(os.path.join(bisect_repo_dir, 'platforms'), os.path.join(install_path, 'platforms'))
+    shutil.copy(os.path.join(bisect_repo_dir, 'cppcheck'), cppcheck_path)
+
+    # reset the patches so the subsequent checkout works
+    print('resetting repo')
+    subprocess.check_call(['git', 'reset', '--hard'])
+
+    return cppcheck_path
diff --git a/tools/bisect/bisect_hang.py b/tools/bisect/bisect_hang.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+import time
+import sys
+
+from bisect_common import *
+
+# TODO: detect missing file
+def run(cppcheck_path, options, elapsed_time=None):
+    timeout = None
+    if elapsed_time:
+        timeout = elapsed_time * 2
+    cmd = options.split()
+    cmd.insert(0, cppcheck_path)
+    print('running {}'.format(cppcheck_path))
+    p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    try:
+        p.communicate(timeout=timeout)
+        if p.returncode != 0:
+            print('error')
+            return None
+        print('done')
+    except subprocess.TimeoutExpired:
+        print('timeout')
+        p.kill()
+        p.communicate()
+        return False
+
+    return True
+
+
+# TODO: check arguments
+bisect_path = sys.argv[1]
+options = sys.argv[2]
+if '--error-exitcode=0' not in options:
+    options += ' --error-exitcode=0'
+if len(sys.argv) == 4:
+    elapsed_time = float(sys.argv[3])
+else:
+    elapsed_time = None
+
+try:
+    cppcheck_path = build_cppcheck(bisect_path)
+except Exception as e:
+    # TODO: how to persist this so we don't keep compiling these
+    print(e)
+    sys.exit(EC_SKIP)
+
+if not elapsed_time:
+    t = time.perf_counter()
+    # TODO: handle error result
+    run(cppcheck_path, options)
+    elapsed_time = time.perf_counter() - t
+    print('elapsed_time: {}'.format(elapsed_time))
+    # TODO: write to stdout and redirect all all printing to stderr
+    sys.exit(round(elapsed_time + .5))  # return the time
+
+t = time.perf_counter()
+run_res = run(cppcheck_path, options, elapsed_time)
+run_time = time.perf_counter() - t
+
+if not elapsed_time:
+    # TODO: handle error result
+    print('elapsed_time: {}'.format(run_time))
+    # TODO: write to stdout and redirect all printing to stderr
+    sys.exit(round(run_time + .5))  # return the time
+
+if run_res is None:
+    sys.exit(EC_SKIP)  # error occured
+
+if not run_res:
+    sys.exit(EC_BAD)  # timeout occured
+
+print('run_time: {}'.format(run_time))
+
+sys.exit(EC_GOOD)  # no timeout
diff --git a/tools/bisect/bisect_res.py b/tools/bisect/bisect_res.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+import sys
+
+from bisect_common import *
+
+# TODO: detect missing file
+def run(cppcheck_path, options):
+    cmd = options.split()
+    cmd.insert(0, cppcheck_path)
+    print('running {}'.format(cppcheck_path))
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+    stdout, stderr = p.communicate()
+    # only 0 and 1 are well-defined in this case
+    if p.returncode > 1:
+        print('error')
+        return None, None, None
+    # signals are report as negative exitcode (e.g. SIGSEGV -> -11)
+    if p.returncode < 0:
+        print('crash')
+        return None, None, None
+    print('done')
+    return p.returncode, stderr, stdout
+
+
+# TODO: check arguments
+bisect_path = sys.argv[1]
+options = sys.argv[2]
+expected = None
+if len(sys.argv) == 4:
+    expected = sys.argv[3]
+if len(expected) == 0:
+    expected = None
+if expected is None:
+    if '--error-exitcode=1' not in options:
+        options += ' --error-exitcode=1'
+else:
+    if '--error-exitcode=0' not in options:
+        options += ' --error-exitcode=0'
+
+try:
+    cppcheck_path = build_cppcheck(bisect_path)
+except Exception as e:
+    # TODO: how to persist this so we don't keep compiling these
+    print(e)
+    sys.exit(EC_SKIP)
+
+run_ec, run_stderr, run_stdout = run(cppcheck_path, options)
+
+if expected is None:
+    print(run_ec)
+print(run_stdout)
+print(run_stderr)
+
+# if no ec is set we encountered an unexpected error
+if run_ec is None:
+    sys.exit(EC_SKIP)  # error occured
+
+# check output for expected string
+if expected is not None:
+    if (expected not in run_stderr) and (expected not in run_stdout):
+        sys.exit(EC_BAD)  # output not found occured
+
+    sys.exit(EC_GOOD)  # output found
+
+sys.exit(run_ec) # return the elapsed time - not a result for bisect
