diff --git a/.github/workflows/pip-install.yml b/.github/workflows/pip-install.yml new file mode 100644 index 0000000000..91d54504ed --- /dev/null +++ b/.github/workflows/pip-install.yml @@ -0,0 +1,128 @@ +name: pip install + +on: + push: + paths: + - 'setup.py' + - 'pyproject.toml' + - 'MANIFEST.in' + - 'config/**' + - 'codechecker_common/**' + - 'analyzer/**' + - 'web/**' + - 'tools/**' + - '.github/workflows/pip-install.yml' + pull_request: + paths: + - 'setup.py' + - 'pyproject.toml' + - 'MANIFEST.in' + - 'config/**' + - 'codechecker_common/**' + - 'analyzer/**' + - 'web/**' + - 'tools/**' + - '.github/workflows/pip-install.yml' + +permissions: read-all + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pip-install-smoke: + name: "pip install (${{ matrix.os }}, Python ${{ matrix.python }})" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-latest, windows-latest] + python: ['3.9', '3.12', '3.13'] + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + + - name: pip install . + run: pip install . + + - name: Smoke test + run: | + CodeChecker --help + CodeChecker version + report-converter --help + merge-clang-extdef-mappings --help + post-process-stats --help + tu_collector --help + + - name: Verify data files installed + run: | + python -c " + import sysconfig, os + data = sysconfig.get_path('data') + cfg = os.path.join(data, 'share', 'codechecker', 'config') + assert os.path.isdir(cfg), f'Config dir missing: {cfg}' + assert os.path.isfile(os.path.join(cfg, 'package_layout.json')), 'package_layout.json missing' + " + + pip-install-editable: + name: "pip install -e . (ubuntu, Python 3.12)" + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: pip install -e . + run: pip install -e . + + - name: Smoke test + run: | + CodeChecker --help + CodeChecker version + + pip-install-analyze: + name: "pip install + analyze (ubuntu, Python 3.12)" + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -y clang clang-tidy cppcheck + + - name: pip install . + run: pip install . + + - name: Test analyze and parse + run: | + WORK="$RUNNER_TEMP/analyze-test" + mkdir -p "$WORK" + + cat > "$WORK/main.c" <<'EOF' + int main() { int i = 1 / 0; return i; } + EOF + + cat > "$WORK/compile_commands.json" < 0, 'Expected at least one report' + " diff --git a/.gitignore b/.gitignore index 2f4aa4c92e..36b5b924eb 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,18 @@ venv_dev .coverage Makefile.local +# Generated config files (copied/extended from sub-project dirs by setup.py) +config/analyzer_version.json +config/web_version.json +config/git_commit_urls.json +config/session_client.json +config/system_comment_kinds.json +config/server_config.json + +# Setuptools artifacts +*.egg-info +dist + /web/server/vue-cli/dist # tools diff --git a/MANIFEST.in b/MANIFEST.in index 572cb96d36..221e18537c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,9 @@ -recursive-include build_dist/CodeChecker/lib/python3/codechecker_report_converter/report/output/html/static * +recursive-include tools/report-converter/codechecker_report_converter/report/output/html/static * +recursive-include config * +recursive-include analyzer/config * +recursive-include web/config * +recursive-include web/server/config * include LICENSE.TXT +include docs/README.md +include analyzer/requirements.txt +include web/requirements.txt diff --git a/codechecker_common/cli.py b/codechecker_common/cli.py index acffa32604..97285d2ce6 100755 --- a/codechecker_common/cli.py +++ b/codechecker_common/cli.py @@ -121,6 +121,18 @@ def get_data_files_dir_path(): if os.path.exists(data_dir_path): return data_dir_path + # Editable / development install fallback: config/ lives at the + # repository root, which is the parent of this package's directory. + repo_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir) + ) + if os.path.isfile( + os.path.join(repo_root, "pyproject.toml") + ) and os.path.isfile( + os.path.join(repo_root, "config", "package_layout.json") + ): + return repo_root + print("Failed to get CodeChecker data files directory path in: ", data_dir_paths) sys.exit(1) diff --git a/docs/README.md b/docs/README.md index adde249b5f..66ed9b57ea 100644 --- a/docs/README.md +++ b/docs/README.md @@ -292,6 +292,29 @@ set the `BUILD_UI_DIST` environment variable to `NO` before the package build: - Use `make standalone_package` instead of `make package` to avoid having to manually activate the environment before running CodeChecker. +### Alternative: `pip install` + +```sh +# Standard install: +pip install . + +# Editable install (source changes take effect immediately): +pip install -e . + +# Verify: +CodeChecker version +``` + +#### `make package` vs `pip install` + +| Feature | `make package` | `pip install` | +|---------|---------------|---------------| +| Static analysis (`analyze`, `parse`, `check`) | supported | supported | +| Build logging (`CodeChecker log`) | supported (with ldlogger 32+64 bit on Linux and `intercept-build` on OSX) | **not** supported (unless you use `intercept-build` on OSX) | +| Web server and storage | supported | supported, but must build API packages with `make package_api`, then `pip install api/py/codechecker_api/dist/codechecker_api.tar.gz api/py/codechecker_api_shared/dist/codechecker_api_shared.tar.gz` | +| Web frontend (Vue.js UI) | supported | **not** supported | +| Editable / development install | **not** supported | supported (`pip install -e .`) | + ### Minimum Recommended package versions * In production it is recommended to execute CodeChecker with the minimum Python versions: 3.7.14, 3.8.14, 3.9.14, 3.10.6, 3.11.0, otherwise it may be vulnerable to open-redirect attacks. For more info see https://python-security.readthedocs.io/vuln/http-server-redirection.html (CVE-2021-28861). diff --git a/pyproject.toml b/pyproject.toml index b547b1b141..d762d7b8ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + [tool.mypy] verbosity = 1 show_error_codes = true diff --git a/setup.py b/setup.py index a31b766799..c335b859f9 100644 --- a/setup.py +++ b/setup.py @@ -1,166 +1,264 @@ #!/usr/bin/env python3 +import json import os -import platform -import setuptools +import re +import shutil import subprocess -import sys +import time +import warnings -from setuptools.command.build_ext import build_ext -from setuptools.command.install import install +import setuptools +from setuptools.command.build_py import build_py from setuptools.command.sdist import sdist -from setuptools.extension import Extension - -curr_dir = os.path.dirname(os.path.realpath(__file__)) -build_dir = os.path.join(curr_dir, "build_dist") -package_dir = os.path.join("build_dist", "CodeChecker") -lib_dir = os.path.join(package_dir, "lib", "python3") -req_file_paths = [ - os.path.join("analyzer", "requirements.txt"), - os.path.join("web", "requirements.txt")] -data_files_dir_path = os.path.join('share', 'codechecker') - -data_files = [ - (os.path.join(data_files_dir_path, "docs"), [ - os.path.join("docs", "README.md")]), - - *[(os.path.join(data_files_dir_path, os.path.dirname(p)), [p]) - for p in req_file_paths] -] - -packages = [] - -def get_requirements(): - """ Get install requirements. """ - requirements = set() - for req_file_path in req_file_paths: - with open(req_file_path, 'r') as f: - requirements.update([s for s in [ - line.split('#', 1)[0].strip(' \t\n') for line in f] - if s and 'codechecker' not in s]) - - return requirements - - -def init_data_files(): - """ Initalize data files which will be copied into the package. """ - for data_dir_name in ['config', 'www']: - data_dir_path = os.path.join(package_dir, data_dir_name) - for root, _, files in os.walk(data_dir_path): - if not files: - continue - - data_files.append(( - os.path.normpath( - os.path.join(data_files_dir_path, data_dir_name, - os.path.relpath(root, data_dir_path))), - [os.path.join(root, file_path) for file_path in files])) - - -def init_packages(): - """ Find and initialize the list of packages. """ - global packages - packages.extend(setuptools.find_packages(where=lib_dir)) - - -ld_logger_src_dir_path = \ - os.path.join("analyzer", "tools", "build-logger", "src") - -ld_logger_sources = [ - 'ldlogger-hooks.c', - 'ldlogger-logger.c', - 'ldlogger-tool.c', - 'ldlogger-tool-gcc.c', - 'ldlogger-tool-javac.c', - 'ldlogger-util.c' +ROOT = os.path.dirname(os.path.abspath(__file__)) +DATA_PREFIX = os.path.join("share", "codechecker") +CONFIG_DIR = os.path.join(ROOT, "config") + +PACKAGE_DIR = { + "codechecker_common": "codechecker_common", + "codechecker_analyzer": "analyzer/codechecker_analyzer", + "codechecker_merge_clang_extdef_mappings": + "analyzer/tools/merge_clang_extdef_mappings" + "/codechecker_merge_clang_extdef_mappings", + "codechecker_statistics_collector": "analyzer/tools/statistics_collector" + "/codechecker_statistics_collector", + "codechecker_report_converter": + "tools/report-converter/codechecker_report_converter", + "tu_collector": "tools/tu_collector/tu_collector", + "codechecker_web": "web/codechecker_web", + "codechecker_client": "web/client/codechecker_client", + "codechecker_server": "web/server/codechecker_server", +} + +CONFIG_EXTRA_ROOTS = [ + os.path.join("analyzer", "config"), + os.path.join("web", "config"), + os.path.join("web", "server", "config"), ] -ld_logger_includes = [ - 'ldlogger-hooks.h', - 'ldlogger-tool.h', - 'ldlogger-util.h' +VERSION_SOURCES = [ + os.path.join("analyzer", "config", "analyzer_version.json"), + os.path.join("web", "config", "web_version.json"), ] -data_files.append( - ( - os.path.join(data_files_dir_path, 'ld_logger', 'include'), - [os.path.join(ld_logger_src_dir_path, i) for i in ld_logger_includes] - ) -) - -module_logger_name = 'codechecker_analyzer.ld_logger.lib.ldlogger' -module_logger = Extension( - module_logger_name, - define_macros=[('__LOGGER_MAIN__', None), ('_GNU_SOURCE', None)], - extra_link_args=[ - '-O2', '-fomit-frame-pointer', '-fvisibility=hidden', '-pedantic', - '-Wl,--no-as-needed', '-ldl' +PACKAGE_DATA = { + "codechecker_report_converter": [ + "report/output/html/static/**/*", + "report/output/html/static/*", ], - sources=[ - os.path.join(ld_logger_src_dir_path, s) for s in ld_logger_sources]) - - -class BuildExt(build_ext): - def get_ext_filename(self, ext_name): - return os.path.join(platform.uname().machine, f"{ext_name}.so") - - def build_extension(self, ext): - if sys.platform == "linux": - build_ext.build_extension(self, ext) - - -class Sdist(sdist): - def run(self): - res = subprocess.call( - ["make", "clean_package", "package", "package_api"], - env=dict(os.environ, - BUILD_DIR=build_dir), +} + + +def find_all_packages(): + pkgs = [] + for top_pkg, src_dir in PACKAGE_DIR.items(): + pkgs.append(top_pkg) + abs_src = os.path.join(ROOT, src_dir) + for sub in setuptools.find_packages(where=abs_src): + pkgs.append(f"{top_pkg}.{sub}") + return pkgs + + +def build_package_dir(): + mapping = {} + for top_pkg, src_dir in PACKAGE_DIR.items(): + mapping[top_pkg] = src_dir + abs_src = os.path.join(ROOT, src_dir) + for sub in setuptools.find_packages(where=abs_src): + mapping[f"{top_pkg}.{sub}"] = os.path.join( + src_dir, sub.replace(".", os.sep) + ) + return mapping + + +def add_git_info(data): + if not os.path.exists(os.path.join(ROOT, ".git")): + return + try: + data["git_hash"] = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, encoding="utf-8", - errors="ignore") - - if res: - sys.exit(1) - - init_data_files() - init_packages() + errors="ignore", + ).strip() + except (subprocess.CalledProcessError, OSError): + return + + version = data.get("version", {}) + ver_str = "{}.{}.{}".format( + version.get("major", "0"), + version.get("minor", "0"), + version.get("revision", "0"), + ) + rc = version.get("rc", "") + if rc: + ver_str += f"-rc{rc}" + + try: + tag = subprocess.check_output( + ["git", "describe", "--always", "--tags", "--abbrev=0"], + cwd=ROOT, + encoding="utf-8", + errors="ignore", + ).strip() + dirty = subprocess.check_output( + ["git", "describe", "--always", "--tags", "--dirty=-tainted"], + cwd=ROOT, + encoding="utf-8", + errors="ignore", + ).strip() + data["git_describe"] = { + "tag": ver_str, + "dirty": dirty.replace(tag, ver_str), + } + except (subprocess.CalledProcessError, OSError): + pass + + +def merge_config(): + version_basenames = {os.path.basename(v) for v in VERSION_SOURCES} + for config_root in CONFIG_EXTRA_ROOTS: + abs_root = os.path.join(ROOT, config_root) + if not os.path.isdir(abs_root): + continue + for name in os.listdir(abs_root): + src = os.path.join(abs_root, name) + dest = os.path.join(CONFIG_DIR, name) + if name in version_basenames: + continue # handled below with git extension + if os.path.isfile(src) and not os.path.exists(dest): + shutil.copy2(src, dest) + + for vf in VERSION_SOURCES: + src = os.path.join(ROOT, vf) + if not os.path.isfile(src): + continue + with open(src) as f: + data = json.load(f) + data["package_build_date"] = time.strftime("%Y-%m-%dT%H:%M") + add_git_info(data) + dest = os.path.join(CONFIG_DIR, os.path.basename(vf)) + with open(dest, "w") as f: + json.dump(data, f, sort_keys=True, indent=4) + + +def collect_data_files(): + files = [] + + files.append( + ( + os.path.join(DATA_PREFIX, "docs"), + [os.path.join("docs", "README.md")], + ) + ) - return sdist.run(self) + for req in [ + os.path.join("analyzer", "requirements.txt"), + os.path.join("web", "requirements.txt"), + ]: + files.append((os.path.join(DATA_PREFIX, os.path.dirname(req)), [req])) + + for dirpath, _, filenames in os.walk(CONFIG_DIR): + if not filenames: + continue + rel = os.path.relpath(dirpath, CONFIG_DIR) + dest = os.path.join(DATA_PREFIX, "config") + if rel != ".": + dest = os.path.join(dest, rel) + files.append( + ( + os.path.normpath(dest), + [ + os.path.join( + "config", + os.path.relpath(os.path.join(dirpath, f), CONFIG_DIR), + ) + for f in filenames + ], + ) + ) + + return files -class Install(install): +def get_requirements(): + req_files = [ + os.path.join("analyzer", "requirements.txt"), + os.path.join("web", "requirements.txt"), + ] + seen = {} # normalised name -> (specifier, source file) + reqs = [] + for path in req_files: + with open(os.path.join(ROOT, path)) as f: + for line in f: + line = line.split("#", 1)[0].strip() + if not line or "codechecker" in line: + continue + name = re.split(r"[~!=<>]", line, 1)[0].strip().lower() + if name in seen: + prev_spec, prev_file = seen[name] + if prev_spec != line: + warnings.warn( + f"Requirement {name!r} differs: " + f"{prev_spec!r} (from {prev_file}) vs " + f"{line!r} (from {path})" + ) + continue + seen[name] = (line, path) + reqs.append(line) + return reqs + + +def read_long_description(): + with open( + os.path.join(ROOT, "docs", "README.md"), + encoding="utf-8", + errors="ignore", + ) as fh: + return fh.read() + + +class BuildPyWithConfig(build_py): def run(self): - init_data_files() - init_packages() + merge_config() + self.distribution.data_files = collect_data_files() + super().run() - return install.run(self) -with open(os.path.join("docs", "README.md"), "r", - encoding="utf-8", errors="ignore") as fh: - long_description = fh.read() +class SdistWithConfig(sdist): + def run(self): + merge_config() + self.distribution.data_files = collect_data_files() + super().run() setuptools.setup( name="codechecker", version="6.28.0", - author='CodeChecker Team (Ericsson)', - author_email='codechecker-tool@googlegroups.com', - description="CodeChecker is an analyzer tooling, defect database and " - "viewer extension", - long_description=long_description, - long_description_content_type = "text/markdown", + author="CodeChecker Team (Ericsson)", + author_email="codechecker-tool@googlegroups.com", + description=( + "CodeChecker is an analyzer tooling, defect database and " + "viewer extension" + ), + long_description=read_long_description(), + long_description_content_type="text/markdown", url="https://github.com/Ericsson/CodeChecker", - project_urls = { + project_urls={ "Documentation": "http://codechecker.readthedocs.io", "Issue Tracker": "http://github.com/Ericsson/CodeChecker/issues", }, - keywords=['codechecker', 'plist', 'sarif'], - license='Apache-2.0 WITH LLVM-exception', - packages=packages, - package_dir={ - "": lib_dir - }, - data_files=data_files, + keywords=["codechecker", "plist", "sarif"], + license="Apache-2.0 WITH LLVM-exception", + packages=find_all_packages(), + package_dir=build_package_dir(), + package_data=PACKAGE_DATA, + # data_files are collected inside the build hooks (see + # BuildPyWithConfig) because merge_config() must create the + # version JSON files before collect_data_files() walks config/. + data_files=[], include_package_data=True, classifiers=[ "Development Status :: 5 - Production/Stable", @@ -175,24 +273,21 @@ def run(self): "Topic :: Software Development :: Bug Tracking", "Topic :: Software Development :: Quality Assurance", ], - install_requires=list(get_requirements()), - ext_modules=[module_logger], + install_requires=get_requirements(), cmdclass={ - 'sdist': Sdist, - 'install': Install, - 'build_ext': BuildExt, + "build_py": BuildPyWithConfig, + "sdist": SdistWithConfig, }, - python_requires='>=3.9', - scripts=[ - 'scripts/gerrit_changed_files_to_skipfile.py' - ], + python_requires=">=3.9", + scripts=["scripts/gerrit_changed_files_to_skipfile.py"], entry_points={ - 'console_scripts': [ - 'CodeChecker = codechecker_common.cli:main', - 'merge-clang-extdef-mappings = codechecker_merge_clang_extdef_mappings.cli:main', - 'post-process-stats = codechecker_statistics_collector.cli:main', - 'report-converter = codechecker_report_converter.cli:main', - 'tu_collector = tu_collector.tu_collector:main' - ] + "console_scripts": [ + "CodeChecker = codechecker_common.cli:main", + "merge-clang-extdef-mappings = " + "codechecker_merge_clang_extdef_mappings.cli:main", + "post-process-stats = codechecker_statistics_collector.cli:main", + "report-converter = codechecker_report_converter.cli:main", + "tu_collector = tu_collector.tu_collector:main", + ], }, )