enhance: use user lockfiles for tools

We create new resolves for each tool instead of using the implicit
tool resolve which has been deprecated. Now all of the tool requirements
are defined in //BUILD.tools instead of //pants.toml

lockfile regen will go in the next commit.

Author: cognifloyd

BUILD.tools

@@ -0,0 +1,71 @@
+# This BUILD file has requirements for most of the tools resolves
+
+python_requirement(
+    name="bandit-reqs",
+    resolve="bandit",
+    requirements=[
+        "bandit==1.7.0",
+        "setuptools",
+        "GitPython==3.1.18",
+        # bandit needs stevedore which needs importlib-metadata<5
+        # see: https://github.com/PyCQA/bandit/pull/952
+        "importlib-metadata<5;python_version<'3.8'",
+    ],
+)
+
+python_requirement(
+    name="black-reqs",
+    resolve="black",
+    requirements=[
+        "black==22.3.0",
+        "typing-extensions>=3.10.0.0;python_version<'3.10'",
+    ],
+)
+
+python_requirement(
+    name="flake8-reqs",
+    resolve="flake8",
+    requirements=[
+        "flake8==4.0.1",  # st2flake8 does not support flake8 v5
+        # license check plugin
+        "st2flake8==0.1.0",  # TODO: remove in favor of regex-lint or preamble
+    ],
+)
+
+# for pants-plugins, see //pants-plugins/BUILD
+# for pylint, see //pylint_plugins/BUILD
+
+python_requirement(
+    name="pytest-reqs",
+    resolve="pytest",
+    requirements=[
+        "pytest==7.0.1",  # copied from https://www.pantsbuild.org/v2.14/docs/reference-pytest#version
+        "pytest-benchmark[histogram]==3.4.1",
+        # "pytest-timer[colorama]",
+        "pytest-icdiff",
+        "pygments",
+        # "pytest-timeout",
+        # "pytest-mock",
+        "pytest-cov>=2.12,!=2.12.1,<3.1",
+        "pytest-xdist>=2.5,<3",
+    ],
+)
+
+python_requirement(
+    name="setuptools-reqs",
+    resolve="setuptools",
+    requirements=[
+        # setuptools 59.7 (at least) does not support python 3.6
+        "setuptools>=50.3.0,<59.0",
+        "wheel>=0.35.1,<0.38",
+    ],
+)
+
+python_requirement(
+    name="twine-reqs",
+    resolve="twine",
+    requirements=[
+        "twine>=3.7.1,<3.8",
+        "colorama>=0.4.3",
+    ],
+)

pants.toml

@@ -110,20 +110,37 @@ interpreter_constraints = [
   # python_distributions needs a single constraint (vs one line per python version).
   # officially, we exclude 3.7 support, but that adds unnecessary complexity: "CPython>=3.6,!=3.7.*,<3.9",
   "CPython>=3.6,<3.9",
+  # NB: constraints for tools defined below
 ]
 
 [python.resolves]
+# st2 is the primary resolve
 st2 = "lockfiles/st2.lock"
-pylint_plugins = "lockfiles/pylint_plugins.lock" # lockfiles/pylint.lock should have same contents
-pants-plugins = "lockfiles/pants-plugins.lock"
+# tool and misc other resolves (for most, see //BUILD.tools)
+bandit = "lockfiles/bandit.lock"
+black = "lockfiles/black.lock"
+flake8 = "lockfiles/flake8.lock"
+pants-plugins = "lockfiles/pants-plugins.lock" # see //pants-plugins/BUILD
+pylint = "lockfiles/pylint.lock" # see //pylint_plugins/BUILD
+pytest = "lockfiles/pytest.lock"
+setuptools = "lockfiles/setuptools.lock"
+twine = "lockfiles/twine.lock"
 
 [python.resolves_to_interpreter_constraints]
+# for tools, we have to include constraints for st2 and pants-plugins
+bandit = ["CPython>=3.6,<3.10"]
+black = ["CPython>=3.6.2,<3.10"] # black doesn't support <3.6.2
+flake8 = ["CPython>=3.6,<3.10"]
 pants-plugins = [
   # this should match the pants interpreter_constraints:
   # https://github.com/pantsbuild/pants/blob/2.14.x/pants.toml#L125
   # See: https://www.pantsbuild.org/docs/prerequisites
   "CPython>=3.7,<3.10",
 ]
+pylint = ["CPython>=3.6,<3.10"]
+pytest = ["CPython>=3.6,<3.10"]
+setuptools = ["CPython>=3.6,<3.10"]
+twine = ["CPython>=3.6,<3.10"]
 
 [python.resolves_to_constraints_file]
 # Our direct requirements are in requirements-pants.txt;
@@ -158,34 +175,25 @@ ambiguity_resolution = "by_source_root"
 generate_setup_default = true # true by default
 
 [bandit]
-lockfile = "lockfiles/bandit.lock"
-version = "bandit==1.7.0"
 args = [
   "-lll",  # only HIGH severity level
   "--exclude",
   "build,dist",
   "--quiet", # only show output in the case of an error
 ]
-extra_requirements = [
-  "setuptools",
-  # bandit needs stevedore which needs importlib-metadata<5
-  # see: https://github.com/PyCQA/bandit/pull/952
-  "importlib-metadata<5;python_version<'3.8'",
-]
+install_from_resolve = "bandit"
+requirements = ["bandit", "setuptools", "GitPython"] # versions in BUILD.tools
 
 [black]
-lockfile = "lockfiles/black.lock"
-version = "black==22.3.0"
-interpreter_constraints = [
-  "CPython>=3.6.2,<3.9",
-]
+install_from_resolve = "black"
+requirements = ["black"] # version in BUILD.tools
 
 [flake8]
-lockfile = "lockfiles/flake8.lock"
-version = "flake8==4.0.1" # st2flake8 does not support flake8 v5
-extra_requirements = [
+install_from_resolve = "flake8"
+requirements = [ # versions in BUILD.tools
+  "flake8",
   # license check plugin
-  "st2flake8==0.1.0",  # TODO: remove in favor of regex-lint
+  "st2flake8",  # TODO: remove in favor of regex-lint or preamble
 ]
 config = "lint-configs/python/.flake8"
 
@@ -207,11 +215,8 @@ known_versions = [
 ]
 
 [pylint]
-lockfile = "lockfiles/pylint.lock"
-version = "pylint~=2.8.2"
-extra_requirements = [
-  "setuptools", # includes pkg_resources
-]
+install_from_resolve = "pylint"
+requirements = ["pylint", "setuptools"] # versions in pylint_plugins/BUILD
 config = "lint-configs/python/.pylintrc"
 source_plugins = [
   # the /pylint_plugins directory
@@ -227,10 +232,10 @@ args = [
 ]
 
 [pytest]
-lockfile = "lockfiles/pytest.lock"
-version = "pytest==7.0.1"  # copied from https://www.pantsbuild.org/v2.14/docs/reference-pytest#version
-extra_requirements.add = [
-  "pytest-benchmark[histogram]==3.4.1",  # used for st2common/benchmarks
+install_from_resolve = "pytest"
+requirements = [ # versions in BUILD.tools
+  "pytest",
+  "pytest-benchmark[histogram]",  # used for st2common/benchmarks
   #"pytest-timer[colorama]",  # report test timing (--with-timer ala nose-timer)
   "pytest-icdiff",  # make diff output easier to read
   "pygments",  # highlight code in tracebacks
@@ -239,9 +244,9 @@ extra_requirements.add = [
   #"pytest-timeout",  # time limit on tests
   #"pytest-mock",  # more convenient mocking
 
-  # included by default with pants
-  #"pytest-cov",  # coverage
-  #"pytest-xdist",  # parallel test runs (pants uses this if [pytest].xdist_enabled)
+  # needed by pants
+  "pytest-cov",  # coverage
+  "pytest-xdist",  # parallel test runs (pants uses this if [pytest].xdist_enabled)
 ]
 args = [
   "--no-header",  # don't print pytest version for every tested file
@@ -252,10 +257,9 @@ execution_slot_var = "ST2TESTS_PARALLEL_SLOT"
 config = "@lint-configs/regex-lint.yaml"
 
 [setuptools]
-# setuptools 59.7 (at least) does not support python 3.6
-version = "setuptools>=50.3.0,<59.0"
-lockfile = "lockfiles/setuptools.lock"
+install_from_resolve = "setuptools"
+requirements = ["setuptools", "wheel"] # versions in BUILD.tools
 
 [twine]
-lockfile = "lockfiles/twine.lock"
-version = "twine>=3.7.1,<3.8"
+install_from_resolve = "twine"
+requirements = ["twine", "colorama"] # versions in BUILD.tools

pylint_plugins/BUILD

@@ -1,6 +1,6 @@
 __defaults__(
     all=dict(
-        resolve="pylint_plugins",
+        resolve="pylint",
     )
 )
 
@@ -17,10 +17,8 @@ python_tests(
 python_requirement(
     name="pylint",
     requirements=[
-        # This must be the same as [pylint].version in pants.toml
         "pylint~=2.8.2",
-        # other requirements in [pylint].extra_requirements in pants.toml
-        "setuptools",
+        "setuptools", # includes pkg_resources
     ],
 )