update pants-plugins/schemas to pants 2.15

The Fmt/Lint plugin API changed in pants 2.15

It now requires a Subsystem with a SkipOption.
I took advantage of this to consolidate the constants and all
the address and pex request definitions in the Subsystem subclass.

It now requires 2 rules instead of 1: to partition, then fmt/lint.
That means the structure of the rule requests had to change
slightly. The tests also had to account for the modified rules.

Author: cognifloyd

pants-plugins/schemas/rules.py

@@ -11,35 +11,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 from dataclasses import dataclass
 
-from pants.backend.python.target_types import EntryPoint
 from pants.backend.python.util_rules import pex, pex_from_targets
 from pants.backend.python.util_rules.pex import (
     VenvPex,
     VenvPexProcess,
 )
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.core.goals.fmt import FmtResult, FmtTargetsRequest
-from pants.engine.addresses import Address
+from pants.core.util_rules.partitions import PartitionerType
 from pants.engine.fs import MergeDigests, Snapshot
 from pants.engine.process import FallibleProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import FieldSet
-from pants.engine.unions import UnionRule
 from pants.util.logging import LogLevel
 from pants.util.strutil import strip_v2_chroot_path
 
+from schemas.subsystem import GenerateSchemas
 from schemas.target_types import SchemasSourcesField
 
 
-# these constants are also used in the tests.
-CMD_SOURCE_ROOT = "st2common"
-CMD_DIR = "st2common/st2common/cmd"
-CMD_MODULE = "st2common.cmd"
-CMD = "generate_schemas"
-
-
 @dataclass(frozen=True)
 class GenerateSchemasFieldSet(FieldSet):
     required_fields = (SchemasSourcesField,)
@@ -49,37 +42,24 @@ class GenerateSchemasFieldSet(FieldSet):
 
 class GenerateSchemasViaFmtTargetsRequest(FmtTargetsRequest):
     field_set_type = GenerateSchemasFieldSet
-    name = CMD
+    tool_subsystem = GenerateSchemas
+    partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
 
 
 @rule(
     desc="Update contrib/schemas/*.json with st2-generate-schemas",
     level=LogLevel.DEBUG,
 )
 async def generate_schemas_via_fmt(
-    request: GenerateSchemasViaFmtTargetsRequest,
+    request: GenerateSchemasViaFmtTargetsRequest.Batch,
+    subsystem: GenerateSchemas,
 ) -> FmtResult:
     # We use a pex to actually generate the schemas with an external script.
     # Generation cannot be inlined here because it needs to import the st2 code.
-    pex = await Get(
-        VenvPex,
-        PexFromTargetsRequest(
-            [
-                Address(
-                    CMD_DIR,
-                    target_name="cmd",
-                    relative_file_path=f"{CMD}.py",
-                )
-            ],
-            output_filename=f"{CMD}.pex",
-            internal_only=True,
-            main=EntryPoint.parse(f"{CMD_MODULE}.{CMD}:main"),
-        ),
-    )
+    pex = await Get(VenvPex, PexFromTargetsRequest, subsystem.pex_request())
 
-    # There will probably only be one target+field_set, but we iterate
-    # to satisfy how fmt expects that there could be more than one.
-    output_directories = [fs.address.spec_path for fs in request.field_sets]
+    # There will probably only be one target+field_set, and therefor only one directory
+    output_directories = {os.path.dirname(f) for f in request.files}
 
     results = await MultiGet(
         Get(
@@ -112,14 +92,14 @@ async def generate_schemas_via_fmt(
         output=output_snapshot,
         stdout=stdout,
         stderr=stderr,
-        formatter_name=request.name,
+        tool_name=request.tool_name,
     )
 
 
 def rules():
     return [
         *collect_rules(),
-        UnionRule(FmtTargetsRequest, GenerateSchemasViaFmtTargetsRequest),
+        *GenerateSchemasViaFmtTargetsRequest.rules(),
         *pex.rules(),
         *pex_from_targets.rules(),
     ]

pants-plugins/schemas/rules_test.py

@@ -25,16 +25,15 @@
 from pants.engine.fs import CreateDigest, Digest, FileContent, Snapshot
 from pants.engine.target import Target
 from pants.core.goals.fmt import FmtResult
+from pants.core.util_rules.partitions import Partitions
 from pants.testutil.rule_runner import QueryRule, RuleRunner
 
 from .rules import (
-    CMD,
-    CMD_DIR,
-    CMD_SOURCE_ROOT,
     GenerateSchemasFieldSet,
     GenerateSchemasViaFmtTargetsRequest,
     rules as schemas_rules,
 )
+from .subsystem import GenerateSchemas
 from .target_types import Schemas
 
 
@@ -44,7 +43,10 @@ def rule_runner() -> RuleRunner:
         rules=[
             *schemas_rules(),
             *target_types_rules.rules(),
-            QueryRule(FmtResult, (GenerateSchemasViaFmtTargetsRequest,)),
+            QueryRule(
+                Partitions, (GenerateSchemasViaFmtTargetsRequest.PartitionRequest,)
+            ),
+            QueryRule(FmtResult, (GenerateSchemasViaFmtTargetsRequest.Batch,)),
             QueryRule(SourceFiles, (SourceFilesRequest,)),
         ],
         target_types=[Schemas, PythonSourcesGeneratorTarget],
@@ -60,23 +62,35 @@ def run_st2_generate_schemas(
     rule_runner.set_options(
         [
             "--backend-packages=schemas",
-            f"--source-root-patterns=/{CMD_SOURCE_ROOT}",
+            f"--source-root-patterns=/{GenerateSchemas.source_root}",
             *(extra_args or ()),
         ],
         env_inherit={"PATH", "PYENV_ROOT", "HOME"},
     )
-    field_sets = [GenerateSchemasFieldSet.create(tgt) for tgt in targets]
+    field_sets = tuple(GenerateSchemasFieldSet.create(tgt) for tgt in targets)
     input_sources = rule_runner.request(
         SourceFiles,
         [
             SourceFilesRequest(field_set.sources for field_set in field_sets),
         ],
     )
+
+    # run generate_schemas_partitioner rule
+    partitions = rule_runner.request(
+        Partitions,
+        [GenerateSchemasViaFmtTargetsRequest.PartitionRequest(field_sets)],
+    )
+    assert len(partitions) == 1
+
+    # run generate_schemas_via_fmt rule
     fmt_result = rule_runner.request(
         FmtResult,
         [
-            GenerateSchemasViaFmtTargetsRequest(
-                field_sets, snapshot=input_sources.snapshot
+            GenerateSchemasViaFmtTargetsRequest.Batch(
+                tool_name="",
+                elements=partitions[0].elements,  # ie: files
+                partition_metadata=partitions[0].metadata,
+                snapshot=input_sources.snapshot,
             ),
         ],
     )
@@ -114,14 +128,14 @@ def write_files(
         f"{schemas_dir}/{schema_file}": before,
         f"{schemas_dir}/BUILD": "schemas(name='t')",
         # add in the target that's hard-coded in the generate_schemas_via_fmt rue
-        f"{CMD_DIR}/{CMD}.py": GENERATE_SCHEMAS_PY.format(
+        f"{GenerateSchemas.directory}/{GenerateSchemas.cmd}.py": GENERATE_SCHEMAS_PY.format(
             schemas_dir=schemas_dir, schema_text=after
         ),
-        f"{CMD_DIR}/BUILD": "python_sources()",
+        f"{GenerateSchemas.directory}/BUILD": "python_sources()",
     }
 
-    module = CMD_DIR
-    while module != CMD_SOURCE_ROOT:
+    module = GenerateSchemas.directory
+    while module != GenerateSchemas.source_root:
         files[f"{module}/__init__.py"] = ""
         module = os.path.dirname(module)
 

pants-plugins/schemas/subsystem.py

@@ -0,0 +1,45 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pants.backend.python.target_types import EntryPoint
+from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
+from pants.engine.addresses import Address
+from pants.option.option_types import SkipOption
+from pants.option.subsystem import Subsystem
+
+
+class GenerateSchemas(Subsystem):
+    name = "StackStorm Content Schemas Generator"
+    options_scope = "st2-generate-schemas"
+    skip = SkipOption("fmt", "lint")
+    help = "The StackStorm content schemas generator."
+
+    source_root = "st2common"
+    directory = "st2common/st2common/cmd"
+    module = "st2common.cmd"
+    cmd = "generate_schemas"
+
+    def address(self) -> Address:
+        return Address(
+            self.directory,
+            target_name="cmd",
+            relative_file_path=f"{self.cmd}.py",
+        )
+
+    def pex_request(self) -> PexFromTargetsRequest:
+        return PexFromTargetsRequest(
+            [self.address()],
+            output_filename=f"{self.cmd}.pex",
+            internal_only=True,
+            main=EntryPoint.parse(f"{self.module}.{self.cmd}:main"),
+        )