Skip to content

Commit 4e36d4f

Browse files
rluvatonrluvaton
committed
refactor(shim): add new shim infra with enableIfVer macro to avoid shim duplication
1 parent 95be2c3 commit 4e36d4f

10 files changed

Lines changed: 533 additions & 134 deletions

File tree

‎enable-if-version-annotation-macros/pom.xml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
3+
<!--
4+
Licensed to the Apache Software Foundation (ASF) under one
5+
or more contributor license agreements. See the NOTICE file
6+
distributed with this work for additional information
7+
regarding copyright ownership. The ASF licenses this file
8+
to you under the Apache License, Version 2.0 (the
9+
"License"); you may not use this file except in compliance
10+
with the License. You may obtain a copy of the License at
11+
12+
http://www.apache.org/licenses/LICENSE-2.0
13+
14+
Unless required by applicable law or agreed to in writing,
15+
software distributed under the License is distributed on an
16+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
KIND, either express or implied. See the License for the
18+
specific language governing permissions and limitations
19+
under the License.
20+
-->
21+
22+
23+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
24+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
25+
<modelVersion>4.0.0</modelVersion>
26+
<parent>
27+
<groupId>org.apache.datafusion</groupId>
28+
<artifactId>comet-parent-spark${spark.version.short}_${scala.binary.version}</artifactId>
29+
<version>0.17.0-SNAPSHOT</version>
30+
<relativePath>../pom.xml</relativePath>
31+
</parent>
32+
33+
<artifactId>comet-enable-if-version-annotation-macros-spark${spark.version.short}_${scala.binary.version}</artifactId>
34+
<name>comet-enable-if-version-annotation-macros</name>
35+
36+
<!--
37+
Defines the @enableIfVer (and @enableIfAllVer / @enableIfAnyVer / @implementIfVer /
38+
@enableOverrideIfVer) macro annotations used for conditional, version-specific compilation
39+
in a single source file (instead of parallel spark-3.x / spark-4.x source folders).
40+
41+
The targeted versions are fed to the macro at expansion time via the compiler flag
42+
-Xmacro-settings:enableIfVer.<dimension>=<version> (configured in the parent pom's
43+
scala-maven-plugin). Expanding the annotations requires macro-paradise on Scala 2.12 and the
44+
-Ymacro-annotations flag on Scala 2.13 — both wired up in the parent pom.
45+
46+
This module is compile-time-only: expanded annotations leave no runtime reference to it, so
47+
consumers depend on it with "provided" scope and it is not published.
48+
-->
49+
50+
<dependencies>
51+
<dependency>
52+
<groupId>org.scala-lang</groupId>
53+
<artifactId>scala-library</artifactId>
54+
</dependency>
55+
<!-- Needed to define the macro implementations (scala.reflect.macros.whitebox). -->
56+
<dependency>
57+
<groupId>org.scala-lang</groupId>
58+
<artifactId>scala-reflect</artifactId>
59+
<scope>provided</scope>
60+
</dependency>
61+
<!-- Backs the macro's semver range matching; runs at the *consumer's* compile time, so it is
62+
"provided" (expanded annotations leave no runtime reference to it). -->
63+
<dependency>
64+
<groupId>org.semver4j</groupId>
65+
<artifactId>semver4j</artifactId>
66+
<scope>provided</scope>
67+
</dependency>
68+
</dependencies>
69+
70+
<build>
71+
<plugins>
72+
<!-- scala-maven-plugin compiles the macro definitions; macro-annotation enablement
73+
(paradise / -Ymacro-annotations) is inherited from the parent pom. -->
74+
<plugin>
75+
<groupId>net.alchim31.maven</groupId>
76+
<artifactId>scala-maven-plugin</artifactId>
77+
</plugin>
78+
</plugins>
79+
</build>
80+
81+
</project>

‎enable-if-version-annotation-macros/src/main/scala/org/apache/comet/enableIfVer.scala‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet
21+
22+
import scala.annotation.{compileTimeOnly, nowarn, StaticAnnotation}
23+
import scala.language.experimental._
24+
import scala.reflect.macros.whitebox
25+
26+
import org.semver4j.{RangesListFactory, Semver}
27+
28+
/**
29+
* Shared machinery behind the version annotations [[enableIfVer]], [[implementIfVer]] and
30+
* [[enableOverrideIfVer]].
31+
*
32+
* Every annotation performs the same compile-time tree rewrite (drop / empty-body / strip
33+
* `override`). the ONLY thing that differs is a single `Boolean` - does the build's targeted
34+
* version satisfy the given range? So the match decision is computed per annotation and handed to
35+
* the shared expansions here.
36+
*
37+
* The build feeds the targeted version of each dimension to the macro via
38+
* `-Xmacro-settings:enableIfVer.<dimension>=<version>`, read at expansion time from `c.settings`
39+
* (see [[versionOf]]). Adding a dimension is therefore just one more build entry - no generated
40+
* sources. Comet currently configures a single dimension, `spark`.
41+
*
42+
* Ranges are matched by <a href="https://github.com/semver4j/semver4j">semver4j</a>: full
43+
* `major.minor.patch` versions with `>` `>=` `<` `<=` `=` `!=`, space = AND, `||` = OR, `A - B`
44+
* hyphen ranges and more.
45+
*/
46+
object EnableIfVerSupport {
47+
48+
/** Does the configured `version` satisfy the semver `range`? */
49+
def satisfies(range: String, version: String): Boolean =
50+
new Semver(version).satisfies(RangesListFactory.create(range.trim))
51+
52+
/** Prefix of the `-Xmacro-settings` keys this macro understands. */
53+
private val SettingPrefix = "enableIfVer."
54+
55+
/** Parse `enableIfVer.<dimension>=<version>` entries out of `-Xmacro-settings`. */
56+
private def configuredVersions(c: whitebox.Context): Map[String, String] =
57+
c.settings.collect {
58+
case s if s.startsWith(SettingPrefix) =>
59+
s.stripPrefix(SettingPrefix).split("=", 2) match {
60+
case Array(dim, ver) => dim -> parseVersionFromSetting(dim, ver)
61+
case _ =>
62+
c.abort(
63+
c.enclosingPosition,
64+
s"@enableIfVer: malformed macro setting '$s' " +
65+
s"(expected $SettingPrefix<dimension>=<version>)")
66+
}
67+
}.toMap
68+
69+
private def parseVersionFromSetting(name: String, version: String): String = {
70+
try {
71+
// Not using Semver.parse as it will return null instead of giving us meaningful
72+
// exceptions on invalid input
73+
new Semver(version)
74+
75+
version
76+
} catch {
77+
case e: Throwable =>
78+
sys.error(
79+
s"malformed version passed in macro setting for '$name', expected a valid " +
80+
s"SemVer got '$version' (error: ${e.toString})")
81+
}
82+
}
83+
84+
/**
85+
* Build-time version configured for `dimension`, read from `-Xmacro-settings`. The single
86+
* extensibility seam: to add a dimension, pass
87+
* `-Xmacro-settings:${SettingPrefix}<dimension>=<version>` from the build.
88+
*
89+
* Aborts compilation when the dimension was not configured at all.
90+
*/
91+
private def versionOf(c: whitebox.Context, dimension: String): String = {
92+
val versions = configuredVersions(c)
93+
versions.getOrElse(
94+
dimension,
95+
// we do not treat a missing dimension as a match since we want to avoid silent failures
96+
c.abort(
97+
c.enclosingPosition,
98+
s"@enableIfVer: no version configured for dimension '$dimension'" +
99+
s" (configured: ${versions.keys.toList.sorted.mkString(", ")}). " +
100+
s"""Pass it via the compiler flag """ +
101+
s""""-Xmacro-settings:$SettingPrefix$dimension=<version>"."""))
102+
}
103+
104+
object Macros {
105+
106+
/**
107+
* Extract the named `dimension = range` argument of a version annotation. A named argument is
108+
* `name = value`, whose immediate children are `[Ident(name), value]` - matched structurally
109+
* so this works on both Scala 2.12 (`AssignOrNamedArg`) and 2.13 (`NamedArg`). A positional
110+
* arg (e.g. a bare literal) has no such children and is rejected.
111+
*/
112+
private def namedRanges(c: whitebox.Context): List[(String, String)] = {
113+
import c.universe._
114+
val args = c.macroApplication match {
115+
case Apply(Select(Apply(_, as), _), _) => as
116+
}
117+
118+
args.map { arg =>
119+
arg.children match {
120+
case List(Ident(name), value) =>
121+
(name.decodedName.toString, c.eval(c.Expr[String](value)))
122+
case _ =>
123+
c.abort(
124+
c.enclosingPosition,
125+
"@enableIfVer (and all the related version annotations) require named " +
126+
s"""arguments, e.g. spark = ">=3.5.0"". got: ${showRaw(arg)}""")
127+
}
128+
}
129+
}
130+
131+
/** Require exactly one named dimension arg and return whether the build matches its range. */
132+
def singleKeep(c: whitebox.Context, specificMacroPrefix: String): Boolean = {
133+
val ranges = namedRanges(c)
134+
if (ranges.size != 1) {
135+
val ifCase = if (specificMacroPrefix.isEmpty) "enableIf" else "If"
136+
c.abort(
137+
c.enclosingPosition,
138+
s"@${specificMacroPrefix}${ifCase}Ver accepts exactly one dimension " +
139+
s"(got ${ranges.size}).")
140+
}
141+
val (dim, range) = ranges.head
142+
satisfies(range, versionOf(c, dim))
143+
}
144+
145+
// ----- generic tree-rewrite expansions (take a precomputed keep) ---------------------------
146+
147+
/** Keep the annotated member as-is when `keep`, otherwise drop it entirely. */
148+
def enable(c: whitebox.Context)(annottees: Seq[c.Expr[Any]])(keep: Boolean): c.Expr[Any] = {
149+
import c.universe._
150+
if (keep) c.Expr[Any](q"..$annottees")
151+
else c.Expr(EmptyTree)
152+
}
153+
154+
/**
155+
* Keep the annotated member as-is when `keep`, otherwise remove the class body and the
156+
* inheritance
157+
*/
158+
def implementIf(c: whitebox.Context)(annottees: Seq[c.Expr[Any]])(
159+
keep: Boolean): c.Expr[Any] = {
160+
import c.universe._
161+
162+
if (keep) return c.Expr[Any](q"..$annottees")
163+
val head = annottees.head.tree match {
164+
case ClassDef(mods, name, tparams, Template(_, self, _)) =>
165+
ClassDef(mods, name, tparams, Template(List(), self, List(EmptyTree)))
166+
case ModuleDef(mods, name, Template(_, self, _)) =>
167+
ModuleDef(mods, name, Template(List(), self, List(EmptyTree)))
168+
}
169+
c.Expr(q"$head; ..${annottees.tail}")
170+
}
171+
172+
/** Keep the def/val as-is when `keep`. otherwise strip its `override` modifier. */
173+
def enableOverride(c: whitebox.Context)(annottees: Seq[c.Expr[Any]])(
174+
keep: Boolean): c.Expr[Any] = {
175+
import c.universe._
176+
import scala.reflect.internal.Flags
177+
if (keep) return c.Expr[Any](q"..$annottees")
178+
179+
val head = annottees.head.tree match {
180+
case DefDef(mods, name, tparams, vparams, tpt, rhs) =>
181+
val newMods = Modifiers(
182+
(mods.flags.asInstanceOf[Long] & ~Flags.OVERRIDE).asInstanceOf[FlagSet],
183+
mods.privateWithin,
184+
mods.annotations)
185+
DefDef(newMods, name, tparams, vparams, tpt, rhs)
186+
187+
case ValDef(mods, name, tpt, rhs) =>
188+
val newMods = Modifiers(
189+
(mods.flags.asInstanceOf[Long] & ~Flags.OVERRIDE).asInstanceOf[FlagSet],
190+
mods.privateWithin,
191+
mods.annotations)
192+
ValDef(newMods, name, tpt, rhs)
193+
}
194+
c.Expr(q"$head; ..${annottees.tail}")
195+
}
196+
}
197+
}
198+
199+
object enableIfVer {
200+
object Macros {
201+
def verEnable(c: whitebox.Context)(annottees: c.Expr[Any]*): c.Expr[Any] =
202+
EnableIfVerSupport.Macros.enable(c)(annottees)(EnableIfVerSupport.Macros.singleKeep(c, ""))
203+
def verImplementIf(c: whitebox.Context)(annottees: c.Expr[Any]*): c.Expr[Any] =
204+
EnableIfVerSupport.Macros.implementIf(c)(annottees)(
205+
EnableIfVerSupport.Macros.singleKeep(c, "implement"))
206+
def verEnableOverride(c: whitebox.Context)(annottees: c.Expr[Any]*): c.Expr[Any] =
207+
EnableIfVerSupport.Macros.enableOverride(c)(annottees)(
208+
EnableIfVerSupport.Macros.singleKeep(c, "enableOverride"))
209+
}
210+
}
211+
212+
/**
213+
* Keep the annotated member only when the build matches a single dimension's range. otherwise
214+
* drop it entirely. Exactly one dimension must be given as a named semver range. Known dimensions
215+
* are whatever the build configures via `-Xmacro-settings` (currently `spark`).
216+
*
217+
* Example:
218+
* {{{
219+
* @enableIfVer(spark = ">=3.5.0") // keep only on Spark 3.5+
220+
* def onlyOn35Plus(): Unit = ...
221+
*
222+
* @enableIfVer(spark = ">=3.4.0 <4.0.0") // keep on the 3.4 / 3.5 line, drop on 4.0
223+
* override protected def withNewChildInternal(c: SparkPlan): SparkPlan = copy(child = c)
224+
* }}}
225+
*/
226+
@nowarn("cat=unused") // params are used by the macro
227+
@compileTimeOnly("enable macro paradise to expand macro annotations")
228+
final class enableIfVer(spark: String = "") extends StaticAnnotation {
229+
def macroTransform(annottees: Any*): Any = macro enableIfVer.Macros.verEnable
230+
}
231+
232+
/**
233+
* Like [[enableIfVer]], but on a non-matching version the class/object is KEPT (so the type still
234+
* exists) with its body emptied. The inheritance is removed. Use this when the type must stay
235+
* referenceable on every version but its body touches symbols that only exist on some versions.
236+
*
237+
* For example, `WindowGroupLimitExec` was added in Spark 3.5, so this does not compile on 3.4 -
238+
* the top-level import and the parameter type are resolved on every version:
239+
* {{{
240+
* import org.apache.spark.sql.execution.window.WindowGroupLimitExec
241+
*
242+
* class RunWithWGL {
243+
* def run(a: WindowGroupLimitExec): Unit = ...
244+
* }
245+
* }}}
246+
*
247+
* Gate the class so its body (and the 3.5-only import, scoped inside it) only exists on 3.5+:
248+
* {{{
249+
* @implementIfVer(spark = ">=3.5")
250+
* class RunWithWGL {
251+
* import org.apache.spark.sql.execution.window.WindowGroupLimitExec
252+
* def run(a: WindowGroupLimitExec): Unit = ...
253+
* }
254+
* }}}
255+
*
256+
* this will also remove inheritance so `isSomething` won't be required to implement when only
257+
* removing the class body
258+
* {{{
259+
* abstract class Base {
260+
* protected val isSomething: Boolean
261+
* def run(): Unit = if (isSomething) println("something")
262+
* }
263+
*
264+
* @implementIfVer(spark = ">=4")
265+
* class OnlySpark4OrAbove extends Base {
266+
* protected val isSomething: Boolean = false
267+
*
268+
* override def run(): Unit = println("only for spark 4+") // dropped on < 4.0
269+
* }
270+
*
271+
* // For spark below 4 the expanded code will be
272+
* class OnlySpark4OrAbove {
273+
* }
274+
* }}}
275+
*/
276+
@nowarn("cat=unused") // params are used by the macro
277+
@compileTimeOnly("enable macro paradise to expand macro annotations")
278+
final class implementIfVer(spark: String = "") extends StaticAnnotation {
279+
def macroTransform(annottees: Any*): Any = macro enableIfVer.Macros.verImplementIf
280+
}
281+
282+
/**
283+
* Like [[enableIfVer]], but on a non-matching version the `override` modifier is stripped from
284+
* the annotated def/val instead of removing it. Use this for a member that overrides a base
285+
* member only on some versions (because the base member only exists there).
286+
*
287+
* Example:
288+
* {{{
289+
* // `withNewChildInternal` only exists in the base on Spark 3.2+. On < 3.2 the `override` is
290+
* // stripped and it becomes a plain (non-overriding) def, so it still compiles.
291+
* @enableOverrideIfVer(spark = ">=3.2")
292+
* override def withNewChildInternal(c: SparkPlan): SparkPlan = copy(child = c)
293+
* }}}
294+
*/
295+
@nowarn("cat=unused") // params are used by the macro
296+
@compileTimeOnly("enable macro paradise to expand macro annotations")
297+
final class enableOverrideIfVer(spark: String = "") extends StaticAnnotation {
298+
def macroTransform(annottees: Any*): Any = macro enableIfVer.Macros.verEnableOverride
299+
}

0 commit comments

Comments
 (0)