diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ApplyForbiddenApisPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ApplyForbiddenApisPlugin.java index f08bf3d63e54..aa440736504b 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ApplyForbiddenApisPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ApplyForbiddenApisPlugin.java @@ -132,6 +132,21 @@ public void apply(Project project) { .toFile()))); }); + // Junit4->Junit5 transition. + var forbiddenApisTestTask = allForbiddenApisTasks.named("forbiddenApisTest"); + switch (project.getPath()) { + case ":lucene:expressions", ":lucene:memory": + forbiddenApisTestTask.configure( + task -> { + task.setSignaturesFiles( + task.getSignaturesFiles() + .plus( + project.files( + forbiddenApisDir.resolve("non-standard/junit4.txt").toFile()))); + }); + break; + } + // Configure non-standard, per-project stuff. var forbiddenApisMainTask = allForbiddenApisTasks.named("forbiddenApisMain"); diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowFailedTestsAtEndPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowFailedTestsAtEndPlugin.java index ebb044d2339c..0889ddf66586 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowFailedTestsAtEndPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowFailedTestsAtEndPlugin.java @@ -104,7 +104,14 @@ public void afterTest(TestDescriptor desc, TestResult result) { if (desc.getName().equals("classMethod")) { qTestName = desc.getClassName(); } else { - qTestName = desc.getClassName() + "." + desc.getName(); + // For junit5/jupiter, gradle support is very limited for now. + // ideally, we should report the repro line using test metadata + // (from within the test). Or use uniqueid on the context that failed + // to reproduce it. + // + // for now, just trim the trailing (). + String name = desc.getName().replaceAll("\\(\\).*$", ""); + qTestName = desc.getClassName() + "." + name; } var randomizationParameters = ""; diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b8edfba15e06..aa9d976d7132 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -34,6 +34,7 @@ jmh = "1.37" jts = "1.20.0" # unit tests junit = "4.13.2" +junit5 = "6.0.3" # @keep Minimum gradle version to run the build minGradle = "9.5.0" # @keep This is the minimum required Java version. @@ -50,6 +51,7 @@ opennlp = "2.5.9" procfork = "1.0.6" # unit tests randomizedtesting = "2.8.4" +randomizedtesting-jupiter = "0.3.0" # spatial-extras/ support s2-geometry = "1.0.0" # spatial-extras/ support @@ -83,26 +85,23 @@ jmh-annprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version. jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } jts = { module = "org.locationtech.jts:jts-core", version.ref = "jts" } junit = { module = "junit:junit", version.ref = "junit" } -junitplatform-launcher = "org.junit.platform:junit-platform-launcher:6.0.3" -junitplatform-vintage = "org.junit.vintage:junit-vintage-engine:6.0.3" +junitplatform-jupiter = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit5" } +junitplatform-launcher = { module = "org.junit.platform:junit-platform-launcher", version.ref = "junit5" } +junitplatform-testkit = { module = "org.junit.platform:junit-platform-testkit", version.ref = "junit5" } +junitplatform-vintage = { module = "org.junit.vintage:junit-vintage-engine", version.ref = "junit5" } morfologik-polish = { module = "org.carrot2:morfologik-polish", version.ref = "morfologik" } morfologik-stemming = { module = "org.carrot2:morfologik-stemming", version.ref = "morfologik" } morfologik-ukrainian = { module = "ua.net.nlp:morfologik-ukrainian-search", version.ref = "morfologik-ukrainian" } nekohtml = { module = "net.sourceforge.nekohtml:nekohtml", version.ref = "nekohtml" } opennlp-tools = { module = "org.apache.opennlp:opennlp-tools", version.ref = "opennlp" } procfork = { module = "com.carrotsearch:procfork", version.ref = "procfork" } +randomizedtesting-jupiter = { module = "com.carrotsearch.randomizedtesting:randomizedtesting-jupiter", version.ref = "randomizedtesting-jupiter" } randomizedtesting-runner = { module = "com.carrotsearch.randomizedtesting:randomizedtesting-runner", version.ref = "randomizedtesting" } s2-geometry = { module = "io.sgr:s2-geometry-library-java", version.ref = "s2-geometry" } spatial4j = { module = "org.locationtech.spatial4j:spatial4j", version.ref = "spatial4j" } xerces = { module = "xerces:xercesImpl", version.ref = "xerces" } zstd = { module = "com.github.luben:zstd-jni", version.ref = "zstd" } -[bundles] -junitplatform = [ - "junitplatform-launcher", - "junitplatform-vintage", -] - [plugins] benmanes-versions = "com.github.ben-manes.versions:0.54.0" carrotsearch-buildopts = "com.carrotsearch.gradle.opts:0.2.2" diff --git a/gradle/validation/forbidden-apis/non-standard/junit4.txt b/gradle/validation/forbidden-apis/non-standard/junit4.txt new file mode 100644 index 000000000000..dd7aa51d0bbc --- /dev/null +++ b/gradle/validation/forbidden-apis/non-standard/junit4.txt @@ -0,0 +1,209 @@ +@defaultMessage Use JUnit5/jupiter assertions or assertj (preferred) +junit.textui.TestRunner +junit.textui.ResultPrinter +junit.framework.Protectable +junit.framework.AssertionFailedError +junit.framework.Test +junit.framework.Assert +junit.framework.TestResult +junit.framework.JUnit4TestAdapterCache +junit.framework.ComparisonFailure +junit.framework.TestListener +junit.framework.JUnit4TestCaseFacade +junit.framework.TestSuite +junit.framework.ComparisonCompactor +junit.framework.TestCase +junit.framework.JUnit4TestAdapter +junit.framework.TestFailure +junit.runner.TestRunListener +junit.runner.BaseTestRunner +junit.runner.Version +junit.extensions.RepeatedTest +junit.extensions.TestDecorator +junit.extensions.TestSetup +junit.extensions.ActiveTestSuite +org.junit.experimental.theories.ParametersSuppliedBy +org.junit.experimental.theories.suppliers.TestedOn +org.junit.experimental.theories.suppliers.TestedOnSupplier +org.junit.experimental.theories.PotentialAssignment +org.junit.experimental.theories.ParameterSignature +org.junit.experimental.theories.Theory +org.junit.experimental.theories.internal.Assignments +org.junit.experimental.theories.internal.SpecificDataPointsSupplier +org.junit.experimental.theories.internal.BooleanSupplier +org.junit.experimental.theories.internal.EnumSupplier +org.junit.experimental.theories.internal.ParameterizedAssertionError +org.junit.experimental.theories.internal.AllMembersSupplier +org.junit.experimental.theories.FromDataPoints +org.junit.experimental.theories.ParameterSupplier +org.junit.experimental.theories.DataPoint +org.junit.experimental.theories.DataPoints +org.junit.experimental.theories.Theories +org.junit.experimental.max.MaxCore +org.junit.experimental.max.MaxHistory +org.junit.experimental.max.CouldNotReadCoreException +org.junit.experimental.results.FailureList +org.junit.experimental.results.ResultMatchers +org.junit.experimental.results.PrintableResult +org.junit.experimental.runners.Enclosed +org.junit.experimental.ParallelComputer +org.junit.experimental.categories.IncludeCategories +org.junit.experimental.categories.CategoryValidator +org.junit.experimental.categories.Categories +org.junit.experimental.categories.CategoryFilterFactory +org.junit.experimental.categories.Category +org.junit.experimental.categories.ExcludeCategories +org.junit.TestCouldNotBeSkippedException +org.junit.validator.AnnotationValidatorFactory +org.junit.validator.AnnotationValidator +org.junit.validator.PublicClassValidator +org.junit.validator.TestClassValidator +org.junit.validator.ValidateWith +org.junit.validator.AnnotationsValidator +org.junit.Test +# we need to support this in superclass, for now. +# org.junit.Assert +org.junit.Ignore +org.junit.ComparisonFailure +org.junit.runner.Description +org.junit.runner.Describable +org.junit.runner.OrderWith +org.junit.runner.manipulation.Sortable +org.junit.runner.manipulation.InvalidOrderingException +org.junit.runner.manipulation.Orderable +org.junit.runner.manipulation.Sorter +org.junit.runner.manipulation.Orderer +org.junit.runner.manipulation.Alphanumeric +org.junit.runner.manipulation.Filter +org.junit.runner.manipulation.Filterable +org.junit.runner.manipulation.NoTestsRemainException +org.junit.runner.manipulation.Ordering +org.junit.runner.JUnitCore +org.junit.runner.notification.Failure +org.junit.runner.notification.SynchronizedRunListener +org.junit.runner.notification.StoppedByUserException +org.junit.runner.notification.RunListener +org.junit.runner.notification.RunNotifier +org.junit.runner.OrderWithValidator +org.junit.runner.Result +org.junit.runner.FilterFactories +org.junit.runner.Runner +org.junit.runner.FilterFactory +org.junit.runner.Request +org.junit.runner.JUnitCommandLineParseResult +org.junit.runner.FilterFactoryParams +org.junit.runner.Computer +org.junit.runner.RunWith +org.junit.After +org.junit.internal.MethodSorter +org.junit.internal.TextListener +org.junit.internal.InexactComparisonCriteria +org.junit.internal.JUnitSystem +org.junit.internal.builders.AnnotatedBuilder +org.junit.internal.builders.SuiteMethodBuilder +org.junit.internal.builders.NullBuilder +org.junit.internal.builders.AllDefaultPossibilitiesBuilder +org.junit.internal.builders.IgnoredBuilder +org.junit.internal.builders.JUnit4Builder +org.junit.internal.builders.JUnit3Builder +org.junit.internal.builders.IgnoredClassRunner +org.junit.internal.ComparisonCriteria +org.junit.internal.RealSystem +org.junit.internal.Classes +org.junit.internal.SerializableValueDescription +org.junit.internal.management.ThreadMXBean +org.junit.internal.management.FakeRuntimeMXBean +org.junit.internal.management.RuntimeMXBean +org.junit.internal.management.ManagementFactory +org.junit.internal.management.ReflectiveThreadMXBean +org.junit.internal.management.FakeThreadMXBean +org.junit.internal.management.ReflectiveRuntimeMXBean +org.junit.internal.Checks +org.junit.internal.requests.MemoizingRequest +org.junit.internal.requests.OrderingRequest +org.junit.internal.requests.FilterRequest +org.junit.internal.requests.SortingRequest +org.junit.internal.requests.ClassRequest +org.junit.internal.AssumptionViolatedException +org.junit.internal.ArrayComparisonFailure +org.junit.internal.SerializableMatcherDescription +org.junit.internal.runners.MethodValidator +org.junit.internal.runners.InitializationError +org.junit.internal.runners.JUnit4ClassRunner +org.junit.internal.runners.ClassRoadie +org.junit.internal.runners.statements.RunAfters +org.junit.internal.runners.statements.FailOnTimeout +org.junit.internal.runners.statements.InvokeMethod +org.junit.internal.runners.statements.ExpectException +org.junit.internal.runners.statements.Fail +org.junit.internal.runners.statements.RunBefores +org.junit.internal.runners.TestClass +org.junit.internal.runners.model.EachTestNotifier +org.junit.internal.runners.model.ReflectiveCallable +org.junit.internal.runners.model.MultipleFailureException +org.junit.internal.runners.rules.ValidationError +org.junit.internal.runners.rules.RuleMemberValidator +org.junit.internal.runners.SuiteMethod +org.junit.internal.runners.JUnit38ClassRunner +org.junit.internal.runners.ErrorReportingRunner +org.junit.internal.runners.FailedBefore +org.junit.internal.runners.MethodRoadie +org.junit.internal.runners.TestMethod +org.junit.internal.matchers.TypeSafeMatcher +org.junit.internal.matchers.StacktracePrintingMatcher +org.junit.internal.matchers.ThrowableCauseMatcher +org.junit.internal.matchers.ThrowableMessageMatcher +org.junit.internal.Throwables +org.junit.internal.ExactComparisonCriteria +org.junit.function.ThrowingRunnable +org.junit.ClassRule +org.junit.AssumptionViolatedException +org.junit.Before +org.junit.AfterClass +org.junit.rules.Timeout +org.junit.rules.TemporaryFolder +org.junit.rules.Verifier +org.junit.rules.TestWatcher +org.junit.rules.RuleChain +org.junit.rules.DisableOnDebug +org.junit.rules.ErrorCollector +org.junit.rules.ExpectedExceptionMatcherBuilder +org.junit.rules.RunRules +org.junit.rules.MethodRule +org.junit.rules.Stopwatch +org.junit.rules.ExpectedException +org.junit.rules.TestRule +org.junit.rules.ExternalResource +org.junit.rules.TestName +org.junit.rules.TestWatchman +org.junit.BeforeClass +org.junit.Rule +org.junit.runners.RuleContainer +org.junit.runners.parameterized.ParametersRunnerFactory +org.junit.runners.parameterized.BlockJUnit4ClassRunnerWithParametersFactory +org.junit.runners.parameterized.TestWithParameters +org.junit.runners.parameterized.BlockJUnit4ClassRunnerWithParameters +org.junit.runners.ParentRunner +org.junit.runners.BlockJUnit4ClassRunner +org.junit.runners.Parameterized +org.junit.runners.Suite +org.junit.runners.AllTests +org.junit.runners.MethodSorters +org.junit.runners.model.MemberValueConsumer +org.junit.runners.model.FrameworkMember +org.junit.runners.model.InitializationError +org.junit.runners.model.RunnerBuilder +org.junit.runners.model.NoGenericTypeParametersValidator +org.junit.runners.model.TestClass +org.junit.runners.model.RunnerScheduler +org.junit.runners.model.TestTimedOutException +org.junit.runners.model.MultipleFailureException +org.junit.runners.model.InvalidTestClassError +org.junit.runners.model.FrameworkMethod +org.junit.runners.model.Annotatable +org.junit.runners.model.Statement +org.junit.runners.model.FrameworkField +org.junit.runners.JUnit4 +org.junit.FixMethodOrder +org.junit.matchers.JUnitMatchers +org.junit.Assume diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index eeae8f559f4b..bbc90b9ed39c 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -93,6 +93,8 @@ API Changes * GITHUB#15605: Remove unused getCollectors() and internal collector tracking from TopFieldCollectorManager. Clarify thread-safety documentation. (Prudhvi Godithi) +* GITHUB#: Add junit5/jupiter support to the test-framework module. (Dawid Weiss) + New Features --------------------- * GITHUB#15505: Upgrade snowball to 2d2e312df56f2ede014a4ffb3e91e6dea43c24be. New stemmer: PolishStemmer (and diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md index e7afaaedabd8..41fd8a4e18f9 100644 --- a/lucene/MIGRATE.md +++ b/lucene/MIGRATE.md @@ -19,6 +19,35 @@ ## Migration from Lucene 10.x to Lucene 11.0 +### JUnit5/jupiter support in the test-framework + +Lucene 11 brings initial support for writing test cases +using JUnit Jupiter. + +The test-framework module exports both junit4 and junit5/jupiter +modules. `LuceneTestCase` remains the parent abstract class for JUnit4 tests. +`LuceneTestCaseJupiter` is the parent class to extend from for JUnit5 +support. + +#### Key changes + +- All tests must be Jupiter tests, typically this means +methods must be annotated with `@Test`. Method prefix +`test*` is not sufficient. Methods that are named `test*` but are not tests +will cause validation errors. +- You can use parameterized tests, dynamic tests, etc. All these are supported. +- You *must not* call the static `random()` method on the parent +class, even though it is there. Add a `Random` parameter to your test methods +or callbacks - it will +be automatically injected by the test framework. See the `memory` +module tests for examples. +- Use `@BeforeEach`, `@AfterEach` and other junit5-specific callback +annotations instead of `setUp` and `tearDown` methods. +- Static utility methods have been pulled up to a parent class +called `LuceneTestCaseParent` but you should reference them either +without an explicit type or via the type of the parent class +for your test framework. The parent class may be removed in the future. + ### Relaxed Index Upgrade Policy (GITHUB#13797) Starting with Lucene 11.0.0, the index upgrade policy has been relaxed to allow safe upgrades across multiple major version numbers without reindexing when no format breaks occur. @@ -308,7 +337,7 @@ Support for the optional complement syntax (`~`) that was deprecated in Lucene 1 has been removed. The `DEPRECATED_COMPLEMENT` flag and `REGEXP_DEPRECATED_COMPLEMENT` enum value are no longer available. -Users should migrate to using _complement bracket expressions_ (`[^...]`) instead. +Users should migrate to using *complement bracket expressions* (`[^...]`) instead. For example, `[^fo]` matches any character that is not an `f` or `o`. ### DocValuesFieldExistsQuery, NormsFieldExistsQuery and KnnVectorFieldExistsQuery removed in favor of FieldExistsQuery (LUCENE-10436) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestClassloadingDeadlock.java b/lucene/core/src/test/org/apache/lucene/index/TestClassloadingDeadlock.java index 0989a4e08078..2b7c80b8b156 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestClassloadingDeadlock.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestClassloadingDeadlock.java @@ -49,7 +49,7 @@ import org.junit.runner.RunWith; /* WARNING: This test does *not* extend LuceneTestCase to prevent static class - * initialization when spawned as subprocess (and please let default codecs alive)! */ + * initialization when spawned as a subprocess (and please let default codecs alive)! */ @RunWith(RandomizedRunner.class) public class TestClassloadingDeadlock extends Assert { diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java index 11b064463578..3af9dc20463f 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java @@ -20,6 +20,7 @@ import static org.apache.lucene.expressions.js.VariableContext.Type.MEMBER; import static org.apache.lucene.expressions.js.VariableContext.Type.STR_INDEX; +import java.util.Random; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; @@ -37,19 +38,22 @@ import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; +import org.apache.lucene.util.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** simple demo of using expressions */ -public class TestDemoExpressions extends LuceneTestCase { +public class TestDemoExpressions extends LuceneTestCaseJupiter { IndexSearcher searcher; DirectoryReader reader; Directory dir; - @Override - public void setUp() throws Exception { - super.setUp(); + @BeforeEach + final void beforeEach(Random random) throws Exception { dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + RandomIndexWriter iw = new RandomIndexWriter(random, dir); Document doc = new Document(); doc.add(newStringField("id", "1", Field.Store.YES)); @@ -80,14 +84,13 @@ public void setUp() throws Exception { iw.close(); } - @Override - public void tearDown() throws Exception { - reader.close(); - dir.close(); - super.tearDown(); + @AfterEach + final void afterEach() throws Exception { + IOUtils.close(reader, dir); } /** an example of how to rank by an expression */ + @Test public void test() throws Exception { // compile an expression: Expression expr = JavascriptCompiler.compile("sqrt(_score) + ln(popularity)"); @@ -104,6 +107,7 @@ public void test() throws Exception { } /** tests the returned sort values are correct */ + @Test public void testSortValues() throws Exception { Expression expr = JavascriptCompiler.compile("sqrt(_score)"); @@ -122,6 +126,7 @@ public void testSortValues() throws Exception { } /** tests same binding used more than once in an expression */ + @Test public void testTwoOfSameBinding() throws Exception { Expression expr = JavascriptCompiler.compile("_score + _score"); @@ -140,6 +145,7 @@ public void testTwoOfSameBinding() throws Exception { } /** Uses variables with $ */ + @Test public void testDollarVariable() throws Exception { Expression expr = JavascriptCompiler.compile("$0+$score"); @@ -159,6 +165,7 @@ public void testDollarVariable() throws Exception { } /** tests expression referring to another expression */ + @Test public void testExpressionRefersToExpression() throws Exception { Expression expr1 = JavascriptCompiler.compile("_score"); Expression expr2 = JavascriptCompiler.compile("2*expr1"); @@ -179,6 +186,7 @@ public void testExpressionRefersToExpression() throws Exception { } /** tests huge amounts of variables in the expression */ + @Test public void testLotsOfBindings() throws Exception { doTestLotsOfBindings(Byte.MAX_VALUE - 1); doTestLotsOfBindings(Byte.MAX_VALUE); @@ -211,6 +219,7 @@ private void doTestLotsOfBindings(int n) throws Exception { } } + @Test public void testDistanceSort() throws Exception { Expression distance = JavascriptCompiler.compile("haversin(40.7143528,-74.0059731,latitude,longitude)"); @@ -230,6 +239,7 @@ public void testDistanceSort() throws Exception { assertEquals(5.2859D, (Double) d.fields[0], 1E-1); } + @Test public void testHaversinMetersDistanceSort() throws Exception { Expression distance = JavascriptCompiler.compile("haversinMeters(40.7143528,-74.0059731,latitude,longitude)"); @@ -249,6 +259,7 @@ public void testHaversinMetersDistanceSort() throws Exception { assertEquals(5285.9D, (Double) d.fields[0], 1E-1); } + @Test public void testStaticExtendedVariableExample() throws Exception { Expression popularity = JavascriptCompiler.compile("doc[\"popularity\"].value"); SimpleBindings bindings = new SimpleBindings(); @@ -266,6 +277,7 @@ public void testStaticExtendedVariableExample() throws Exception { assertEquals(2D, (Double) d.fields[0], 1E-4); } + @Test public void testDynamicExtendedVariableExample() throws Exception { Expression popularity = JavascriptCompiler.compile("doc['popularity'].value + magicarray[0] + fourtytwo"); diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java index 01a012de3469..0b29b236843e 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java @@ -16,6 +16,7 @@ */ package org.apache.lucene.expressions; +import java.util.Random; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; @@ -32,20 +33,23 @@ import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; +import org.apache.lucene.util.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -public class TestExpressionRescorer extends LuceneTestCase { +public class TestExpressionRescorer extends LuceneTestCaseJupiter { IndexSearcher searcher; DirectoryReader reader; Directory dir; - @Override - public void setUp() throws Exception { - super.setUp(); + @BeforeEach + final void beforeEach(Random random) throws Exception { dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter( - random(), dir, newIndexWriterConfig().setSimilarity(new ClassicSimilarity())); + random, dir, newIndexWriterConfig().setSimilarity(new ClassicSimilarity())); Document doc = new Document(); doc.add(newStringField("id", "1", Field.Store.YES)); @@ -72,13 +76,12 @@ public void setUp() throws Exception { iw.close(); } - @Override - public void tearDown() throws Exception { - reader.close(); - dir.close(); - super.tearDown(); + @AfterEach + final void afterEach() throws Exception { + IOUtils.close(reader, dir); } + @Test public void testBasic() throws Exception { // create a sort field and sort by it (reverse order) diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSortField.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSortField.java index 6351dec17855..312fcb08931c 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSortField.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSortField.java @@ -19,10 +19,12 @@ import org.apache.lucene.expressions.js.JavascriptCompiler; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.SortField; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; +import org.junit.jupiter.api.Test; -public class TestExpressionSortField extends LuceneTestCase { +public class TestExpressionSortField extends LuceneTestCaseJupiter { + @Test public void testToString() throws Exception { Expression expr = JavascriptCompiler.compile("sqrt(_score) + ln(popularity)"); @@ -35,6 +37,7 @@ public void testToString() throws Exception { } @SuppressWarnings("SelfAssertion") + @Test public void testEquals() throws Exception { Expression expr = JavascriptCompiler.compile("sqrt(_score) + ln(popularity)"); @@ -68,6 +71,7 @@ public void testEquals() throws Exception { assertEquals(sf1, sf1); } + @Test public void testNeedsScores() throws Exception { SimpleBindings bindings = new SimpleBindings(); // refers to score directly diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSorts.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSorts.java index 33e2a8b9437d..aa2ba3793875 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSorts.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSorts.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.Random; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.Field; @@ -40,24 +41,27 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.search.CheckHits; import org.apache.lucene.tests.util.English; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** * Tests some basic expressions against different queries, and fieldcache/docvalues fields against * an equivalent sort. */ -public class TestExpressionSorts extends LuceneTestCase { +public class TestExpressionSorts extends LuceneTestCaseJupiter { private Directory dir; private IndexReader reader; private IndexSearcher searcher; - @Override - public void setUp() throws Exception { - super.setUp(); + @BeforeEach + final void beforeEach(Random random) throws Exception { dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + RandomIndexWriter iw = new RandomIndexWriter(random, dir); int numDocs = atLeast(500); for (int i = 0; i < numDocs; i++) { Document document = new Document(); @@ -74,13 +78,12 @@ public void setUp() throws Exception { searcher = newSearcher(reader); } - @Override - public void tearDown() throws Exception { - reader.close(); - dir.close(); - super.tearDown(); + @AfterEach + final void afterEach() throws Exception { + IOUtils.close(reader, dir); } + @Test public void testQueries() throws Exception { int n = atLeast(1); for (int i = 0; i < n; i++) { diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValidation.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValidation.java index dbb98dbf749f..255e32a60a3e 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValidation.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValidation.java @@ -18,11 +18,13 @@ import org.apache.lucene.expressions.js.JavascriptCompiler; import org.apache.lucene.search.DoubleValuesSource; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; +import org.junit.jupiter.api.Test; /** Tests validation of bindings */ -public class TestExpressionValidation extends LuceneTestCase { +public class TestExpressionValidation extends LuceneTestCaseJupiter { + @Test public void testValidExternals() throws Exception { SimpleBindings bindings = new SimpleBindings(); bindings.add("valid0", DoubleValuesSource.fromIntField("valid0")); @@ -37,6 +39,7 @@ public void testValidExternals() throws Exception { bindings.validate(); } + @Test public void testInvalidExternal() throws Exception { SimpleBindings bindings = new SimpleBindings(); bindings.add("valid", DoubleValuesSource.fromIntField("valid")); @@ -50,6 +53,7 @@ public void testInvalidExternal() throws Exception { assertTrue(expected.getMessage().contains("Invalid reference")); } + @Test public void testInvalidExternal2() throws Exception { SimpleBindings bindings = new SimpleBindings(); bindings.add("valid", DoubleValuesSource.fromIntField("valid")); @@ -63,6 +67,7 @@ public void testInvalidExternal2() throws Exception { assertTrue(expected.getMessage().contains("Invalid reference")); } + @Test public void testSelfRecursion() throws Exception { SimpleBindings bindings = new SimpleBindings(); bindings.add("cycle0", JavascriptCompiler.compile("cycle0")); @@ -75,6 +80,7 @@ public void testSelfRecursion() throws Exception { assertTrue(expected.getMessage().contains("Cycle detected")); } + @Test public void testCoRecursion() throws Exception { SimpleBindings bindings = new SimpleBindings(); bindings.add("cycle0", JavascriptCompiler.compile("cycle1")); @@ -88,6 +94,7 @@ public void testCoRecursion() throws Exception { assertTrue(expected.getMessage().contains("Cycle detected")); } + @Test public void testCoRecursion2() throws Exception { SimpleBindings bindings = new SimpleBindings(); bindings.add("cycle0", JavascriptCompiler.compile("cycle1")); @@ -102,6 +109,7 @@ public void testCoRecursion2() throws Exception { assertTrue(expected.getMessage().contains("Cycle detected")); } + @Test public void testCoRecursion3() throws Exception { SimpleBindings bindings = new SimpleBindings(); bindings.add("cycle0", JavascriptCompiler.compile("100")); @@ -116,6 +124,7 @@ public void testCoRecursion3() throws Exception { assertTrue(expected.getMessage().contains("Cycle detected")); } + @Test public void testCoRecursion4() throws Exception { SimpleBindings bindings = new SimpleBindings(); bindings.add("cycle0", JavascriptCompiler.compile("100")); diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java index 6f6dcce17cf6..5d61881b9594 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java @@ -17,6 +17,7 @@ package org.apache.lucene.expressions; import java.io.IOException; +import java.util.Random; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; @@ -30,19 +31,22 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; +import org.apache.lucene.util.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -public class TestExpressionValueSource extends LuceneTestCase { +public class TestExpressionValueSource extends LuceneTestCaseJupiter { DirectoryReader reader; Directory dir; - @Override - public void setUp() throws Exception { - super.setUp(); + @BeforeEach + final void beforeEach(Random random) throws Exception { dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random)); iwc.setMergePolicy(newLogMergePolicy()); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter iw = new RandomIndexWriter(random, dir, iwc); Document doc = new Document(); doc.add(newStringField("id", "1", Field.Store.YES)); @@ -68,13 +72,12 @@ public void setUp() throws Exception { iw.close(); } - @Override - public void tearDown() throws Exception { - reader.close(); - dir.close(); - super.tearDown(); + @AfterEach + final void afterEach() throws Exception { + IOUtils.close(reader, dir); } + @Test public void testDoubleValuesSourceTypes() throws Exception { Expression expr = JavascriptCompiler.compile("2*popularity + count"); SimpleBindings bindings = new SimpleBindings(); @@ -95,6 +98,7 @@ public void testDoubleValuesSourceTypes() throws Exception { } @SuppressWarnings({"unlikely-arg-type", "SelfAssertion"}) + @Test public void testDoubleValuesSourceEquals() throws Exception { Expression expr = JavascriptCompiler.compile("sqrt(a) + ln(b)"); @@ -127,6 +131,7 @@ public void testDoubleValuesSourceEquals() throws Exception { assertFalse(vs1.equals(vs4)); } + @Test public void testFibonacciExpr() throws Exception { int n = 40; SimpleBindings bindings = new SimpleBindings(); @@ -150,6 +155,7 @@ public void testFibonacciExpr() throws Exception { assertEquals(fib(n), (int) values.doubleValue()); } + @Test public void testLazyDependencies() throws Exception { SimpleBindings bindings = new SimpleBindings(); bindings.add("f0", DoubleValuesSource.constant(1)); @@ -187,6 +193,7 @@ private int fib(int n) { return curr; } + @Test public void testRewrite() throws Exception { Expression expr = JavascriptCompiler.compile("a"); diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/CompilerTestCase.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/CompilerTestCase.java index 54547275a66b..d7cc843efdac 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/CompilerTestCase.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/CompilerTestCase.java @@ -21,10 +21,10 @@ import java.text.ParseException; import java.util.Map; import org.apache.lucene.expressions.Expression; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; /** Base class for testing JS compiler */ -public abstract class CompilerTestCase extends LuceneTestCase { +public abstract class CompilerTestCase extends LuceneTestCaseJupiter { /** compiles expression for sourceText with default functions list */ protected Expression compile(String sourceText) throws ParseException { diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestAPISanity.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestAPISanity.java index faf73c980fc2..8dedd22dfcf8 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestAPISanity.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestAPISanity.java @@ -19,6 +19,7 @@ import java.io.IOException; import org.apache.lucene.expressions.Expression; import org.apache.lucene.search.DoubleValues; +import org.junit.jupiter.api.Test; /** Some sanity checks with API as those are not detected by {@link JavascriptCompiler} */ public class TestAPISanity extends CompilerTestCase { @@ -37,6 +38,7 @@ public double evaluate(DoubleValues[] functionValues) throws IOException { } } + @Test public void testBytecodeExceptions() throws Exception { Expression expr = compile("1"); assertArrayEquals( diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestCustomFunctions.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestCustomFunctions.java index 84248f477d1f..58fe14013e11 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestCustomFunctions.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestCustomFunctions.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.lucene.expressions.Expression; import org.apache.lucene.tests.util.LuceneTestCase; +import org.junit.jupiter.api.Test; /** Tests customing the function map */ public class TestCustomFunctions extends CompilerTestCase { @@ -34,6 +35,7 @@ public class TestCustomFunctions extends CompilerTestCase { private static final Lookup LOOKUP = MethodHandles.lookup(); /** empty list of methods */ + @Test public void testEmpty() throws Exception { Map functions = Map.of(); ParseException expected = @@ -48,6 +50,7 @@ public void testEmpty() throws Exception { } /** using the default map explicitly */ + @Test public void testDefaultList() throws Exception { Map functions = JavascriptCompiler.DEFAULT_FUNCTIONS; Expression expr = compile("sqrt(20)", functions); @@ -70,6 +73,7 @@ private static MethodHandle localMethod(String name, int arity) } /** tests a method with no arguments */ + @Test public void testNoArgMethod() throws Exception { Map functions = Map.of("foo", localMethod("zeroArgMethod", 0)); Expression expr = compile("foo()", functions); @@ -81,6 +85,7 @@ public static double oneArgMethod(double arg1) { } /** tests a method with one arguments */ + @Test public void testOneArgMethod() throws Exception { Map functions = Map.of("foo", localMethod("oneArgMethod", 1)); Expression expr = compile("foo(3)", functions); @@ -92,6 +97,7 @@ public static double threeArgMethod(double arg1, double arg2, double arg3) { } /** tests a method with three arguments */ + @Test public void testThreeArgMethod() throws Exception { Map functions = Map.of("foo", localMethod("threeArgMethod", 3)); Expression expr = compile("foo(3, 4, 5)", functions); @@ -99,6 +105,7 @@ public void testThreeArgMethod() throws Exception { } /** tests a map with 2 functions */ + @Test public void testTwoMethods() throws Exception { Map functions = Map.of("foo", localMethod("zeroArgMethod", 0), "bar", localMethod("oneArgMethod", 1)); @@ -107,6 +114,7 @@ public void testTwoMethods() throws Exception { } /** tests invalid methods that are not allowed to become variables to be mapped */ + @Test public void testInvalidVariableMethods() { ParseException expected = expectThrows( @@ -147,6 +155,7 @@ public static String bogusReturnType() { } /** wrong return type: must be double */ + @Test public void testWrongReturnType() throws Exception { Map functions = Map.of("foo", localMethod("bogusReturnType", MethodType.methodType(String.class))); @@ -164,6 +173,7 @@ public static double bogusParameterType(String s) { } /** wrong param type: must be doubles */ + @Test public void testWrongParameterType() throws Exception { Map functions = Map.of( @@ -183,6 +193,7 @@ public double nonStaticMethod() { } /** wrong modifiers: must be static */ + @Test public void testWrongNotStatic() throws Exception { Map functions = Map.of( @@ -204,6 +215,7 @@ private static double nonPublicMethod() { } /** non public methods work as the lookup allows access */ + @Test public void testNotPublic() throws Exception { Map functions = Map.of("foo", localMethod("nonPublicMethod", 0)); Expression expr = compile("foo()", functions); @@ -218,6 +230,7 @@ public static double method() { } /** class containing method is not public */ + @Test public void testNestedNotPublic() throws Exception { Map functions = Map.of( @@ -229,6 +242,7 @@ public void testNestedNotPublic() throws Exception { } /** class containing method is not public */ + @Test public void testNonDirectMethodHandle() throws Exception { Map functions = Map.of( @@ -250,6 +264,7 @@ public static double staticThrowingException() { * the method throws an exception. We should check the stack trace that it contains the source * code of the expression as file name. */ + @Test public void testThrowingException() throws Exception { Map functions = Map.of("foo", localMethod("staticThrowingException", 0)); String source = "3 * foo() / 5"; @@ -274,6 +289,7 @@ public void testThrowingException() throws Exception { } /** test that namespaces work with custom expressions as direct method handle. */ + @Test public void testNamespacesWithDirectMH() throws Exception { Map functions = Map.of("foo.bar", localMethod("zeroArgMethod", 0)); String source = "foo.bar()"; @@ -284,6 +300,7 @@ public void testNamespacesWithDirectMH() throws Exception { /** * test that namespaces work with general method handles (ensure field name of handle is correct). */ + @Test public void testNamespacesWithoutDirectMH() throws Exception { Map functions = Map.of( diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestExpressionMath.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestExpressionMath.java index 5428a9e4e98b..1be629ee834f 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestExpressionMath.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestExpressionMath.java @@ -20,10 +20,12 @@ import static org.apache.lucene.expressions.js.ExpressionMath.haversinKilometers; import java.util.Random; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; +import org.junit.jupiter.api.Test; -public class TestExpressionMath extends LuceneTestCase { +public class TestExpressionMath extends LuceneTestCaseJupiter { + @Test public void testHaversin() { assertTrue(Double.isNaN(haversinKilometers(1, 1, 1, Double.NaN))); assertTrue(Double.isNaN(haversinKilometers(1, 1, Double.NaN, 1))); diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java index 402236d2e096..94d180df5f9d 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java @@ -18,9 +18,11 @@ import java.text.ParseException; import org.apache.lucene.expressions.Expression; +import org.junit.jupiter.api.Test; public class TestJavascriptCompiler extends CompilerTestCase { + @Test public void testNullExpression() throws Exception { expectThrows( NullPointerException.class, @@ -29,6 +31,7 @@ public void testNullExpression() throws Exception { }); } + @Test public void testNullFunctions() throws Exception { expectThrows( NullPointerException.class, @@ -37,11 +40,13 @@ public void testNullFunctions() throws Exception { }); } + @Test public void testEnormousExpressionSource() throws Exception { String expr = " ".repeat(20000) + "100"; assertNotNull(compile(expr)); } + @Test public void testValidCompiles() throws Exception { assertNotNull(compile("100")); assertNotNull(compile("valid0+100")); @@ -49,6 +54,7 @@ public void testValidCompiles() throws Exception { assertNotNull(compile("logn(2, 20+10-5.0)")); } + @Test public void testValidVariables() throws Exception { doTestValidVariable("object.valid0"); doTestValidVariable("object0.object1.valid1"); @@ -92,6 +98,7 @@ void doTestValidVariable(String variable, String output) throws Exception { assertEquals(output, e.variables[0]); } + @Test public void testInvalidVariables() throws Exception { doTestInvalidVariable("object.0invalid"); doTestInvalidVariable("0.invalid"); @@ -117,6 +124,7 @@ void doTestInvalidVariable(String variable) { }); } + @Test public void testInvalidLexer() throws Exception { ParseException expected = expectThrows( @@ -127,6 +135,7 @@ public void testInvalidLexer() throws Exception { assertTrue(expected.getMessage().contains("unexpected character '.' on line (2) position (1)")); } + @Test public void testInvalidCompiles() throws Exception { expectThrows( ParseException.class, @@ -159,6 +168,7 @@ public void testInvalidCompiles() throws Exception { }); } + @Test public void testEmpty() { expectThrows( ParseException.class, @@ -179,6 +189,7 @@ public void testEmpty() { }); } + @Test public void testNull() throws Exception { expectThrows( NullPointerException.class, @@ -187,6 +198,7 @@ public void testNull() throws Exception { }); } + @Test public void testWrongArity() throws Exception { ParseException expected = expectThrows( @@ -231,6 +243,7 @@ public void testWrongArity() throws Exception { assertEquals(expected.getErrorOffset(), 4); } + @Test public void testVariableNormalization() throws Exception { // multiple double quotes Expression x = compile("foo[\"a\"][\"b\"]"); diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptFunction.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptFunction.java index 54b7417dafe2..12545e8785f7 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptFunction.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptFunction.java @@ -17,6 +17,7 @@ package org.apache.lucene.expressions.js; import org.apache.lucene.expressions.Expression; +import org.junit.jupiter.api.Test; public class TestJavascriptFunction extends CompilerTestCase { private static final double DELTA = 0.0000001; @@ -27,6 +28,7 @@ private void assertEvaluatesTo(String expression, double expected) throws Except assertEquals(expected, actual, DELTA); } + @Test public void testAbsMethod() throws Exception { assertEvaluatesTo("abs(0)", 0); assertEvaluatesTo("abs(119)", 119); @@ -35,6 +37,7 @@ public void testAbsMethod() throws Exception { assertEvaluatesTo("abs(-1)", 1); } + @Test public void testAcosMethod() throws Exception { assertEvaluatesTo("acos(-1)", Math.PI); assertEvaluatesTo("acos(-0.8660254)", Math.PI * 5 / 6); @@ -47,12 +50,14 @@ public void testAcosMethod() throws Exception { assertEvaluatesTo("acos(1)", 0); } + @Test public void testAcoshMethod() throws Exception { assertEvaluatesTo("acosh(1)", 0); assertEvaluatesTo("acosh(2.5)", 1.5667992369724109); assertEvaluatesTo("acosh(1234567.89)", 14.719378760739708); } + @Test public void testAsinMethod() throws Exception { assertEvaluatesTo("asin(-1)", -Math.PI / 2); assertEvaluatesTo("asin(-0.8660254)", -Math.PI / 3); @@ -65,6 +70,7 @@ public void testAsinMethod() throws Exception { assertEvaluatesTo("asin(1)", Math.PI / 2); } + @Test public void testAsinhMethod() throws Exception { assertEvaluatesTo("asinh(-1234567.89)", -14.719378760740035); assertEvaluatesTo("asinh(-2.5)", -1.6472311463710958); @@ -75,6 +81,7 @@ public void testAsinhMethod() throws Exception { assertEvaluatesTo("asinh(1234567.89)", 14.719378760740035); } + @Test public void testAtanMethod() throws Exception { assertEvaluatesTo("atan(-1.732050808)", -Math.PI / 3); assertEvaluatesTo("atan(-1)", -Math.PI / 4); @@ -85,6 +92,7 @@ public void testAtanMethod() throws Exception { assertEvaluatesTo("atan(1.732050808)", Math.PI / 3); } + @Test public void testAtan2Method() throws Exception { assertEvaluatesTo("atan2(+0,+0)", +0.0); assertEvaluatesTo("atan2(+0,-0)", +Math.PI); @@ -96,6 +104,7 @@ public void testAtan2Method() throws Exception { assertEvaluatesTo("atan2(-2,-2)", -Math.PI * 3 / 4); } + @Test public void testAtanhMethod() throws Exception { assertEvaluatesTo("atanh(-1)", Double.NEGATIVE_INFINITY); assertEvaluatesTo("atanh(-0.5)", -0.5493061443340549); @@ -104,6 +113,7 @@ public void testAtanhMethod() throws Exception { assertEvaluatesTo("atanh(1)", Double.POSITIVE_INFINITY); } + @Test public void testCeilMethod() throws Exception { assertEvaluatesTo("ceil(0)", 0); assertEvaluatesTo("ceil(0.1)", 1); @@ -114,6 +124,7 @@ public void testCeilMethod() throws Exception { assertEvaluatesTo("ceil(-1.1)", -1); } + @Test public void testCosMethod() throws Exception { assertEvaluatesTo("cos(0)", 1); assertEvaluatesTo("cos(" + Math.PI / 2 + ")", 0); @@ -126,6 +137,7 @@ public void testCosMethod() throws Exception { assertEvaluatesTo("cos(" + -Math.PI / 6 + ")", 0.8660254); } + @Test public void testCoshMethod() throws Exception { assertEvaluatesTo("cosh(0)", 1); assertEvaluatesTo("cosh(-1)", 1.5430806348152437); @@ -136,6 +148,7 @@ public void testCoshMethod() throws Exception { assertEvaluatesTo("cosh(12.3456789)", 114982.09728671524); } + @Test public void testExpMethod() throws Exception { assertEvaluatesTo("exp(0)", 1); assertEvaluatesTo("exp(-1)", 0.36787944117); @@ -146,6 +159,7 @@ public void testExpMethod() throws Exception { assertEvaluatesTo("exp(12.3456789)", 229964.194569); } + @Test public void testFloorMethod() throws Exception { assertEvaluatesTo("floor(0)", 0); assertEvaluatesTo("floor(0.1)", 0); @@ -156,10 +170,12 @@ public void testFloorMethod() throws Exception { assertEvaluatesTo("floor(-1.1)", -2); } + @Test public void testHaversinMethod() throws Exception { assertEvaluatesTo("haversin(40.7143528,-74.0059731,40.759011,-73.9844722)", 5.285885589128259); } + @Test public void testLnMethod() throws Exception { assertEvaluatesTo("ln(0)", Double.NEGATIVE_INFINITY); assertEvaluatesTo("ln(" + Math.E + ")", 1); @@ -169,6 +185,7 @@ public void testLnMethod() throws Exception { assertEvaluatesTo("ln(12.3456789)", 2.51330611521); } + @Test public void testLog10Method() throws Exception { assertEvaluatesTo("log10(0)", Double.NEGATIVE_INFINITY); assertEvaluatesTo("log10(1)", 0); @@ -177,6 +194,7 @@ public void testLog10Method() throws Exception { assertEvaluatesTo("log10(12.3456789)", 1.0915149771692705); } + @Test public void testLognMethod() throws Exception { assertEvaluatesTo("logn(2, 0)", Double.NEGATIVE_INFINITY); assertEvaluatesTo("logn(2, 1)", 0); @@ -190,6 +208,7 @@ public void testLognMethod() throws Exception { assertEvaluatesTo("logn(2.5, 12.3456789)", 2.7429133874016745); } + @Test public void testMaxMethod() throws Exception { assertEvaluatesTo("max(0, 0)", 0); assertEvaluatesTo("max(1, 0)", 1); @@ -198,6 +217,7 @@ public void testMaxMethod() throws Exception { assertEvaluatesTo("max(25, 23)", 25); } + @Test public void testMinMethod() throws Exception { assertEvaluatesTo("min(0, 0)", 0); assertEvaluatesTo("min(1, 0)", 0); @@ -206,6 +226,7 @@ public void testMinMethod() throws Exception { assertEvaluatesTo("min(25, 23)", 23); } + @Test public void testPowMethod() throws Exception { assertEvaluatesTo("pow(0, 0)", 1); assertEvaluatesTo("pow(0.1, 2)", 0.01); @@ -216,6 +237,7 @@ public void testPowMethod() throws Exception { assertEvaluatesTo("pow(-1.1, 2)", 1.21); } + @Test public void testSinMethod() throws Exception { assertEvaluatesTo("sin(0)", 0); assertEvaluatesTo("sin(" + Math.PI / 2 + ")", 1); @@ -228,6 +250,7 @@ public void testSinMethod() throws Exception { assertEvaluatesTo("sin(" + -Math.PI / 6 + ")", -0.5); } + @Test public void testSinhMethod() throws Exception { assertEvaluatesTo("sinh(0)", 0); assertEvaluatesTo("sinh(-1)", -1.1752011936438014); @@ -238,6 +261,7 @@ public void testSinhMethod() throws Exception { assertEvaluatesTo("sinh(12.3456789)", 114982.09728236674); } + @Test public void testSqrtMethod() throws Exception { assertEvaluatesTo("sqrt(0)", 0); assertEvaluatesTo("sqrt(-1)", Double.NaN); @@ -245,6 +269,7 @@ public void testSqrtMethod() throws Exception { assertEvaluatesTo("sqrt(49)", 7); } + @Test public void testTanMethod() throws Exception { assertEvaluatesTo("tan(0)", 0); assertEvaluatesTo("tan(-1)", -1.55740772465); @@ -255,6 +280,7 @@ public void testTanMethod() throws Exception { assertEvaluatesTo("tan(1.3)", 3.60210244797); } + @Test public void testTanhMethod() throws Exception { assertEvaluatesTo("tanh(0)", 0); assertEvaluatesTo("tanh(-1)", -0.76159415595); @@ -266,6 +292,7 @@ public void testTanhMethod() throws Exception { } /** checks that dynamic constants work and only produce one entry in constant pool */ + @Test public void testSameFunction() throws Exception { assertEvaluatesTo("sqrt(49) * abs(-2) * sqrt(25)", 7 * 2 * 5); } diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptOperations.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptOperations.java index 44e86bbaa9f7..cfeb474e2bc9 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptOperations.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptOperations.java @@ -17,6 +17,7 @@ package org.apache.lucene.expressions.js; import org.apache.lucene.expressions.Expression; +import org.junit.jupiter.api.Test; public class TestJavascriptOperations extends CompilerTestCase { private void assertEvaluatesTo(String expression, long expected) throws Exception { @@ -25,6 +26,7 @@ private void assertEvaluatesTo(String expression, long expected) throws Exceptio assertEquals(expected, actual); } + @Test public void testNegationOperation() throws Exception { assertEvaluatesTo("-1", -1); assertEvaluatesTo("--1", 1); @@ -33,6 +35,7 @@ public void testNegationOperation() throws Exception { assertEvaluatesTo("--0", 0); } + @Test public void testAddOperation() throws Exception { assertEvaluatesTo("1+1", 2); assertEvaluatesTo("1+0.5+0.5", 2); @@ -45,6 +48,7 @@ public void testAddOperation() throws Exception { assertEvaluatesTo("0+0", 0); } + @Test public void testSubtractOperation() throws Exception { assertEvaluatesTo("1-1", 0); assertEvaluatesTo("5-10", -5); @@ -57,6 +61,7 @@ public void testSubtractOperation() throws Exception { assertEvaluatesTo("0-0", 0); } + @Test public void testMultiplyOperation() throws Exception { assertEvaluatesTo("1*1", 1); assertEvaluatesTo("5*10", 50); @@ -68,6 +73,7 @@ public void testMultiplyOperation() throws Exception { assertEvaluatesTo("0*0", 0); } + @Test public void testDivisionOperation() throws Exception { assertEvaluatesTo("1*1", 1); assertEvaluatesTo("10/5", 2); @@ -78,6 +84,7 @@ public void testDivisionOperation() throws Exception { assertEvaluatesTo("1/0", 9223372036854775807L); } + @Test public void testModuloOperation() throws Exception { assertEvaluatesTo("1%1", 0); assertEvaluatesTo("10%3", 1); @@ -86,6 +93,7 @@ public void testModuloOperation() throws Exception { assertEvaluatesTo("27%(9%5)", 3); } + @Test public void testLessThanOperation() throws Exception { assertEvaluatesTo("1 < 1", 0); assertEvaluatesTo("2 < 1", 0); @@ -97,6 +105,7 @@ public void testLessThanOperation() throws Exception { assertEvaluatesTo("-1 < 0", 1); } + @Test public void testLessThanEqualsOperation() throws Exception { assertEvaluatesTo("1 <= 1", 1); assertEvaluatesTo("2 <= 1", 0); @@ -108,6 +117,7 @@ public void testLessThanEqualsOperation() throws Exception { assertEvaluatesTo("-1 <= 0", 1); } + @Test public void testGreaterThanOperation() throws Exception { assertEvaluatesTo("1 > 1", 0); assertEvaluatesTo("2 > 1", 1); @@ -119,6 +129,7 @@ public void testGreaterThanOperation() throws Exception { assertEvaluatesTo("-1 > 0", 0); } + @Test public void testGreaterThanEqualsOperation() throws Exception { assertEvaluatesTo("1 >= 1", 1); assertEvaluatesTo("2 >= 1", 1); @@ -130,6 +141,7 @@ public void testGreaterThanEqualsOperation() throws Exception { assertEvaluatesTo("-1 >= 0", 0); } + @Test public void testEqualsOperation() throws Exception { assertEvaluatesTo("1 == 1", 1); assertEvaluatesTo("0 == 0", 1); @@ -145,6 +157,7 @@ public void testEqualsOperation() throws Exception { assertEvaluatesTo("-2 == -1", 0); } + @Test public void testNotEqualsOperation() throws Exception { assertEvaluatesTo("1 != 1", 0); assertEvaluatesTo("0 != 0", 0); @@ -160,6 +173,7 @@ public void testNotEqualsOperation() throws Exception { assertEvaluatesTo("-2 != -1", 1); } + @Test public void testBoolNotOperation() throws Exception { assertEvaluatesTo("!1", 0); assertEvaluatesTo("!!1", 1); @@ -170,6 +184,7 @@ public void testBoolNotOperation() throws Exception { assertEvaluatesTo("!-2", 0); } + @Test public void testBoolAndOperation() throws Exception { assertEvaluatesTo("1 && 1", 1); assertEvaluatesTo("1 && 0", 0); @@ -181,6 +196,7 @@ public void testBoolAndOperation() throws Exception { assertEvaluatesTo("-0 && -0", 0); } + @Test public void testBoolOrOperation() throws Exception { assertEvaluatesTo("1 || 1", 1); assertEvaluatesTo("1 || 0", 1); @@ -192,6 +208,7 @@ public void testBoolOrOperation() throws Exception { assertEvaluatesTo("-0 || -0", 0); } + @Test public void testConditionalOperation() throws Exception { assertEvaluatesTo("1 ? 2 : 3", 2); assertEvaluatesTo("-1 ? 2 : 3", 2); @@ -206,6 +223,7 @@ public void testConditionalOperation() throws Exception { assertEvaluatesTo("(0 ? 1 : 0) ? 3 : 4", 4); } + @Test public void testBitShiftLeft() throws Exception { assertEvaluatesTo("1 << 1", 2); assertEvaluatesTo("2 << 1", 4); @@ -220,6 +238,7 @@ public void testBitShiftLeft() throws Exception { assertEvaluatesTo("-15 << 62", 4611686018427387904L); } + @Test public void testBitShiftRight() throws Exception { assertEvaluatesTo("1 >> 1", 0); assertEvaluatesTo("2 >> 1", 1); @@ -234,6 +253,7 @@ public void testBitShiftRight() throws Exception { assertEvaluatesTo("-2147483646 >> 1", -1073741823); } + @Test public void testBitShiftRightUnsigned() throws Exception { assertEvaluatesTo("1 >>> 1", 0); assertEvaluatesTo("2 >>> 1", 1); @@ -248,6 +268,7 @@ public void testBitShiftRightUnsigned() throws Exception { assertEvaluatesTo("2147483648 >>> 1", 1073741824); } + @Test public void testBitwiseAnd() throws Exception { assertEvaluatesTo("4 & 4", 4); assertEvaluatesTo("3 & 2", 2); @@ -259,6 +280,7 @@ public void testBitwiseAnd() throws Exception { assertEvaluatesTo("1 & 0", 0); } + @Test public void testBitwiseOr() throws Exception { assertEvaluatesTo("4 | 4", 4); assertEvaluatesTo("5 | 2", 7); @@ -270,6 +292,7 @@ public void testBitwiseOr() throws Exception { assertEvaluatesTo("1 | 0", 1); } + @Test public void testBitwiseXor() throws Exception { assertEvaluatesTo("4 ^ 4", 0); assertEvaluatesTo("5 ^ 2", 7); @@ -282,6 +305,7 @@ public void testBitwiseXor() throws Exception { assertEvaluatesTo("0 ^ 0", 0); } + @Test public void testBitwiseNot() throws Exception { assertEvaluatesTo("~-5", 4); assertEvaluatesTo("~25", -26); @@ -289,6 +313,7 @@ public void testBitwiseNot() throws Exception { assertEvaluatesTo("~-1", 0); } + @Test public void testDecimalConst() throws Exception { assertEvaluatesTo("0", 0); assertEvaluatesTo("1", 1); @@ -298,6 +323,7 @@ public void testDecimalConst() throws Exception { assertEvaluatesTo("500E-2", 5); } + @Test public void testHexConst() throws Exception { assertEvaluatesTo("0x0", 0); assertEvaluatesTo("0x1", 1); @@ -309,6 +335,7 @@ public void testHexConst() throws Exception { assertEvaluatesTo("0xA << 2", 0xA << 2); } + @Test public void testHexConst2() throws Exception { assertEvaluatesTo("0X0", 0); assertEvaluatesTo("0X1", 1); @@ -316,6 +343,7 @@ public void testHexConst2() throws Exception { assertEvaluatesTo("0X1234ABCDEF", 78193085935L); } + @Test public void testOctalConst() throws Exception { assertEvaluatesTo("00", 0); assertEvaluatesTo("01", 1); diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestVariableContext.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestVariableContext.java index 5b75b78f071a..80f02bdae422 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestVariableContext.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestVariableContext.java @@ -22,9 +22,11 @@ import static org.apache.lucene.expressions.js.VariableContext.Type.STR_INDEX; import org.apache.lucene.tests.util.LuceneTestCase; +import org.junit.jupiter.api.Test; public class TestVariableContext extends LuceneTestCase { + @Test public void testSimpleVar() { VariableContext[] x = VariableContext.parse("foo"); assertEquals(1, x.length); @@ -32,6 +34,7 @@ public void testSimpleVar() { assertEquals(x[0].text, "foo"); } + @Test public void testEmptyString() { VariableContext[] x = VariableContext.parse("foo['']"); assertEquals(2, x.length); @@ -39,6 +42,7 @@ public void testEmptyString() { assertEquals(x[1].text, ""); } + @Test public void testUnescapeString() { VariableContext[] x = VariableContext.parse("foo['\\'\\\\']"); assertEquals(2, x.length); @@ -46,6 +50,7 @@ public void testUnescapeString() { assertEquals(x[1].text, "'\\"); } + @Test public void testMember() { VariableContext[] x = VariableContext.parse("foo.bar"); assertEquals(2, x.length); @@ -53,6 +58,7 @@ public void testMember() { assertEquals(x[1].text, "bar"); } + @Test public void testMemberFollowedByMember() { VariableContext[] x = VariableContext.parse("foo.bar.baz"); assertEquals(3, x.length); @@ -60,6 +66,7 @@ public void testMemberFollowedByMember() { assertEquals(x[2].text, "baz"); } + @Test public void testMemberFollowedByIntArray() { VariableContext[] x = VariableContext.parse("foo.bar[1]"); assertEquals(3, x.length); @@ -67,6 +74,7 @@ public void testMemberFollowedByIntArray() { assertEquals(x[2].integer, 1); } + @Test public void testMethodWithMember() { VariableContext[] x = VariableContext.parse("m.m()"); assertEquals(2, x.length); @@ -74,6 +82,7 @@ public void testMethodWithMember() { assertEquals(x[1].text, "m"); } + @Test public void testMethodWithStrIndex() { VariableContext[] x = VariableContext.parse("member['blah'].getMethod()"); assertEquals(3, x.length); @@ -81,6 +90,7 @@ public void testMethodWithStrIndex() { assertEquals(x[2].text, "getMethod"); } + @Test public void testMethodWithNumericalIndex() { VariableContext[] x = VariableContext.parse("member[0].getMethod()"); assertEquals(3, x.length); diff --git a/lucene/licenses/apiguardian-api-1.1.2.jar.sha1 b/lucene/licenses/apiguardian-api-1.1.2.jar.sha1 new file mode 100644 index 000000000000..4fb58e602aa8 --- /dev/null +++ b/lucene/licenses/apiguardian-api-1.1.2.jar.sha1 @@ -0,0 +1 @@ +a231e0d844d2721b0fa1b238006d15c6ded6842a diff --git a/lucene/licenses/apiguardian-api-LICENSE-ASL.txt b/lucene/licenses/apiguardian-api-LICENSE-ASL.txt new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/lucene/licenses/apiguardian-api-LICENSE-ASL.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lucene/licenses/apiguardian-api-NOTICE.txt b/lucene/licenses/apiguardian-api-NOTICE.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lucene/licenses/jspecify-1.0.0.jar.sha1 b/lucene/licenses/jspecify-1.0.0.jar.sha1 new file mode 100644 index 000000000000..6a8a9a9d37c7 --- /dev/null +++ b/lucene/licenses/jspecify-1.0.0.jar.sha1 @@ -0,0 +1 @@ +7425a601c1c7ec76645a78d22b8c6a627edee507 diff --git a/lucene/licenses/jspecify-LICENSE-ASL.txt b/lucene/licenses/jspecify-LICENSE-ASL.txt new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/lucene/licenses/jspecify-LICENSE-ASL.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lucene/licenses/jspecify-NOTICE.txt b/lucene/licenses/jspecify-NOTICE.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lucene/licenses/junit-jupiter-6.0.3.jar.sha1 b/lucene/licenses/junit-jupiter-6.0.3.jar.sha1 new file mode 100644 index 000000000000..78a08a02fd46 --- /dev/null +++ b/lucene/licenses/junit-jupiter-6.0.3.jar.sha1 @@ -0,0 +1 @@ +da72f4bc0feccbca639b901ace26e3a62512ebec diff --git a/lucene/licenses/junit-jupiter-api-6.0.3.jar.sha1 b/lucene/licenses/junit-jupiter-api-6.0.3.jar.sha1 new file mode 100644 index 000000000000..21ea8d898975 --- /dev/null +++ b/lucene/licenses/junit-jupiter-api-6.0.3.jar.sha1 @@ -0,0 +1 @@ +2e6cfb62db85350179f0408ed5270f7cdf8cefda diff --git a/lucene/licenses/junit-jupiter-engine-6.0.3.jar.sha1 b/lucene/licenses/junit-jupiter-engine-6.0.3.jar.sha1 new file mode 100644 index 000000000000..59ac4203c558 --- /dev/null +++ b/lucene/licenses/junit-jupiter-engine-6.0.3.jar.sha1 @@ -0,0 +1 @@ +e26f7e17d06cfc85ba7643b5ad00c87d5f2084dd diff --git a/lucene/licenses/junit-jupiter-params-6.0.3.jar.sha1 b/lucene/licenses/junit-jupiter-params-6.0.3.jar.sha1 new file mode 100644 index 000000000000..8497dc9f8b73 --- /dev/null +++ b/lucene/licenses/junit-jupiter-params-6.0.3.jar.sha1 @@ -0,0 +1 @@ +6cd3efadd171a3ddd70413868a9c3af988a45907 diff --git a/lucene/licenses/junit-platform-testkit-6.0.3.jar.sha1 b/lucene/licenses/junit-platform-testkit-6.0.3.jar.sha1 new file mode 100644 index 000000000000..4bf5e448d2bf --- /dev/null +++ b/lucene/licenses/junit-platform-testkit-6.0.3.jar.sha1 @@ -0,0 +1 @@ +61051eac88ffe34d24cb32146e2fc9539d177281 diff --git a/lucene/licenses/randomizedtesting-jupiter-0.3.0.jar.sha1 b/lucene/licenses/randomizedtesting-jupiter-0.3.0.jar.sha1 new file mode 100644 index 000000000000..6959fd5037eb --- /dev/null +++ b/lucene/licenses/randomizedtesting-jupiter-0.3.0.jar.sha1 @@ -0,0 +1 @@ +a733add4ffdacbc2cfdfc15d77de69867c647d00 diff --git a/lucene/licenses/randomizedtesting-jupiter-LICENSE-ASL.txt b/lucene/licenses/randomizedtesting-jupiter-LICENSE-ASL.txt new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/lucene/licenses/randomizedtesting-jupiter-LICENSE-ASL.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lucene/licenses/randomizedtesting-jupiter-NOTICE.txt b/lucene/licenses/randomizedtesting-jupiter-NOTICE.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java index 0c6782d671fa..3af421f86d77 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java @@ -18,6 +18,7 @@ import static org.hamcrest.core.StringContains.containsString; +import com.carrotsearch.randomizedtesting.jupiter.generators.RandomBytes; import java.io.IOException; import java.io.Reader; import java.nio.charset.StandardCharsets; @@ -26,6 +27,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.Random; import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.LongStream; @@ -91,26 +93,24 @@ import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.analysis.MockPayloadAnalyzer; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.BytesRef; import org.hamcrest.MatcherAssert; -import org.junit.Before; -import org.junit.Test; - -public class TestMemoryIndex extends LuceneTestCase { +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +public class TestMemoryIndex extends LuceneTestCaseJupiter { private MockAnalyzer analyzer; - @Before - public void setup() { - analyzer = new MockAnalyzer(random()); + @BeforeEach + public void setup(Random random) { + analyzer = new MockAnalyzer(random); analyzer.setEnableChecks(false); // MemoryIndex can close a TokenStream on init error } @Test public void testFreezeAPI() { - MemoryIndex mi = new MemoryIndex(); mi.addField("f1", "some text", analyzer); @@ -143,6 +143,7 @@ public void testFreezeAPI() { mi.setSimilarity(new ClassicSimilarity()); } + @Test public void testSeekByTermOrd() throws IOException { MemoryIndex mi = new MemoryIndex(); mi.addField("field", "some terms be here", analyzer); @@ -154,6 +155,7 @@ public void testSeekByTermOrd() throws IOException { TestUtil.checkReader(reader); } + @Test public void testFieldsOnlyReturnsIndexedFields() throws IOException { Document doc = new Document(); @@ -167,6 +169,7 @@ public void testFieldsOnlyReturnsIndexedFields() throws IOException { assertEquals(reader.termVectors().get(0).size(), 1); } + @Test public void testReaderConsistency() throws IOException { Analyzer analyzer = new MockPayloadAnalyzer(); @@ -266,6 +269,7 @@ public void testBuildFromDocument() { assertEquals(0.0f, mi.search(new PhraseQuery("field1", "text", "some")), 0); } + @Test public void testDocValues() throws Exception { Document doc = new Document(); doc.add(new NumericDocValuesField("numeric", 29L)); @@ -320,6 +324,7 @@ public void testDocValues() throws Exception { assertEquals(DocIdSetIterator.NO_MORE_DOCS, sortedDocValues.nextDoc()); } + @Test public void testDocValues_resetIterator() throws Exception { Document doc = new Document(); @@ -360,6 +365,7 @@ public void testDocValues_resetIterator() throws Exception { } } + @Test public void testInvalidDocValuesUsage() throws Exception { Document doc = new Document(); doc.add(new NumericDocValuesField("field", 29L)); @@ -406,6 +412,7 @@ public void testInvalidDocValuesUsage() throws Exception { } } + @Test public void testDocValuesDoNotAffectBoostPositionsOrOffset() throws Exception { Document doc = new Document(); doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox"))); @@ -443,10 +450,10 @@ public void testDocValuesDoNotAffectBoostPositionsOrOffset() throws Exception { assertEquals("quick brown fox", binaryDocValues.binaryValue().utf8ToString()); } - public void testBigBinaryDocValues() throws Exception { + @Test + public void testBigBinaryDocValues(Random random) throws Exception { Document doc = new Document(); - byte[] bytes = new byte[33 * 1024]; - random().nextBytes(bytes); + byte[] bytes = RandomBytes.randomBytesOfLength(random, 33 * 1024); doc.add(new BinaryDocValuesField("binary", new BytesRef(bytes))); MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer, true, true); LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader(); @@ -455,10 +462,10 @@ public void testBigBinaryDocValues() throws Exception { assertArrayEquals(bytes, binaryDocValues.binaryValue().bytes); } - public void testBigSortedDocValues() throws Exception { + @Test + public void testBigSortedDocValues(Random random) throws Exception { Document doc = new Document(); - byte[] bytes = new byte[33 * 1024]; - random().nextBytes(bytes); + byte[] bytes = RandomBytes.randomBytesOfLength(random, 33 * 1024); doc.add(new SortedDocValuesField("binary", new BytesRef(bytes))); MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer, true, true); LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader(); @@ -467,6 +474,7 @@ public void testBigSortedDocValues() throws Exception { assertArrayEquals(bytes, sortedDocValues.lookupOrd(0).bytes); } + @Test public void testPointValues() throws Exception { List> fieldFunctions = Arrays.asList( @@ -534,6 +542,7 @@ public void testPointValues() throws Exception { } } + @Test public void testMissingPoints() throws IOException { Document doc = new Document(); doc.add(new StoredField("field", 42)); @@ -551,6 +560,7 @@ public void testMissingPoints() throws IOException { .getPointValues("some_missing_field")); } + @Test public void testPointValuesDoNotAffectPositionsOrOffset() throws Exception { MemoryIndex mi = new MemoryIndex(true, true); mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), analyzer); @@ -598,6 +608,7 @@ public void testPointValuesDoNotAffectPositionsOrOffset() throws Exception { BinaryPoint.newExactQuery("text", "jumps".getBytes(StandardCharsets.UTF_8)))); } + @Test public void test2DPoints() throws Exception { Document doc = new Document(); doc.add(new IntPoint("ints", 0, -100)); @@ -631,6 +642,7 @@ public void test2DPoints() throws Exception { "doubles", new double[] {10D, 10D}, new double[] {30D, 30D}))); } + @Test public void testMultiValuedPointsSortedCorrectly() throws Exception { Document doc = new Document(); doc.add(new IntPoint("ints", 3)); @@ -655,6 +667,7 @@ public void testMultiValuedPointsSortedCorrectly() throws Exception { assertEquals(1, s.count(DoublePoint.newSetQuery("doubles", 2))); } + @Test public void testIndexingPointsAndDocValues() throws Exception { FieldType type = new FieldType(); type.setDimensions(1, 4); @@ -675,6 +688,7 @@ public void testIndexingPointsAndDocValues() throws Exception { assertEquals("term", dvs.binaryValue().utf8ToString()); } + @Test public void testToStringDebug() { MemoryIndex mi = new MemoryIndex(true, true); Analyzer analyzer = new MockPayloadAnalyzer(); @@ -701,7 +715,8 @@ public void testToStringDebug() { mi.toStringDebug()); } - public void testStoredFields() throws IOException { + @Test + public void testStoredFields(Random random) throws IOException { List fields = new ArrayList<>(); fields.add(new StoredField("float", 1.5f)); fields.add(new StoredField("multifloat", 2.5f)); @@ -724,7 +739,7 @@ public void testStoredFields() throws IOException { fields.add(new StoredField("multibinary", "bbar".getBytes(StandardCharsets.UTF_8))); fields.add(new StoredField("multibinary", "bbaz".getBytes(StandardCharsets.UTF_8))); - Collections.shuffle(fields, random()); + Collections.shuffle(fields, random); Document doc = new Document(); for (IndexableField f : fields) { doc.add(f); @@ -750,6 +765,7 @@ public void testStoredFields() throws IOException { d, "multibinary", new BytesRef[] {new BytesRef("bbar"), new BytesRef("bbaz")}); } + @Test public void testKnnFloatVectorOnlyOneVectorAllowed() throws IOException { Document doc = new Document(); doc.add(new KnnFloatVectorField("knnFloatA", new float[] {1.0f, 2.0f})); @@ -759,14 +775,15 @@ public void testKnnFloatVectorOnlyOneVectorAllowed() throws IOException { () -> MemoryIndex.fromDocument(doc, new StandardAnalyzer())); } - public void testKnnFloatVectors() throws IOException { + @Test + public void testKnnFloatVectors(Random random) throws IOException { List fields = new ArrayList<>(); fields.add(new KnnFloatVectorField("knnFloatA", new float[] {1.0f, 2.0f})); fields.add(new KnnFloatVectorField("knnFloatB", new float[] {3.0f, 4.0f, 5.0f, 6.0f})); fields.add( new KnnFloatVectorField( "knnFloatC", new float[] {7.0f, 8.0f, 9.0f}, VectorSimilarityFunction.DOT_PRODUCT)); - Collections.shuffle(fields, random()); + Collections.shuffle(fields, random); Document doc = new Document(); for (IndexableField f : fields) { doc.add(f); @@ -777,9 +794,10 @@ public void testKnnFloatVectors() throws IOException { assertFloatVectorValue(mi, "knnFloatB", new float[] {3.0f, 4.0f, 5.0f, 6.0f}); assertFloatVectorValue(mi, "knnFloatC", new float[] {7.0f, 8.0f, 9.0f}); - assertFloatVectorScore(mi, "knnFloatA", new float[] {1.0f, 1.0f}, 0.5f); - assertFloatVectorScore(mi, "knnFloatB", new float[] {3.0f, 3.0f, 3.0f, 3.0f}, 0.06666667f); - assertFloatVectorScore(mi, "knnFloatC", new float[] {7.0f, 7.0f, 7.0f}, 84.5f); + assertFloatVectorScore(random, mi, "knnFloatA", new float[] {1.0f, 1.0f}, 0.5f); + assertFloatVectorScore( + random, mi, "knnFloatB", new float[] {3.0f, 3.0f, 3.0f, 3.0f}, 0.06666667f); + assertFloatVectorScore(random, mi, "knnFloatC", new float[] {7.0f, 7.0f, 7.0f}, 84.5f); assertNull( mi.createSearcher() @@ -802,6 +820,7 @@ public void testKnnFloatVectors() throws IOException { assertEquals(0, docs.totalHits.value()); } + @Test public void testKnnByteVectorOnlyOneVectorAllowed() throws IOException { Document doc = new Document(); doc.add(new KnnByteVectorField("knnByteA", new byte[] {1, 2})); @@ -811,14 +830,15 @@ public void testKnnByteVectorOnlyOneVectorAllowed() throws IOException { () -> MemoryIndex.fromDocument(doc, new StandardAnalyzer())); } - public void testKnnByteVectors() throws IOException { + @Test + public void testKnnByteVectors(Random random) throws IOException { List fields = new ArrayList<>(); fields.add(new KnnByteVectorField("knnByteA", new byte[] {1, 2})); fields.add(new KnnByteVectorField("knnByteB", new byte[] {3, 4, 5, 6})); fields.add( new KnnByteVectorField( "knnByteC", new byte[] {7, 8, 9}, VectorSimilarityFunction.DOT_PRODUCT)); - Collections.shuffle(fields, random()); + Collections.shuffle(fields, random); Document doc = new Document(); for (IndexableField f : fields) { doc.add(f); @@ -829,9 +849,9 @@ public void testKnnByteVectors() throws IOException { assertByteVectorValue(mi, "knnByteB", new byte[] {3, 4, 5, 6}); assertByteVectorValue(mi, "knnByteC", new byte[] {7, 8, 9}); - assertByteVectorScore(mi, "knnByteA", new byte[] {1, 1}, 0.5f); - assertByteVectorScore(mi, "knnByteB", new byte[] {3, 3, 3, 3}, 0.06666667f); - assertByteVectorScore(mi, "knnByteC", new byte[] {7, 7, 7}, 0.501709f); + assertByteVectorScore(random, mi, "knnByteA", new byte[] {1, 1}, 0.5f); + assertByteVectorScore(random, mi, "knnByteB", new byte[] {3, 3, 3, 3}, 0.06666667f); + assertByteVectorScore(random, mi, "knnByteC", new byte[] {7, 7, 7}, 0.501709f); assertNull( mi.createSearcher() @@ -870,7 +890,7 @@ private static void assertFloatVectorValue(MemoryIndex mi, String fieldName, flo } private static void assertFloatVectorScore( - MemoryIndex mi, String fieldName, float[] queryVector, float expectedScore) + Random random, MemoryIndex mi, String fieldName, float[] queryVector, float expectedScore) throws IOException { FloatVectorValues fvv = mi.createSearcher() @@ -880,7 +900,7 @@ private static void assertFloatVectorScore( .reader() .getFloatVectorValues(fieldName); assertNotNull(fvv); - if (random().nextBoolean()) { + if (random.nextBoolean()) { fvv.iterator().nextDoc(); } VectorScorer scorer = fvv.scorer(queryVector); @@ -906,7 +926,7 @@ private static void assertByteVectorValue(MemoryIndex mi, String fieldName, byte } private static void assertByteVectorScore( - MemoryIndex mi, String fieldName, byte[] queryVector, float expectedScore) + Random random, MemoryIndex mi, String fieldName, byte[] queryVector, float expectedScore) throws IOException { ByteVectorValues bvv = mi.createSearcher() @@ -916,7 +936,7 @@ private static void assertByteVectorScore( .reader() .getByteVectorValues(fieldName); assertNotNull(bvv); - if (random().nextBoolean()) { + if (random.nextBoolean()) { bvv.iterator().nextDoc(); } VectorScorer scorer = bvv.scorer(queryVector); @@ -978,6 +998,7 @@ private static boolean arrayBinaryContains(BytesRef[] array, BytesRef value) { return false; } + @Test public void testIntegerNumericDocValue() throws IOException { // MemoryIndex used to fail when doc values are enabled and numericValue() returns an Integer // such as with IntField. @@ -1068,6 +1089,7 @@ public InvertableType invertableType() { } } + @Test public void testKeywordWithoutTokenStream() throws IOException { List legalFieldTypes = new ArrayList<>(); { diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstDirectory.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstDirectory.java index f57a7d978fcd..b58c1ea1c171 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstDirectory.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstDirectory.java @@ -18,12 +18,12 @@ import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Random; import java.util.Set; import java.util.function.Supplier; import org.apache.lucene.analysis.Analyzer; @@ -78,53 +78,64 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase; import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.analysis.MockTokenFilter; import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.tests.util.LineFileDocs; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; /** - * Verifies that Lucene MemoryIndex and RAM-resident Directory have the same behaviour, returning - * the same results for queries on some randomish indexes. + * Verifies that Lucene MemoryIndex and RAM-resident Directory have the same behavior, returning the + * same results for queries on some randomish indexes. */ -public class TestMemoryIndexAgainstDirectory extends BaseTokenStreamTestCase { - private final Set queries = new HashSet<>(); +public class TestMemoryIndexAgainstDirectory extends LuceneTestCaseJupiter { + private static final Set queries = new HashSet<>(); - @Override - public void setUp() throws Exception { - super.setUp(); + @BeforeAll + public static void prepare() throws Exception { queries.addAll(readQueries("testqueries.txt")); queries.addAll(readQueries("testqueries2.txt")); } + @AfterAll + public static void cleanup() throws Exception { + queries.clear(); + } + /** read a set of queries from a resource file */ - private Set readQueries(String resource) throws IOException { + private static Set readQueries(String resource) throws IOException { Set queries = new HashSet<>(); - InputStream stream = getClass().getResourceAsStream(resource); - BufferedReader reader = - new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); - String line; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.length() > 0 && !line.startsWith("#") && !line.startsWith("//")) { - queries.add(line); + try (var reader = + new BufferedReader( + new InputStreamReader( + TestMemoryIndexAgainstDirectory.class.getResourceAsStream(resource), + StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (!line.isEmpty() && !line.startsWith("#") && !line.startsWith("//")) { + queries.add(line); + } } + return queries; } - return queries; } /** runs random tests, up to ITERATIONS times. */ - public void testRandomQueries() throws Exception { - MemoryIndex index = randomMemoryIndex(); + @Test + public void testRandomQueries(Random random) throws Exception { + MemoryIndex index = randomMemoryIndex(random); int iterations = TEST_NIGHTLY ? 100 * RANDOM_MULTIPLIER : 10 * RANDOM_MULTIPLIER; for (int i = 0; i < iterations; i++) { - assertAgainstDirectory(index); + assertAgainstDirectory(random, index); } } @@ -132,27 +143,27 @@ public void testRandomQueries() throws Exception { * Build a randomish document for both Directory and MemoryIndex, and run all the queries against * it. */ - public void assertAgainstDirectory(MemoryIndex memory) throws Exception { + public void assertAgainstDirectory(Random random, MemoryIndex memory) throws Exception { memory.reset(); StringBuilder fooField = new StringBuilder(); StringBuilder termField = new StringBuilder(); // add up to 250 terms to field "foo" - final int numFooTerms = random().nextInt(250 * RANDOM_MULTIPLIER); + final int numFooTerms = random.nextInt(250 * RANDOM_MULTIPLIER); for (int i = 0; i < numFooTerms; i++) { fooField.append(" "); - fooField.append(randomTerm()); + fooField.append(randomTerm(random)); } // add up to 250 terms to field "term" - final int numTermTerms = random().nextInt(250 * RANDOM_MULTIPLIER); + final int numTermTerms = random.nextInt(250 * RANDOM_MULTIPLIER); for (int i = 0; i < numTermTerms; i++) { termField.append(" "); - termField.append(randomTerm()); + termField.append(randomTerm(random)); } Directory dir = new ByteBuffersDirectory(); - Analyzer analyzer = randomAnalyzer(); + Analyzer analyzer = randomAnalyzer(random); IndexWriter writer = new IndexWriter( dir, @@ -172,13 +183,13 @@ public void assertAgainstDirectory(MemoryIndex memory) throws Exception { LeafReader reader = (LeafReader) memory.createSearcher().getIndexReader(); TestUtil.checkReader(reader); DirectoryReader competitor = DirectoryReader.open(dir); - duellReaders(competitor, reader); + duelReaders(competitor, reader); IOUtils.close(reader, competitor); assertAllQueries(memory, dir, analyzer); dir.close(); } - private void duellReaders(CompositeReader other, LeafReader memIndexReader) throws IOException { + private void duelReaders(CompositeReader other, LeafReader memIndexReader) throws IOException { Fields memFields = memIndexReader.termVectors().get(0); for (String field : FieldInfos.getIndexedFields(other)) { Terms memTerms = memFields.terms(field); @@ -260,13 +271,13 @@ public void assertAllQueries(MemoryIndex memory, Directory directory, Analyzer a } /** Return a random analyzer (Simple, Stop, Standard) to analyze the terms. */ - private Analyzer randomAnalyzer() { - switch (random().nextInt(4)) { + private Analyzer randomAnalyzer(Random random) { + switch (random.nextInt(4)) { case 0: - return new MockAnalyzer(random(), MockTokenizer.SIMPLE, true); + return new MockAnalyzer(random, MockTokenizer.SIMPLE, true); case 1: return new MockAnalyzer( - random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET); + random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET); case 2: return new Analyzer() { @Override @@ -276,7 +287,7 @@ protected TokenStreamComponents createComponents(String fieldName) { } }; default: - return new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + return new MockAnalyzer(random, MockTokenizer.WHITESPACE, false); } } @@ -335,25 +346,26 @@ public boolean incrementToken() throws IOException { * half of the time, returns a random term from TEST_TERMS. the other half of the time, returns a * random unicode string. */ - private String randomTerm() { - if (random().nextBoolean()) { + private String randomTerm(Random random) { + if (random.nextBoolean()) { // return a random TEST_TERM - return TEST_TERMS[random().nextInt(TEST_TERMS.length)]; + return TEST_TERMS[random.nextInt(TEST_TERMS.length)]; } else { // return a random unicode term - return TestUtil.randomUnicodeString(random()); + return TestUtil.randomUnicodeString(random); } } - public void testDocsEnumStart() throws Exception { - Analyzer analyzer = new MockAnalyzer(random()); + @Test + public void testDocsEnumStart(Random random) throws Exception { + Analyzer analyzer = new MockAnalyzer(random); MemoryIndex memory = - new MemoryIndex(random().nextBoolean(), false, random().nextInt(50) * 1024 * 1024); + new MemoryIndex(random.nextBoolean(), false, random.nextInt(50) * 1024 * 1024); memory.addField("foo", "bar", analyzer); LeafReader reader = (LeafReader) memory.createSearcher().getIndexReader(); TestUtil.checkReader(reader); PostingsEnum disi = - TestUtil.docs(random(), reader, "foo", new BytesRef("bar"), null, PostingsEnum.NONE); + TestUtil.docs(random, reader, "foo", new BytesRef("bar"), null, PostingsEnum.NONE); int docid = disi.docID(); assertEquals(-1, docid); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -368,15 +380,16 @@ public void testDocsEnumStart() throws Exception { reader.close(); } - private MemoryIndex randomMemoryIndex() { + private MemoryIndex randomMemoryIndex(Random random) { return new MemoryIndex( - random().nextBoolean(), random().nextBoolean(), random().nextInt(50) * 1024 * 1024); + random.nextBoolean(), random.nextBoolean(), random.nextInt(50) * 1024 * 1024); } - public void testDocsAndPositionsEnumStart() throws Exception { - Analyzer analyzer = new MockAnalyzer(random()); - int numIters = atLeast(3); - MemoryIndex memory = new MemoryIndex(true, false, random().nextInt(50) * 1024 * 1024); + @Test + public void testDocsAndPositionsEnumStart(Random random) throws Exception { + Analyzer analyzer = new MockAnalyzer(random); + int numIters = atLeast(random, 3); + MemoryIndex memory = new MemoryIndex(true, false, random.nextInt(50) * 1024 * 1024); for (int i = 0; i < numIters; i++) { // check reuse memory.addField("foo", "bar", analyzer); LeafReader reader = (LeafReader) memory.createSearcher().getIndexReader(); @@ -403,12 +416,13 @@ public void testDocsAndPositionsEnumStart() throws Exception { } // LUCENE-3831 - public void testNullPointerException() throws IOException { + @Test + public void testNullPointerException(Random random) throws IOException { RegexpQuery regex = new RegexpQuery(new Term("field", "worl.")); SpanQuery wrappedquery = new SpanMultiTermQueryWrapper<>(regex); - MemoryIndex mindex = randomMemoryIndex(); - mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", "hello there")); + MemoryIndex mindex = randomMemoryIndex(random); + mindex.addField("field", new MockAnalyzer(random).tokenStream("field", "hello there")); // This throws an NPE assertEquals(0, mindex.search(wrappedquery), 0.00001f); @@ -416,21 +430,23 @@ public void testNullPointerException() throws IOException { } // LUCENE-3831 - public void testPassesIfWrapped() throws IOException { + @Test + public void testPassesIfWrapped(Random random) throws IOException { RegexpQuery regex = new RegexpQuery(new Term("field", "worl.")); SpanQuery wrappedquery = new SpanOrQuery(new SpanMultiTermQueryWrapper<>(regex)); - MemoryIndex mindex = randomMemoryIndex(); - mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", "hello there")); + MemoryIndex mindex = randomMemoryIndex(random); + mindex.addField("field", new MockAnalyzer(random).tokenStream("field", "hello there")); // This passes though assertEquals(0, mindex.search(wrappedquery), 0.00001f); TestUtil.checkReader(mindex.createSearcher().getIndexReader()); } - public void testSameFieldAddedMultipleTimes() throws IOException { - MemoryIndex mindex = randomMemoryIndex(); - MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); + @Test + public void testSameFieldAddedMultipleTimes(Random random) throws IOException { + MemoryIndex mindex = randomMemoryIndex(random); + MockAnalyzer mockAnalyzer = new MockAnalyzer(random); mindex.addField("field", "the quick brown fox", mockAnalyzer); mindex.addField("field", "jumps over the", mockAnalyzer); LeafReader reader = (LeafReader) mindex.createSearcher().getIndexReader(); @@ -439,7 +455,7 @@ public void testSameFieldAddedMultipleTimes() throws IOException { PhraseQuery query = new PhraseQuery("field", "fox", "jumps"); assertTrue(mindex.search(query) > 0.1); mindex.reset(); - mockAnalyzer.setPositionIncrementGap(1 + random().nextInt(10)); + mockAnalyzer.setPositionIncrementGap(1 + random.nextInt(10)); mindex.addField("field", "the quick brown fox", mockAnalyzer); mindex.addField("field", "jumps over the", mockAnalyzer); assertEquals(0, mindex.search(query), 0.00001f); @@ -449,9 +465,10 @@ public void testSameFieldAddedMultipleTimes() throws IOException { TestUtil.checkReader(mindex.createSearcher().getIndexReader()); } - public void testNonExistentField() throws IOException { - MemoryIndex mindex = randomMemoryIndex(); - MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); + @Test + public void testNonExistentField(Random random) throws IOException { + MemoryIndex mindex = randomMemoryIndex(random); + MockAnalyzer mockAnalyzer = new MockAnalyzer(random); mindex.addField("field", "the quick brown fox", mockAnalyzer); LeafReader reader = (LeafReader) mindex.createSearcher().getIndexReader(); TestUtil.checkReader(reader); @@ -462,50 +479,51 @@ public void testNonExistentField() throws IOException { assertNull(reader.terms("not-in-index")); } - public void testDocValuesMemoryIndexVsNormalIndex() throws Exception { + @Test + public void testDocValuesMemoryIndexVsNormalIndex(Random random) throws Exception { Document doc = new Document(); - long randomLong = random().nextLong(); + long randomLong = random.nextLong(); doc.add(new NumericDocValuesField("numeric", randomLong)); - int numValues = atLeast(5); + int numValues = atLeast(random, 5); for (int i = 0; i < numValues; i++) { - randomLong = random().nextLong(); + randomLong = random.nextLong(); doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong)); - if (random().nextBoolean()) { + if (random.nextBoolean()) { // randomly duplicate field/value doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong)); } } - BytesRef randomTerm = new BytesRef(randomTerm()); + BytesRef randomTerm = new BytesRef(randomTerm(random)); doc.add(new BinaryDocValuesField("binary", randomTerm)); - if (random().nextBoolean()) { + if (random.nextBoolean()) { doc.add(new StringField("binary", randomTerm, Field.Store.NO)); } - randomTerm = new BytesRef(randomTerm()); + randomTerm = new BytesRef(randomTerm(random)); doc.add(new SortedDocValuesField("sorted", randomTerm)); - if (random().nextBoolean()) { + if (random.nextBoolean()) { doc.add(new StringField("sorted", randomTerm, Field.Store.NO)); } - numValues = atLeast(5); + numValues = atLeast(random, 5); for (int i = 0; i < numValues; i++) { - randomTerm = new BytesRef(randomTerm()); + randomTerm = new BytesRef(randomTerm(random)); doc.add(new SortedSetDocValuesField("sorted_set", randomTerm)); - if (random().nextBoolean()) { + if (random.nextBoolean()) { // randomly duplicate field/value doc.add(new SortedSetDocValuesField("sorted_set", randomTerm)); } - if (random().nextBoolean()) { + if (random.nextBoolean()) { // randomly just add a normal string field doc.add(new StringField("sorted_set", randomTerm, Field.Store.NO)); } } - MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); + MockAnalyzer mockAnalyzer = new MockAnalyzer(random); MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, mockAnalyzer); IndexReader indexReader = memoryIndex.createSearcher().getIndexReader(); LeafReader leafReader = indexReader.leaves().get(0).reader(); Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer)); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random, mockAnalyzer)); writer.addDocument(doc); writer.close(); IndexReader controlIndexReader = DirectoryReader.open(dir); @@ -562,9 +580,10 @@ public void testDocValuesMemoryIndexVsNormalIndex() throws Exception { dir.close(); } - public void testNormsWithDocValues() throws Exception { + @Test + public void testNormsWithDocValues(Random random) throws Exception { MemoryIndex mi = new MemoryIndex(true, true); - MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); + MockAnalyzer mockAnalyzer = new MockAnalyzer(random); mi.addField(new BinaryDocValuesField("text", new BytesRef("quick brown fox")), mockAnalyzer); mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), mockAnalyzer); @@ -575,7 +594,7 @@ public void testNormsWithDocValues() throws Exception { Field field = new TextField("text", "quick brown fox", Field.Store.NO); doc.add(field); Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer)); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random, mockAnalyzer)); writer.addDocument(doc); writer.close(); @@ -592,13 +611,14 @@ public void testNormsWithDocValues() throws Exception { dir.close(); } - public void testPointValuesMemoryIndexVsNormalIndex() throws Exception { - int size = atLeast(12); + @Test + public void testPointValuesMemoryIndexVsNormalIndex(Random random) throws Exception { + int size = atLeast(random, 12); List randomValues = new ArrayList<>(); Document doc = new Document(); - for (Integer randomInteger : random().ints(size).toArray()) { + for (Integer randomInteger : random.ints(size).toArray()) { doc.add(new IntPoint("int", randomInteger)); randomValues.add(randomInteger); doc.add(new LongPoint("long", randomInteger)); @@ -606,18 +626,18 @@ public void testPointValuesMemoryIndexVsNormalIndex() throws Exception { doc.add(new DoublePoint("double", randomInteger)); } - MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); + MockAnalyzer mockAnalyzer = new MockAnalyzer(random); MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, mockAnalyzer); IndexSearcher memoryIndexSearcher = memoryIndex.createSearcher(); - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer)); + Directory dir = newDirectory(random); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random, mockAnalyzer)); writer.addDocument(doc); writer.close(); IndexReader controlIndexReader = DirectoryReader.open(dir); IndexSearcher controlIndexSearcher = new IndexSearcher(controlIndexReader); - Supplier valueSupplier = () -> randomValues.get(random().nextInt(randomValues.size())); + Supplier valueSupplier = () -> randomValues.get(random.nextInt(randomValues.size())); Query[] queries = new Query[] { IntPoint.newExactQuery("int", valueSupplier.get()), @@ -642,21 +662,22 @@ public void testPointValuesMemoryIndexVsNormalIndex() throws Exception { dir.close(); } - public void testDuellMemIndex() throws IOException { - LineFileDocs lineFileDocs = new LineFileDocs(random()); - int numDocs = atLeast(10); - MemoryIndex memory = randomMemoryIndex(); + @Test + public void testDuelMemIndex(Random random) throws IOException { + LineFileDocs lineFileDocs = new LineFileDocs(random); + int numDocs = atLeast(random, 10); + MemoryIndex memory = randomMemoryIndex(random); for (int i = 0; i < numDocs; i++) { - Directory dir = newDirectory(); - MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); - mockAnalyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH)); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer)); + Directory dir = newDirectory(random); + MockAnalyzer mockAnalyzer = new MockAnalyzer(random); + mockAnalyzer.setMaxTokenLength(TestUtil.nextInt(random, 1, IndexWriter.MAX_TERM_LENGTH)); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random, mockAnalyzer)); Document nextDoc = lineFileDocs.nextDoc(); Document doc = new Document(); for (IndexableField field : nextDoc.getFields()) { if (field.fieldType().indexOptions() != IndexOptions.NONE) { doc.add(field); - if (random().nextInt(3) == 0) { + if (random.nextInt(3) == 0) { doc.add(field); // randomly add the same field twice } } @@ -670,7 +691,7 @@ public void testDuellMemIndex() throws IOException { DirectoryReader competitor = DirectoryReader.open(dir); LeafReader memIndexReader = (LeafReader) memory.createSearcher().getIndexReader(); TestUtil.checkReader(memIndexReader); - duellReaders(competitor, memIndexReader); + duelReaders(competitor, memIndexReader); IOUtils.close(competitor, memIndexReader); memory.reset(); dir.close(); @@ -679,6 +700,7 @@ public void testDuellMemIndex() throws IOException { } // LUCENE-4880 + @Test public void testEmptyString() throws IOException { MemoryIndex memory = new MemoryIndex(); memory.addField("foo", new CannedTokenStream(new Token("", 0, 5))); @@ -688,12 +710,12 @@ public void testEmptyString() throws IOException { TestUtil.checkReader(searcher.getIndexReader()); } - public void testDuelMemoryIndexCoreDirectoryWithArrayField() throws Exception { - + @Test + public void testDuelMemoryIndexCoreDirectoryWithArrayField(Random random) throws Exception { final String field_name = "text"; - MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); - if (random().nextBoolean()) { - mockAnalyzer.setOffsetGap(random().nextInt(100)); + MockAnalyzer mockAnalyzer = new MockAnalyzer(random); + if (random.nextBoolean()) { + mockAnalyzer.setOffsetGap(random.nextInt(100)); } // index into a random directory FieldType type = new FieldType(TextField.TYPE_STORED); @@ -708,7 +730,7 @@ public void testDuelMemoryIndexCoreDirectoryWithArrayField() throws Exception { doc.add(new Field(field_name, "foo bar foo bar foo", type)); Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer)); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random, mockAnalyzer)); writer.updateDocument(new Term("id", "1"), doc); writer.commit(); writer.close(); diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestSlicedIntBlockPool.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestSlicedIntBlockPool.java index d8e7c420fea6..9d78be89bf76 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestSlicedIntBlockPool.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestSlicedIntBlockPool.java @@ -18,12 +18,15 @@ import java.util.ArrayList; import java.util.List; -import org.apache.lucene.tests.util.LuceneTestCase; +import java.util.Random; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; import org.apache.lucene.util.Counter; import org.apache.lucene.util.IntBlockPool; +import org.junit.jupiter.api.Test; -public class TestSlicedIntBlockPool extends LuceneTestCase { - public void testSingleWriterReader() { +public class TestSlicedIntBlockPool extends LuceneTestCaseJupiter { + @Test + public void testSingleWriterReader(Random random) { Counter bytesUsed = Counter.newCounter(); MemoryIndex.SlicedIntBlockPool slicedIntBlockPool = new MemoryIndex.SlicedIntBlockPool(new ByteTrackingAllocator(bytesUsed)); @@ -32,7 +35,7 @@ public void testSingleWriterReader() { MemoryIndex.SlicedIntBlockPool.SliceWriter writer = new MemoryIndex.SlicedIntBlockPool.SliceWriter(slicedIntBlockPool); int start = writer.startNewSlice(); - int num = atLeast(100); + int num = atLeast(random, 100); for (int i = 0; i < num; i++) { writer.writeInt(i); } @@ -45,7 +48,7 @@ public void testSingleWriterReader() { assertEquals(i, reader.readInt()); } assertTrue(reader.endOfSlice()); - if (random().nextBoolean()) { + if (random.nextBoolean()) { slicedIntBlockPool.reset(true, false); assertEquals(0, bytesUsed.get()); } else { @@ -55,24 +58,25 @@ public void testSingleWriterReader() { } } - public void testMultipleWriterReader() { + @Test + public void testMultipleWriterReader(Random random) { Counter bytesUsed = Counter.newCounter(); MemoryIndex.SlicedIntBlockPool slicedIntBlockPool = new MemoryIndex.SlicedIntBlockPool(new ByteTrackingAllocator(bytesUsed)); for (int j = 0; j < 2; j++) { List holders = new ArrayList<>(); - int num = atLeast(4); + int num = atLeast(random, 4); for (int i = 0; i < num; i++) { - holders.add(new StartEndAndValues(random().nextInt(1000))); + holders.add(new StartEndAndValues(random.nextInt(1000))); } MemoryIndex.SlicedIntBlockPool.SliceWriter writer = new MemoryIndex.SlicedIntBlockPool.SliceWriter(slicedIntBlockPool); MemoryIndex.SlicedIntBlockPool.SliceReader reader = new MemoryIndex.SlicedIntBlockPool.SliceReader(slicedIntBlockPool); - int numValuesToWrite = atLeast(10000); + int numValuesToWrite = atLeast(random, 10000); for (int i = 0; i < numValuesToWrite; i++) { - StartEndAndValues values = holders.get(random().nextInt(holders.size())); + StartEndAndValues values = holders.get(random.nextInt(holders.size())); if (values.valueCount == 0) { values.start = writer.startNewSlice(); } else { @@ -80,17 +84,17 @@ public void testMultipleWriterReader() { } writer.writeInt(values.nextValue()); values.end = writer.getCurrentOffset(); - if (random().nextInt(5) == 0) { + if (random.nextInt(5) == 0) { // pick one and reader the ints - assertReader(reader, holders.get(random().nextInt(holders.size()))); + assertReader(reader, holders.get(random.nextInt(holders.size()))); } } while (!holders.isEmpty()) { - StartEndAndValues values = holders.remove(random().nextInt(holders.size())); + StartEndAndValues values = holders.remove(random.nextInt(holders.size())); assertReader(reader, values); } - if (random().nextBoolean()) { + if (random.nextBoolean()) { slicedIntBlockPool.reset(true, false); assertEquals(0, bytesUsed.get()); } else { diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java index 3e6c12d18978..cba81e845e99 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/complexPhrase/TestComplexPhraseQuery.java @@ -37,9 +37,11 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.analysis.MockSynonymAnalyzer; -import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCaseJupiter; +import org.apache.lucene.util.IOUtils; +import org.junit.jupiter.api.Test; -public class TestComplexPhraseQuery extends LuceneTestCase { +public class TestComplexPhraseQuery extends LuceneTestCaseJupiter { Directory rd; Analyzer analyzer; DocData[] docsContent = { @@ -60,6 +62,7 @@ public class TestComplexPhraseQuery extends LuceneTestCase { boolean inOrder = true; + @Test public void testComplexPhrases() throws Exception { checkMatches("\"john smith\"", "1"); // Simple multi-term still works checkMatches("\"j* smyth~\"", "1,2"); // wildcards and fuzzies are OK in @@ -85,6 +88,7 @@ public void testComplexPhrases() throws Exception { checkBadQuery("\"jo* \"smith\" \""); // phrases inside phrases is bad } + @Test public void testSingleTermPhrase() throws Exception { checkMatches("\"joh*\"", "1,2,3,5"); checkMatches("\"joh~\"", "1,3,5"); @@ -94,6 +98,7 @@ public void testSingleTermPhrase() throws Exception { checkMatches("+\"j*hn\" +\"sm*h\"", "1,3"); } + @Test public void testSynonyms() throws Exception { checkMatches("\"dogs\"", "8"); MockSynonymAnalyzer synonym = new MockSynonymAnalyzer(); @@ -109,6 +114,7 @@ public void testSynonyms() throws Exception { checkMatches("\"dog cigar*\"~2", "7", synonym); } + @Test public void testUnOrderedProximitySearches() throws Exception { inOrder = true; @@ -159,6 +165,7 @@ private void checkMatches(String qString, String expectedVals, Analyzer anAnalyz assertEquals(qString + " missing some matches ", 0, expecteds.size()); } + @Test public void testFieldedQuery() throws Exception { checkMatches("name:\"john smith\"", "1"); checkMatches("name:\"j* smyth~\"", "1,2"); @@ -171,6 +178,7 @@ public void testFieldedQuery() throws Exception { checkMatches("name:\"john smith\"~2 AND role:designer AND id:3", "3"); } + @Test public void testToStringContainsSlop() throws Exception { ComplexPhraseQueryParser qp = new ComplexPhraseQueryParser("", analyzer); int slop = random().nextInt(31) + 1; @@ -186,6 +194,7 @@ public void testToStringContainsSlop() throws Exception { assertEquals("Don't show implicit slop of zero", q.toString(), string); } + @Test public void testHashcodeEquals() throws Exception { ComplexPhraseQueryParser qp = new ComplexPhraseQueryParser(defaultFieldName, analyzer); qp.setInOrder(true); @@ -213,6 +222,7 @@ public void testHashcodeEquals() throws Exception { assertTrue(!q2.equals(q)); } + @Test public void testBoosts() throws Exception { // top-level boosts should be preserved, interior boosts are ignored as they don't apply to @@ -255,9 +265,8 @@ public void setUp() throws Exception { @Override public void tearDown() throws Exception { - reader.close(); - rd.close(); super.tearDown(); + IOUtils.close(reader, rd); } static class DocData { diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/ShapeRectRelationTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/ShapeRectRelationTestCase.java index 992f228e8f73..108fe97d6d96 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/ShapeRectRelationTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/ShapeRectRelationTestCase.java @@ -18,8 +18,6 @@ import static org.locationtech.spatial4j.distance.DistanceUtils.DEGREES_TO_RADIANS; -import org.junit.Rule; -import org.locationtech.spatial4j.TestLog; import org.locationtech.spatial4j.context.SpatialContext; import org.locationtech.spatial4j.shape.Circle; import org.locationtech.spatial4j.shape.Point; @@ -30,8 +28,6 @@ public abstract class ShapeRectRelationTestCase extends RandomizedShapeTestCase { protected static final double RADIANS_PER_DEGREE = Math.PI / 180.0; - @Rule public final TestLog testLog = TestLog.instance; - protected int maxRadius = 180; public ShapeRectRelationTestCase() { diff --git a/lucene/spatial-test-fixtures/src/java/org/apache/lucene/spatial3d/tests/RandomGeo3dShapeGenerator.java b/lucene/spatial-test-fixtures/src/java/org/apache/lucene/spatial3d/tests/RandomGeo3dShapeGenerator.java index 2f963aa715d1..d5a0f5457463 100644 --- a/lucene/spatial-test-fixtures/src/java/org/apache/lucene/spatial3d/tests/RandomGeo3dShapeGenerator.java +++ b/lucene/spatial-test-fixtures/src/java/org/apache/lucene/spatial3d/tests/RandomGeo3dShapeGenerator.java @@ -42,6 +42,7 @@ import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; import org.apache.lucene.spatial3d.geom.GeoShape; import org.apache.lucene.spatial3d.geom.PlanetModel; +import org.apache.lucene.tests.util.LuceneTestCase; /** * Class for generating random Geo3dShapes. They can be generated under given constraints which are @@ -83,7 +84,7 @@ private RandomGeo3dShapeGenerator() {} * @return Returns a private-use random forked from the current {@link RandomizedContext}. */ private static Random random() { - return new Random(RandomizedContext.current().getRandom().nextLong()); + return LuceneTestCase.nonAssertingRandom(LuceneTestCase.random()); } /** diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestBytesRefSorters.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestBytesRefSorters.java index 119fa5dd9bef..96286e3cb539 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestBytesRefSorters.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestBytesRefSorters.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.search.suggest.fst; -import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.generators.RandomBytes; import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import java.io.IOException; @@ -82,7 +81,7 @@ private void check(BytesRefSorter sorter) throws Exception { } private void appendRandomSequences(BytesRefSorter sorter) throws IOException { - Random rnd = new Random(RandomizedContext.current().getRandom().nextLong()); + Random rnd = LuceneTestCase.nonAssertingRandom(LuceneTestCase.random()); for (int i = 0; i < RandomNumbers.randomIntBetween(rnd, 10, 100); i++) { sorter.add(new BytesRef(RandomBytes.randomBytesOfLengthBetween(rnd, 1, 256))); } diff --git a/lucene/test-framework/build.gradle b/lucene/test-framework/build.gradle index 36687e87b833..0b5ea3335217 100644 --- a/lucene/test-framework/build.gradle +++ b/lucene/test-framework/build.gradle @@ -30,7 +30,13 @@ dependencies { }) moduleApi deps.hamcrest + moduleApi(deps.randomizedtesting.jupiter) + moduleApi(deps.junitplatform.jupiter) + moduleImplementation project(':lucene:codecs') + + moduleTestImplementation deps.junitplatform.testkit + moduleTestImplementation deps.assertj } // diff --git a/lucene/test-framework/src/java/module-info.java b/lucene/test-framework/src/java/module-info.java index 277533540485..b841ae72c441 100644 --- a/lucene/test-framework/src/java/module-info.java +++ b/lucene/test-framework/src/java/module-info.java @@ -22,11 +22,15 @@ requires org.apache.lucene.codecs; requires transitive junit; requires transitive randomizedtesting.runner; + requires transitive com.carrotsearch.randomizedtesting; + requires transitive org.junit.jupiter.api; requires org.hamcrest; // Open certain packages for junit because it scans methods via reflection. opens org.apache.lucene.tests.index to junit; + opens org.apache.lucene.tests.util to + org.junit.platform.commons; exports org.apache.lucene.tests.analysis.standard; exports org.apache.lucene.tests.analysis; diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/analysis/MockTokenizer.java b/lucene/test-framework/src/java/org/apache/lucene/tests/analysis/MockTokenizer.java index c5e5bddf0952..aef160c7fb40 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/analysis/MockTokenizer.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/analysis/MockTokenizer.java @@ -16,13 +16,13 @@ */ package org.apache.lucene.tests.analysis; -import com.carrotsearch.randomizedtesting.RandomizedContext; import java.io.IOException; import java.nio.CharBuffer; import java.util.Random; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.AttributeFactory; import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; @@ -105,7 +105,7 @@ private enum State { private boolean enableChecks = true; // evil: but we don't change the behavior with this random, we only switch up how we read - private final Random random = new Random(RandomizedContext.current().getRandom().nextLong()); + private final Random random; public MockTokenizer( AttributeFactory factory, @@ -117,6 +117,7 @@ public MockTokenizer( this.lowerCase = lowerCase; this.state = 0; this.maxTokenLength = maxTokenLength; + this.random = new Random(LuceneTestCase.random().nextLong()); } public MockTokenizer(CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/geo/GeoTestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/geo/GeoTestUtil.java index 0876795ee324..e697d6f67d99 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/geo/GeoTestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/geo/GeoTestUtil.java @@ -21,7 +21,6 @@ import static org.apache.lucene.geo.GeoUtils.MIN_LAT_INCL; import static org.apache.lucene.geo.GeoUtils.MIN_LON_INCL; -import com.carrotsearch.randomizedtesting.RandomizedContext; import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; @@ -39,6 +38,7 @@ import org.apache.lucene.geo.Point; import org.apache.lucene.geo.Polygon; import org.apache.lucene.geo.Rectangle; +import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.SloppyMath; @@ -589,7 +589,7 @@ private static Polygon surpriseMePolygon() { /** Keep it simple, we don't need to take arbitrary Random for geo tests */ private static Random random() { - return RandomizedContext.current().getRandom(); + return LuceneTestCase.nonAssertingRandom(LuceneTestCase.random()); } /** diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/geo/ShapeTestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/geo/ShapeTestUtil.java index de312b578342..29f5a83e73f7 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/geo/ShapeTestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/geo/ShapeTestUtil.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.tests.geo; -import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.generators.BiasedNumbers; import java.util.ArrayList; import java.util.Random; @@ -243,7 +242,7 @@ public static float nextFloat(Random random) { /** Keep it simple, we don't need to take arbitrary Random for geo tests */ private static Random random() { - return RandomizedContext.current().getRandom(); + return LuceneTestCase.nonAssertingRandom(LuceneTestCase.random()); } /** Simple slow point in polygon check (for testing) */ diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/CloseableDirectory.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/AssertDirectoryClosed.java similarity index 70% rename from lucene/test-framework/src/java/org/apache/lucene/tests/util/CloseableDirectory.java rename to lucene/test-framework/src/java/org/apache/lucene/tests/util/AssertDirectoryClosed.java index 4df658841509..6c8c9fc771b9 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/CloseableDirectory.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/AssertDirectoryClosed.java @@ -21,15 +21,16 @@ import org.junit.Assert; /** - * Attempts to close a {@link BaseDirectoryWrapper}. - * - * @see LuceneTestCase#newDirectory(java.util.Random) + * Fails the test if the {@link BaseDirectoryWrapper} hasn't been closed and the test has no other + * failures. */ -final class CloseableDirectory implements Closeable { +final class AssertDirectoryClosed implements Closeable { + static final String MSG_PREFIX = "Directory not closed: "; + private final BaseDirectoryWrapper dir; - private final TestRuleMarkFailure failureMarker; + private final SuiteFailureState failureMarker; - public CloseableDirectory(BaseDirectoryWrapper dir, TestRuleMarkFailure failureMarker) { + AssertDirectoryClosed(BaseDirectoryWrapper dir, SuiteFailureState failureMarker) { this.dir = dir; this.failureMarker = failureMarker; } @@ -40,11 +41,14 @@ public void close() { // failures. try { if (failureMarker.wasSuccessful() && dir.isOpen()) { - Assert.fail("Directory not closed: " + dir); + Assert.fail(MSG_PREFIX + dir); } } finally { - // TODO: perform real close of the delegate: LUCENE-4058 - // dir.close(); + try { + dir.close(); + } catch (Throwable _) { + // ignore, we can't do much about it. LUCENE-4058 + } } } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/BeforeAfterCallback.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/BeforeAfterCallback.java new file mode 100644 index 000000000000..b4e61f5c3be1 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/BeforeAfterCallback.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.tests.util; + +/** Abstraction for before-test/before-suite callbacks. */ +interface BeforeAfterCallback { + default void before() throws Exception {} + + default void after() throws Exception {} +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/CallbacksToRuleAdapter.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/CallbacksToRuleAdapter.java new file mode 100644 index 000000000000..787517fa856e --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/CallbacksToRuleAdapter.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.tests.util; + +import java.util.ArrayList; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.MultipleFailureException; +import org.junit.runners.model.Statement; + +record CallbacksToRuleAdapter(BeforeAfterCallback callback) implements TestRule { + @Override + public Statement apply(Statement base, Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + final ArrayList errors = new ArrayList<>(); + + try { + callback().before(); + base.evaluate(); + } catch (Throwable t) { + errors.add(t); + } + + try { + callback().after(); + } catch (Throwable t) { + errors.add(t); + } + + MultipleFailureException.assertEmpty(errors); + } + }; + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/CustomMethodOrderer.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/CustomMethodOrderer.java new file mode 100644 index 000000000000..68bec397346a --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/CustomMethodOrderer.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.tests.util; + +import com.carrotsearch.randomizedtesting.Xoroshiro128PlusRandom; +import com.carrotsearch.randomizedtesting.jupiter.Hashing; +import com.carrotsearch.randomizedtesting.jupiter.Seed; +import com.carrotsearch.randomizedtesting.jupiter.SeedChain; +import com.carrotsearch.randomizedtesting.jupiter.SysProps; +import java.util.Collections; +import java.util.Optional; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.MethodOrdererContext; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** A {@link MethodOrderer} that provides random-seed dependent ordering. */ +public final class CustomMethodOrderer implements MethodOrderer { + @Override + public void orderMethods(MethodOrdererContext context) { + var seed = + SeedChain.parse( + context + .getConfigurationParameter(SysProps.TESTS_SEED.propertyKey) + .orElse(new Seed(Hashing.hash(context.getTestClass().getName())).toString())) + .seeds() + .getFirst() + .value(); + + Collections.shuffle(context.getMethodDescriptors(), new Xoroshiro128PlusRandom(seed)); + } + + @Override + public Optional getDefaultExecutionMode() { + return Optional.of(ExecutionMode.SAME_THREAD); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/EnsureSequentialExecution.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/EnsureSequentialExecution.java new file mode 100644 index 000000000000..4944c17e1c78 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/EnsureSequentialExecution.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.tests.util; + +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** An extension to ensure we're running in single-thread (same-thread) mode. */ +public class EnsureSequentialExecution implements BeforeAllCallback { + @Override + public void beforeAll(ExtensionContext context) throws Exception { + if (context.getExecutionMode() != ExecutionMode.SAME_THREAD) { + throw new AssertionError( + "Lucene tests must be run sequentially (static globals everywhere)."); + } + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/FieldToType.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/FieldToType.java new file mode 100644 index 000000000000..a8d0ef494a04 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/FieldToType.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.tests.util; + +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.util.BytesRef; + +class FieldToType implements BeforeAfterCallback { + private final Map fieldToType = new HashMap<>(); + + @Override + public void after() throws Exception { + reset(); + } + + synchronized void reset() { + fieldToType.clear(); + } + + private static Field createField(String name, Object value, FieldType fieldType) { + if (value instanceof String) { + return new Field(name, (String) value, fieldType); + } else if (value instanceof BytesRef) { + return new Field(name, (BytesRef) value, fieldType); + } else { + throw new IllegalArgumentException("value must be String or BytesRef"); + } + } + + public synchronized Field newField(Random random, String name, Object value, FieldType type) { + FieldType prevType = fieldToType.get(name); + + if (prevType != null) { + // always use the same fieldType for the same field name + return createField(name, value, prevType); + } + + // TODO: once all core & test codecs can index + // offsets, sometimes randomly turn on offsets if we are + // already indexing positions... + + FieldType newType = new FieldType(type); + if (!newType.stored() && random.nextBoolean()) { + newType.setStored(true); // randomly store it + } + if (newType.indexOptions() != IndexOptions.NONE + && newType.indexOptions() != IndexOptions.DOCS_AND_CUSTOM_FREQS) { + if (!newType.storeTermVectors() && random.nextBoolean()) { + newType.setStoreTermVectors(true); + if (!newType.storeTermVectorPositions()) { + newType.setStoreTermVectorPositions(random.nextBoolean()); + if (newType.storeTermVectorPositions()) { + if (!newType.storeTermVectorPayloads()) { + newType.setStoreTermVectorPayloads(random.nextBoolean()); + } + } + } + // Check for strings as offsets are disallowed on binary fields + if (value instanceof String && !newType.storeTermVectorOffsets()) { + newType.setStoreTermVectorOffsets(random.nextBoolean()); + } + + if (LuceneTestCaseParent.VERBOSE) { + System.out.println("NOTE: LuceneTestCase: upgrade name=" + name + " type=" + newType); + } + } + } + newType.freeze(); + fieldToType.put(name, newType); + + // TODO: we need to do this, but smarter, ie, most of + // the time we set the same value for a given field but + // sometimes (rarely) we change it up: + /* + if (newType.omitNorms()) { + newType.setOmitNorms(random.nextBoolean()); + } + */ + + return createField(name, value, newType); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/IsSystemThread.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/IsSystemThread.java new file mode 100644 index 000000000000..d6df8e1a2775 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/IsSystemThread.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.tests.util; + +import com.carrotsearch.randomizedtesting.jupiter.DetectThreadLeaks; +import java.util.function.Predicate; +import java.util.stream.Stream; +import org.apache.lucene.util.Constants; + +/** + * This predicate should return {@code true} for threads that should be ignored in {@linkplain + * DetectThreadLeaks thread leak detection}. + */ +public class IsSystemThread implements Predicate { + static final boolean isJ9; + + static { + isJ9 = Constants.JAVA_VENDOR.startsWith("IBM"); + } + + @Override + public boolean test(Thread t) { + var threadName = t.getName(); + switch (threadName) { + case "ClassCache Reaper": // LUCENE-6518 + case "junit-jupiter-timeout-watcher": // junit5/jupiter timeouts. + case "JNA Cleaner": // JNA cleaner thread (system). + return true; + } + + if (isJ9 + && Stream.of(t.getStackTrace()) + .anyMatch(frame -> frame.getClassName().equals("java.util.Timer$TimerImpl"))) { + return true; + } + + return false; + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java index cbe18509ea61..a8f51bf0511d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java @@ -17,20 +17,13 @@ package org.apache.lucene.tests.util; -import static com.carrotsearch.randomizedtesting.RandomizedTest.frequently; -import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsInt; -import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; -import static org.apache.lucene.search.IndexSearcher.LeafSlice; import com.carrotsearch.randomizedtesting.JUnit4MethodProvider; -import com.carrotsearch.randomizedtesting.LifecycleScope; import com.carrotsearch.randomizedtesting.MixWithSuiteName; import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.RandomizedRunner; -import com.carrotsearch.randomizedtesting.RandomizedTest; -import com.carrotsearch.randomizedtesting.Xoroshiro128PlusRandom; import com.carrotsearch.randomizedtesting.annotations.Listeners; import com.carrotsearch.randomizedtesting.annotations.SeedDecorators; import com.carrotsearch.randomizedtesting.annotations.TestGroup; @@ -46,183 +39,44 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; -import com.carrotsearch.randomizedtesting.generators.RandomNumbers; -import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.rules.NoClassHooksShadowingRule; import com.carrotsearch.randomizedtesting.rules.NoInstanceHooksOverridesRule; -import java.io.Closeable; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.lang.StackWalker.Option; -import java.lang.StackWalker.StackFrame; +import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; import java.lang.annotation.Documented; import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import java.lang.reflect.Constructor; import java.lang.reflect.Method; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.text.Collator; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; -import java.util.Map; import java.util.Random; -import java.util.Set; -import java.util.TimeZone; -import java.util.TreeSet; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.regex.Pattern; -import junit.framework.AssertionFailedError; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.codecs.CompoundFormat; -import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.bitvectors.HnswBitVectorsFormat; -import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.CodecReader; -import org.apache.lucene.index.CompositeReader; -import org.apache.lucene.index.ConcurrentMergeScheduler; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.LiveIndexWriterConfig; -import org.apache.lucene.index.LogByteSizeMergePolicy; -import org.apache.lucene.index.LogDocMergePolicy; -import org.apache.lucene.index.LogMergePolicy; -import org.apache.lucene.index.MergePolicy; -import org.apache.lucene.index.MergeScheduler; -import org.apache.lucene.index.MultiBits; -import org.apache.lucene.index.MultiDocValues; -import org.apache.lucene.index.MultiTerms; -import org.apache.lucene.index.NoDeletionPolicy; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.ParallelCompositeReader; -import org.apache.lucene.index.ParallelLeafReader; -import org.apache.lucene.index.PointValues; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.SerialMergeScheduler; -import org.apache.lucene.index.SimpleMergedSegmentWarmer; -import org.apache.lucene.index.SnapshotDeletionPolicy; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.StoredFields; -import org.apache.lucene.index.TermVectors; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.TieredMergePolicy; -import org.apache.lucene.index.VectorEncoding; -import org.apache.lucene.internal.tests.IndexPackageAccess; -import org.apache.lucene.internal.tests.TestSecrets; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LRUQueryCache; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryCache; -import org.apache.lucene.search.QueryCachingPolicy; -import org.apache.lucene.store.ByteBuffersDirectory; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FSLockFactory; -import org.apache.lucene.store.FileSwitchDirectory; -import org.apache.lucene.store.FlushInfo; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.MergeInfo; -import org.apache.lucene.store.NRTCachingDirectory; -import org.apache.lucene.store.ReadOnceHint; -import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.codecs.asserting.AssertingCodec; -import org.apache.lucene.tests.index.AlcoholicMergePolicy; -import org.apache.lucene.tests.index.AssertingDirectoryReader; -import org.apache.lucene.tests.index.AssertingLeafReader; -import org.apache.lucene.tests.index.FieldFilterLeafReader; -import org.apache.lucene.tests.index.MergingCodecReader; -import org.apache.lucene.tests.index.MergingDirectoryReaderWrapper; -import org.apache.lucene.tests.index.MismatchedCodecReader; -import org.apache.lucene.tests.index.MismatchedDirectoryReader; -import org.apache.lucene.tests.index.MismatchedLeafReader; -import org.apache.lucene.tests.index.MockIndexWriterEventListener; -import org.apache.lucene.tests.index.MockRandomMergePolicy; -import org.apache.lucene.tests.mockfile.VirusCheckingFS; -import org.apache.lucene.tests.search.AssertingIndexSearcher; -import org.apache.lucene.tests.store.BaseDirectoryWrapper; -import org.apache.lucene.tests.store.MockDirectoryWrapper; -import org.apache.lucene.tests.store.MockDirectoryWrapper.Throttling; -import org.apache.lucene.tests.store.RawDirectoryWrapper; -import org.apache.lucene.tests.util.automaton.AutomatonTestUtil; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CommandLineUtil; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.InfoStream; -import org.apache.lucene.util.NamedThreadFactory; -import org.apache.lucene.util.SuppressForbidden; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.CompiledAutomaton; -import org.apache.lucene.util.automaton.Operations; -import org.apache.lucene.util.automaton.RegExp; -import org.hamcrest.Matcher; -import org.hamcrest.MatcherAssert; import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; -import org.junit.internal.AssumptionViolatedException; +import org.junit.jupiter.api.Tag; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; import org.junit.runner.RunWith; /// Base class for all Lucene unit tests (JUnit4 variant). /// +/// **Please consider using the new JUnit Jupiter base class [LuceneTestCaseJupiter] instead. This +/// class will be eventually removed from Lucene codebase.** +/// /// ## Class and instance setup /// /// The preferred way to specify class (suite-level) setup/cleanup is to use static methods -/// annotated with [BeforeClass] and [AfterClass]. Any code in these methods is executed +/// annotated with [org.junit.BeforeClass] and [org.junit.AfterClass]. Any code in these methods +/// is executed /// within the test framework's control and ensure proper setup has been made. **Try not to use /// static initializers (including complex final field initializers).** Static initializers are /// executed before any setup rules are fired and may cause you (or somebody else) headaches. @@ -256,7 +110,7 @@ /// - as part of the main thread executing the test case (if your test hangs, just dump the stack /// trace of all threads, and you'll see the seed), /// - the master seed can also be accessed manually by getting the current context ( -/// [RandomizedContext#current()]) and then calling +/// [RandomizedContext#current()]) and then callingd /// [RandomizedContext#getRunnerSeedAsString()]. /// @RunWith(RandomizedRunner.class) @@ -276,47 +130,16 @@ @TestRuleLimitSysouts.Limit( bytes = TestRuleLimitSysouts.DEFAULT_LIMIT, hardLimit = TestRuleLimitSysouts.DEFAULT_HARD_LIMIT) -public abstract class LuceneTestCase extends Assert { - - // -------------------------------------------------------------------- - // Test groups, system properties and other annotations modifying tests - // -------------------------------------------------------------------- - - public static final String SYSPROP_NIGHTLY = "tests.nightly"; - public static final String SYSPROP_WEEKLY = "tests.weekly"; - public static final String SYSPROP_MONSTER = "tests.monster"; - public static final String SYSPROP_AWAITSFIX = "tests.awaitsfix"; - - /** - * @see #ignoreAfterMaxFailures - */ - public static final String SYSPROP_MAXFAILURES = "tests.maxfailures"; - - /** - * @see #ignoreAfterMaxFailures - */ - public static final String SYSPROP_FAILFAST = "tests.failfast"; - - /** - * If specified, limits the number of method calls to each individual instance returned by {@link - * #random()}. - * - * @see #randomSupplier - */ - public static final String SYSPROP_RANDOM_MAXCALLS = "tests.random.maxcalls"; - - /** - * If specified, limits the number of calls {@link #random()} itself. - * - * @see #randomSupplier - */ - public static final String SYSPROP_RANDOM_MAXACQUIRES = "tests.random.maxacquires"; +public abstract non-sealed class LuceneTestCase extends LuceneTestCaseParent { + static final void ensureInitialized() {} /** Annotation for tests that should only be run during nightly builds. */ @Documented @Inherited @Retention(RetentionPolicy.RUNTIME) @TestGroup(enabled = false, sysProperty = SYSPROP_NIGHTLY) + @Tag("nightly") + @TagState(enabled = false, sysProperty = SYSPROP_NIGHTLY) public @interface Nightly {} /** Annotation for tests that should only be run during weekly builds */ @@ -324,6 +147,8 @@ public abstract class LuceneTestCase extends Assert { @Inherited @Retention(RetentionPolicy.RUNTIME) @TestGroup(enabled = false, sysProperty = SYSPROP_WEEKLY) + @Tag("weekly") + @TagState(enabled = false, sysProperty = SYSPROP_WEEKLY) public @interface Weekly {} /** Annotation for monster tests that require special setup (e.g. use tons of disk and RAM) */ @@ -331,6 +156,8 @@ public abstract class LuceneTestCase extends Assert { @Inherited @Retention(RetentionPolicy.RUNTIME) @TestGroup(enabled = false, sysProperty = SYSPROP_MONSTER) + @Tag("monster") + @TagState(enabled = false, sysProperty = SYSPROP_MONSTER) public @interface Monster { String value(); } @@ -340,6 +167,8 @@ public abstract class LuceneTestCase extends Assert { @Inherited @Retention(RetentionPolicy.RUNTIME) @TestGroup(enabled = false, sysProperty = SYSPROP_AWAITSFIX) + @Tag("awaitsfix") + @TagState(enabled = false, sysProperty = SYSPROP_AWAITSFIX) public @interface AwaitsFix { /** Point to JIRA entry. */ public String bugUrl(); @@ -439,134 +268,6 @@ public abstract class LuceneTestCase extends Assert { @Target(ElementType.TYPE) public @interface SuppressReproduceLine {} - // ----------------------------------------------------------------- - // Truly immutable fields and constants, initialized once and valid - // for all suites ever since. - // ----------------------------------------------------------------- - - /** - * True if and only if tests are run in verbose mode. If this flag is false tests are not expected - * to print any messages. Enforced with {@link TestRuleLimitSysouts}. - */ - public static final boolean VERBOSE = systemPropertyAsBoolean("tests.verbose", false); - - /** Enables or disables dumping of {@link InfoStream} messages. */ - public static final boolean INFOSTREAM = systemPropertyAsBoolean("tests.infostream", VERBOSE); - - /** - * True if {@code tests.asserts} is enabled (either explicitly via the build option or, if not - * present, by the default assertion status on this class). - */ - public static final boolean TEST_ASSERTS_ENABLED = - systemPropertyAsBoolean("tests.asserts", LuceneTestCase.class.desiredAssertionStatus()); - - /** - * The default (embedded resource) lines file. - * - * @see #TEST_LINE_DOCS_FILE - */ - public static final String DEFAULT_LINE_DOCS_FILE = "europarl.lines.txt.gz"; - - /** - * Random sample from enwiki used in tests. See {@code help/tests.txt}. gradle task downloading - * this data set: {@code gradlew getEnWikiRandomLines}. - */ - public static final String JENKINS_LARGE_LINE_DOCS_FILE = "enwiki.random.lines.txt"; - - /** Gets the codec to run tests with. */ - public static final String TEST_CODEC = System.getProperty("tests.codec", "random"); - - /** Gets the postingsFormat to run tests with. */ - public static final String TEST_POSTINGSFORMAT = - System.getProperty("tests.postingsformat", "random"); - - /** Gets the docValuesFormat to run tests with */ - public static final String TEST_DOCVALUESFORMAT = - System.getProperty("tests.docvaluesformat", "random"); - - /** Gets the directory to run tests with */ - public static final String TEST_DIRECTORY = System.getProperty("tests.directory", "random"); - - /** The line file used in tests (by {@link LineFileDocs}). */ - public static final String TEST_LINE_DOCS_FILE = - System.getProperty("tests.linedocsfile", DEFAULT_LINE_DOCS_FILE); - - /** Whether or not {@link Nightly} tests should run. */ - public static final boolean TEST_NIGHTLY = - systemPropertyAsBoolean( - SYSPROP_NIGHTLY, Nightly.class.getAnnotation(TestGroup.class).enabled()); - - /** Whether or not {@link Weekly} tests should run. */ - public static final boolean TEST_WEEKLY = - systemPropertyAsBoolean( - SYSPROP_WEEKLY, Weekly.class.getAnnotation(TestGroup.class).enabled()); - - /** Whether or not {@link Monster} tests should run. */ - public static final boolean TEST_MONSTER = - systemPropertyAsBoolean( - SYSPROP_MONSTER, Monster.class.getAnnotation(TestGroup.class).enabled()); - - /** Whether or not {@link AwaitsFix} tests should run. */ - public static final boolean TEST_AWAITSFIX = - systemPropertyAsBoolean( - SYSPROP_AWAITSFIX, AwaitsFix.class.getAnnotation(TestGroup.class).enabled()); - - /** Throttling, see {@link MockDirectoryWrapper#setThrottling(Throttling)}. */ - public static final Throttling TEST_THROTTLING = - TEST_NIGHTLY ? Throttling.SOMETIMES : Throttling.NEVER; - - /** - * A random multiplier which you should use when writing random tests: multiply it by the number - * of iterations to scale your tests (for nightly builds). - */ - public static final int RANDOM_MULTIPLIER = - systemPropertyAsInt("tests.multiplier", defaultRandomMultiplier()); - - /** Compute the default value of the random multiplier (based on {@link #TEST_NIGHTLY}). */ - static int defaultRandomMultiplier() { - return TEST_NIGHTLY ? 2 : 1; - } - - /** Leave temporary files on disk, even on successful runs. */ - public static final boolean LEAVE_TEMPORARY; - - static { - boolean defaultValue = false; - for (String property : - Arrays.asList( - "tests.leaveTemporary" /* ANT tasks' (junit4) flag. */, - "tests.leavetemporary" /* lowercase */, - "tests.leavetmpdir" /* default */)) { - defaultValue |= systemPropertyAsBoolean(property, false); - } - LEAVE_TEMPORARY = defaultValue; - } - - /** Filesystem-based {@link Directory} implementations. */ - private static final List FS_DIRECTORIES = - Arrays.asList("NIOFSDirectory", "MMapDirectory"); - - /** All {@link Directory} implementations. */ - private static final List CORE_DIRECTORIES; - - static { - CORE_DIRECTORIES = new ArrayList<>(FS_DIRECTORIES); - CORE_DIRECTORIES.add(ByteBuffersDirectory.class.getSimpleName()); - } - - /** A {@link org.apache.lucene.search.QueryCachingPolicy} that randomly caches. */ - public static final QueryCachingPolicy MAYBE_CACHE_POLICY = - new QueryCachingPolicy() { - - @Override - public void onUse(Query query) {} - - @Override - public boolean shouldCache(Query query) throws IOException { - return random().nextBoolean(); - } - }; - // ----------------------------------------------------------------- // Fields initialized in class or instance rules. // ----------------------------------------------------------------- @@ -575,19 +276,9 @@ public boolean shouldCache(Query query) throws IOException { // Class level (suite) rules. // ----------------------------------------------------------------- - /** Stores the currently class under test. */ + /** Stores the current class under test. */ private static final TestRuleStoreClassName classNameRule; - /** Class environment setup rule. */ - static final TestRuleSetupAndRestoreClassEnv classEnvRule; - - /** Suite failure marker (any error in the test or suite scope). */ - @SuppressWarnings("NonFinalStaticField") - protected static TestRuleMarkFailure suiteFailureMarker; - - /** Temporary files cleanup rule. */ - private static final TestRuleTemporaryFilesCleanup tempFilesCleanupRule; - /** * Ignore tests after hitting a designated number of initial failures. This is truly a "static" * global singleton since it needs to span the lifetime of all test classes running inside this @@ -649,19 +340,104 @@ public static TestRuleIgnoreAfterMaxFailures replaceMaxFailureRule( */ @ClassRule public static final TestRule classRules; + @SuppressWarnings("NonFinalStaticField") + private static TestRuleMarkFailure suiteFailureMarker; + + private static final FieldToType fieldToType; + static { - RuleChain r = + var setupAndRestoreClassEnv = + new SetupAndRestoreStaticEnv( + () -> RandomizedContext.current().getRandom(), + () -> RandomizedContext.current().getTargetClass()); + + suiteFailureMarker = new TestRuleMarkFailure(); + + var tempFilesSupplier = + new TemporaryFilesSupplier( + suiteFailureMarker, + LuceneTestCase::random, + () -> RandomizedContext.current().getTargetClass()); + + classRules = RuleChain.outerRule(new TestRuleIgnoreTestSuites()) + .around( + new TestRuleAdapter() { + @SuppressWarnings("NonFinalStaticField") + private static TestFrameworkInfra testFrameworkInfra; + + @Override + protected void before() throws Throwable { + int maxCalls = + Integer.parseInt(System.getProperty(SYSPROP_RANDOM_MAXCALLS, "0")); + Supplier supplier = () -> RandomizedContext.current().getRandom(); + if (maxCalls > 0) { + var finalizedSupplier = supplier; + supplier = () -> new MaxCallCountRandom(finalizedSupplier.get(), maxCalls); + } + + int maxAquires = + Integer.parseInt(System.getProperty(SYSPROP_RANDOM_MAXACQUIRES, "0")); + if (maxAquires > 0) { + var finalizedSupplier = supplier; + supplier = + () -> { + if (randomCalls.incrementAndGet() > maxAquires) { + throw new RuntimeException( + "Too many random() calls. Consider using LuceneTestCase.nonAssertingRandom for" + + " large loops or data generation."); + } + return finalizedSupplier.get(); + }; + } + + var finalizedSupplier = supplier; + setTestFrameworkInfra( + null, + testFrameworkInfra = + new TestFrameworkInfra() { + @Override + public Random threadRandom() { + return finalizedSupplier.get(); + } + + @Override + public SetupAndRestoreStaticEnv getClassEnv() { + return setupAndRestoreClassEnv; + } + + @Override + public TemporaryFilesSupplier getTempFilesSupplier() { + return tempFilesSupplier; + } + + @Override + public SuiteFailureState getSuiteFailureState() { + return suiteFailureMarker; + } + + @Override + public Field newField( + Random random, String name, Object value, FieldType type) { + return fieldToType.newField(random, name, value, type); + } + }); + } + + @Override + protected void afterAlways(List errors) { + setTestFrameworkInfra(testFrameworkInfra, null); + } + }) .around(ignoreAfterMaxFailures) - .around(suiteFailureMarker = new TestRuleMarkFailure()) + .around(suiteFailureMarker) .around( new VerifyTestClassNamingConvention( "org.apache.lucene", Pattern.compile("(.+\\.)(Test)([^.]+)"))) .around(new TestRuleAssertionsRequired()) .around(new TestRuleLimitSysouts(suiteFailureMarker)) - .around(tempFilesCleanupRule = new TestRuleTemporaryFilesCleanup(suiteFailureMarker)); - classRules = - r.around(new NoClassHooksShadowingRule()) + .around(new CallbacksToRuleAdapter(tempFilesSupplier)) + .around(new NoClassHooksShadowingRule()) .around( new NoInstanceHooksOverridesRule() { @Override @@ -680,7 +456,24 @@ protected boolean verify(Method key) { // We reset the default locale and timezone; these properties change as a // side-effect "user.language", "user.timezone")) - .around(classEnvRule = new TestRuleSetupAndRestoreClassEnv()); + .around(new CallbacksToRuleAdapter(setupAndRestoreClassEnv)) + .around(new CallbacksToRuleAdapter(fieldToType = new FieldToType())) + .around( + new CallbacksToRuleAdapter( + new BeforeAfterCallback() { + @Override + public void before() { + // Save environment information to reproduce-info listener. + // This listener can be invoked after all the tests and other callbacks have + // completed; I don't see any clean way to pass it there. + RunListenerPrintReproduceInfo.envInfoJunit4 = + new TestEnvInfo( + setupAndRestoreClassEnv.codec, + setupAndRestoreClassEnv.similarity, + setupAndRestoreClassEnv.locale, + setupAndRestoreClassEnv.timeZone); + } + })); } // ----------------------------------------------------------------- @@ -706,56 +499,13 @@ protected boolean verify(Method key) { RuleChain.outerRule(testFailureMarker) .around(ignoreAfterMaxFailures) .around(threadAndTestNameRule) - .around(new TestRuleSetupAndRestoreInstanceEnv()) + .around(new CallbacksToRuleAdapter(new TestRuleSetupAndRestoreInstanceEnv())) .around(parentChainCallRule); - private static final Map fieldToType = new HashMap<>(); - - enum LiveIWCFlushMode { - BY_RAM, - BY_DOCS, - EITHER - } - - /** Set by TestRuleSetupAndRestoreClassEnv */ - @SuppressWarnings("NonFinalStaticField") - static LiveIWCFlushMode liveIWCFlushMode; - - static void setLiveIWCFlushMode(LiveIWCFlushMode flushMode) { - liveIWCFlushMode = flushMode; - } - - private static final Supplier randomSupplier; - /** A counter of calls to {@link #random()} if {@link #SYSPROP_RANDOM_MAXACQUIRES} is defined. */ @SuppressWarnings("NonFinalStaticField") private static AtomicLong randomCalls = new AtomicLong(); - static { - int maxCalls = Integer.parseInt(System.getProperty(SYSPROP_RANDOM_MAXCALLS, "0")); - Supplier supplier = () -> RandomizedContext.current().getRandom(); - if (maxCalls > 0) { - var finalizedSupplier = supplier; - supplier = () -> new MaxCallCountRandom(finalizedSupplier.get(), maxCalls); - } - - int maxAquires = Integer.parseInt(System.getProperty(SYSPROP_RANDOM_MAXACQUIRES, "0")); - if (maxAquires > 0) { - var finalizedSupplier = supplier; - supplier = - () -> { - if (randomCalls.incrementAndGet() > maxAquires) { - throw new RuntimeException( - "Too many random() calls. Consider using LuceneTestCase.nonAssertingRandom for" - + " large loops or data generation."); - } - return finalizedSupplier.get(); - }; - } - - randomSupplier = supplier; - } - // ----------------------------------------------------------------- // Suite and test case setup/ cleanup. // ----------------------------------------------------------------- @@ -771,80 +521,17 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { parentChainCallRule.teardownCalled = true; - fieldToType.clear(); // Test is supposed to call this itself, but we do this defensively in case it forgot: restoreIndexWriterMaxDocs(); - } - - /** - * Tells {@link IndexWriter} to enforce the specified limit as the maximum number of documents in - * one index; call {@link #restoreIndexWriterMaxDocs} once your test is done. - */ - public void setIndexWriterMaxDocs(int limit) { - INDEX_PACKAGE_ACCESS.setIndexWriterMaxDocs(limit); - } - /** Returns to the default {@link IndexWriter#MAX_DOCS} limit. */ - public void restoreIndexWriterMaxDocs() { - INDEX_PACKAGE_ACCESS.setIndexWriterMaxDocs(IndexWriter.MAX_DOCS); + fieldToType.reset(); } - private static final IndexPackageAccess INDEX_PACKAGE_ACCESS = - TestSecrets.getIndexPackageAccess(); - // ----------------------------------------------------------------- // Test facilities and facades for subclasses. // ----------------------------------------------------------------- - /** - * Access to the current {@link RandomizedContext}'s Random instance. It is safe to use this - * method from multiple threads, etc., but it should be called while within a runner's scope (so - * no static initializers). The returned {@link Random} instance will be different when - * this method is called inside a {@link BeforeClass} hook (static suite scope) and within {@link - * Before}/ {@link After} hooks or test methods. - * - *

The returned instance must not be shared with other threads or cross a single scope's - * boundary. For example, a {@link Random} acquired within a test method shouldn't be reused for - * another test case. - * - *

There is an overhead connected with getting the {@link Random} for a particular context and - * thread. It is better to use a non-asserting {@link Random} instance locally if tight loops with - * multiple invocations are present. See {@link #nonAssertingRandom(Random)}. - */ - public static Random random() { - return randomSupplier.get(); - } - - /** - * Returns a Random instance based on the current state of another Random. The returned instance - * should be faster for thousands of consecutive calls because it doesn't assert that it isn't - * shared between threads or used within the correct {@link RandomizedContext}. - * - *

Use this method for local tight loops that generate a lot of random data. - */ - public static Random nonAssertingRandom(Random rnd) { - return new Xoroshiro128PlusRandom(rnd.nextLong()); - } - - /** - * Registers a {@link Closeable} resource that should be closed after the test completes. - * - * @return resource (for call chaining). - */ - public T closeAfterTest(T resource) { - return RandomizedContext.current().closeAtEnd(resource, LifecycleScope.TEST); - } - - /** - * Registers a {@link Closeable} resource that should be closed after the suite completes. - * - * @return resource (for call chaining). - */ - public static T closeAfterSuite(T resource) { - return RandomizedContext.current().closeAtEnd(resource, LifecycleScope.SUITE); - } - /** Return the current class being tested. */ public static Class getTestClass() { return classNameRule.getTestClass(); @@ -855,19 +542,6 @@ public String getTestName() { return threadAndTestNameRule.testMethodName; } - /** - * Some tests expect the directory to contain a single segment, and want to do tests on that - * segment's reader. This is an utility method to help them. - */ - public static LeafReader getOnlyLeafReader(IndexReader reader) { - List subReaders = reader.leaves(); - if (subReaders.size() != 1) { - throw new IllegalArgumentException( - reader + " has " + subReaders.size() + " segments instead of exactly one"); - } - return subReaders.get(0).reader(); - } - /** * Returns true if and only if the calling thread is the primary thread executing the test case. */ @@ -875,2469 +549,4 @@ protected boolean isTestThread() { assertNotNull("Test case thread not set?", threadAndTestNameRule.testCaseThread); return Thread.currentThread() == threadAndTestNameRule.testCaseThread; } - - /** - * Returns a number of at least i - * - *

The actual number returned will be influenced by whether {@link #TEST_NIGHTLY} is active and - * {@link #RANDOM_MULTIPLIER}, but also with some random fudge. - */ - public static int atLeast(Random random, int i) { - int min = i * RANDOM_MULTIPLIER; - int max = min + (min / 2); - return TestUtil.nextInt(random, min, max); - } - - public static int atLeast(int i) { - return atLeast(random(), i); - } - - /** - * Returns true if something should happen rarely, - * - *

The actual number returned will be influenced by whether {@link #TEST_NIGHTLY} is active and - * {@link #RANDOM_MULTIPLIER}. - */ - public static boolean rarely(Random random) { - int p = TEST_NIGHTLY ? 5 : 1; - p += (p * Math.log(RANDOM_MULTIPLIER)); - int min = 100 - Math.min(p, 20); // never more than 20 - return random.nextInt(100) >= min; - } - - public static boolean rarely() { - return rarely(random()); - } - - public static boolean usually(Random random) { - return !rarely(random); - } - - public static boolean usually() { - return usually(random()); - } - - public static void assumeTrue(String msg, boolean condition) { - RandomizedTest.assumeTrue(msg, condition); - } - - public static void assumeFalse(String msg, boolean condition) { - RandomizedTest.assumeFalse(msg, condition); - } - - public static void assumeNoException(String msg, Exception e) { - RandomizedTest.assumeNoException(msg, e); - } - - public static void assertFloatUlpEquals(final float x, final float y, final short maxUlps) { - assertTrue( - x + " and " + y + " are not within " + maxUlps + " ULPs of each other", - TestUtil.floatUlpEquals(x, y, maxUlps)); - } - - public static void assertDoubleUlpEquals(final double x, final double y, final int maxUlps) { - assertTrue( - x + " and " + y + " are not within " + maxUlps + " ULPs of each other", - TestUtil.doubleUlpEquals(x, y, maxUlps)); - } - - /** - * Return args as a {@link Set} instance. The order of elements is not preserved in - * iterators. - */ - @SafeVarargs - @SuppressWarnings("varargs") - public static Set asSet(T... args) { - return new HashSet<>(Arrays.asList(args)); - } - - /** - * Convenience method for logging an iterator. - * - * @param label String logged before/after the items in the iterator - * @param iter Each next() is toString()ed and logged on its own line. If iter is null this is - * logged differently then an empty iterator. - * @param stream Stream to log messages to. - */ - public static void dumpIterator(String label, Iterator iter, PrintStream stream) { - stream.println("*** BEGIN " + label + " ***"); - if (null == iter) { - stream.println(" ... NULL ..."); - } else { - while (iter.hasNext()) { - stream.println(iter.next().toString()); - } - } - stream.println("*** END " + label + " ***"); - } - - /** - * Convenience method for logging an array. Wraps the array in an iterator and delegates - * - * @see #dumpIterator(String, Iterator, PrintStream) - */ - public static void dumpArray(String label, Object[] objs, PrintStream stream) { - Iterator iter = (null == objs) ? null : Arrays.asList(objs).iterator(); - dumpIterator(label, iter, stream); - } - - /** create a new index writer config with a snapshot deletion policy */ - public static IndexWriterConfig newSnapshotIndexWriterConfig(Analyzer analyzer) { - IndexWriterConfig c = newIndexWriterConfig(analyzer); - c.setIndexDeletionPolicy(new SnapshotDeletionPolicy(NoDeletionPolicy.INSTANCE)); - return c; - } - - /** create a new index writer config with random defaults */ - public static IndexWriterConfig newIndexWriterConfig() { - return newIndexWriterConfig(new MockAnalyzer(random())); - } - - /** create a new index writer config with random defaults */ - public static IndexWriterConfig newIndexWriterConfig(Analyzer a) { - return newIndexWriterConfig(random(), a); - } - - /** create a new index writer config with random defaults using the specified random */ - public static IndexWriterConfig newIndexWriterConfig(Random r, Analyzer a) { - IndexWriterConfig c = new IndexWriterConfig(a); - configureRandomCompoundFormat(r, c.getCodec().compoundFormat()); - c.setSimilarity(classEnvRule.similarity); - if (INFOSTREAM) { - // Even though TestRuleSetupAndRestoreClassEnv calls - // InfoStream.setDefault, we do it again here so that - // the PrintStreamInfoStream.messageID increments so - // that when there are separate instances of - // IndexWriter created we see "IW 0", "IW 1", "IW 2", - // ... instead of just always "IW 0": - c.setInfoStream( - new TestRuleSetupAndRestoreClassEnv.ThreadNameFixingPrintStreamInfoStream(System.out)); - } - - if (rarely(r)) { - c.setMergeScheduler(new SerialMergeScheduler()); - } else if (rarely(r)) { - ConcurrentMergeScheduler cms; - if (r.nextBoolean()) { - cms = new TestConcurrentMergeScheduler(); - } else { - cms = - new TestConcurrentMergeScheduler() { - @Override - protected synchronized boolean maybeStall(MergeSource mergeSource) { - return true; - } - }; - } - int maxThreadCount = TestUtil.nextInt(r, 1, 4); - int maxMergeCount = TestUtil.nextInt(r, maxThreadCount, maxThreadCount + 4); - cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount); - if (r.nextBoolean()) { - cms.disableAutoIOThrottle(); - assertFalse(cms.getAutoIOThrottle()); - } - cms.setForceMergeMBPerSec(10 + 10 * r.nextDouble()); - c.setMergeScheduler(cms); - } else { - // Always use consistent settings, else CMS's dynamic (SSD or not) - // defaults can change, hurting reproducibility: - ConcurrentMergeScheduler cms = - r.nextBoolean() ? new TestConcurrentMergeScheduler() : new ConcurrentMergeScheduler(); - - // Only 1 thread can run at once (should maybe help reproducibility), - // with up to 3 pending merges before segment-producing threads are - // stalled: - cms.setMaxMergesAndThreads(3, 1); - c.setMergeScheduler(cms); - } - - if (r.nextBoolean()) { - if (rarely(r)) { - // crazy value - c.setMaxBufferedDocs(TestUtil.nextInt(r, 2, 15)); - } else { - // reasonable value - c.setMaxBufferedDocs(TestUtil.nextInt(r, 16, 1000)); - } - } - - c.setMergePolicy(newMergePolicy(r)); - - if (rarely(r)) { - c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream())); - } - c.setUseCompoundFile(r.nextBoolean()); - c.setReaderPooling(r.nextBoolean()); - if (rarely(r)) { - c.setCheckPendingFlushUpdate(false); - } - - if (rarely(r)) { - c.setIndexWriterEventListener(new MockIndexWriterEventListener()); - } - switch (r.nextInt(3)) { - case 0: - // Disable merge on refresh - c.setMaxFullFlushMergeWaitMillis(0L); - break; - case 1: - // Very low timeout, merges will likely not be able to run in time - c.setMaxFullFlushMergeWaitMillis(1L); - break; - default: - // Very long timeout, merges will almost always be able to run in time - c.setMaxFullFlushMergeWaitMillis(1000L); - break; - } - - c.setMaxFullFlushMergeWaitMillis(rarely(r) ? atLeast(r, 1000) : atLeast(r, 200)); - return c; - } - - public static MergePolicy newMergePolicy(Random r) { - return newMergePolicy(r, true); - } - - public static MergePolicy newMergePolicy(Random r, boolean includeMockMP) { - if (includeMockMP && rarely(r)) { - return new MockRandomMergePolicy(r); - } else if (r.nextBoolean()) { - return newTieredMergePolicy(r); - } else if (rarely(r)) { - return newAlcoholicMergePolicy(r, classEnvRule.timeZone); - } - return newLogMergePolicy(r); - } - - public static MergePolicy newMergePolicy() { - return newMergePolicy(random()); - } - - public static LogMergePolicy newLogMergePolicy() { - return newLogMergePolicy(random()); - } - - public static TieredMergePolicy newTieredMergePolicy() { - return newTieredMergePolicy(random()); - } - - public static AlcoholicMergePolicy newAlcoholicMergePolicy() { - return newAlcoholicMergePolicy(random(), classEnvRule.timeZone); - } - - public static AlcoholicMergePolicy newAlcoholicMergePolicy(Random r, TimeZone tz) { - return new AlcoholicMergePolicy(tz, new Random(r.nextLong())); - } - - public static LogMergePolicy newLogMergePolicy(Random r) { - LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy(); - logmp.setCalibrateSizeByDeletes(r.nextBoolean()); - logmp.setTargetSearchConcurrency(TestUtil.nextInt(r, 1, 16)); - if (rarely(r)) { - logmp.setMergeFactor(TestUtil.nextInt(r, 2, 9)); - } else { - logmp.setMergeFactor(TestUtil.nextInt(r, 10, 50)); - } - return logmp; - } - - private static void configureRandomCompoundFormat(Random r, CompoundFormat compoundFormat) { - compoundFormat.setShouldUseCompoundFile(r.nextBoolean()); - - if (rarely(r)) { - compoundFormat.setMaxCFSSegmentSizeMB(0.2 + r.nextDouble() * 2.0); - } else { - compoundFormat.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY); - } - } - - public static TieredMergePolicy newTieredMergePolicy(Random r) { - TieredMergePolicy tmp = new TieredMergePolicy(); - if (rarely(r)) { - tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0); - } else { - tmp.setMaxMergedSegmentMB(10 + r.nextDouble() * 100); - } - tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0); - tmp.setForceMergeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0); - if (rarely(r)) { - tmp.setSegmentsPerTier(TestUtil.nextInt(r, 2, 20)); - } else { - tmp.setSegmentsPerTier(TestUtil.nextInt(r, 10, 50)); - } - if (rarely(r)) { - tmp.setTargetSearchConcurrency(TestUtil.nextInt(r, 10, 50)); - } else { - tmp.setTargetSearchConcurrency(TestUtil.nextInt(r, 2, 20)); - } - - tmp.setDeletesPctAllowed(20 + r.nextDouble() * 30); - return tmp; - } - - public static LogMergePolicy newLogMergePolicy(int mergeFactor) { - LogMergePolicy logmp = newLogMergePolicy(); - logmp.setMergeFactor(mergeFactor); - return logmp; - } - - // if you want it in LiveIndexWriterConfig: it must and will be tested here. - public static void maybeChangeLiveIndexWriterConfig(Random r, LiveIndexWriterConfig c) { - boolean didChange = false; - - String previous = c.toString(); - - if (rarely(r)) { - // change flush parameters: - // this is complicated because the api requires you "invoke setters in a magical order!" - // LUCENE-5661: workaround for race conditions in the API - synchronized (c) { - boolean flushByRAM; - switch (liveIWCFlushMode) { - case BY_RAM: - flushByRAM = true; - break; - case BY_DOCS: - flushByRAM = false; - break; - case EITHER: - flushByRAM = r.nextBoolean(); - break; - default: - throw new AssertionError(); - } - if (flushByRAM) { - c.setRAMBufferSizeMB(TestUtil.nextInt(r, 1, 10)); - c.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); - } else { - if (rarely(r)) { - // crazy value - c.setMaxBufferedDocs(TestUtil.nextInt(r, 2, 15)); - } else { - // reasonable value - c.setMaxBufferedDocs(TestUtil.nextInt(r, 16, 1000)); - } - c.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); - } - } - didChange = true; - } - - if (rarely(r)) { - IndexWriter.IndexReaderWarmer curWarmer = c.getMergedSegmentWarmer(); - if (curWarmer == null || curWarmer instanceof SimpleMergedSegmentWarmer) { - // change warmer parameters - if (r.nextBoolean()) { - c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream())); - } else { - c.setMergedSegmentWarmer(null); - } - } - didChange = true; - } - - if (rarely(r)) { - // change CFS flush parameters - c.setUseCompoundFile(r.nextBoolean()); - didChange = true; - } - - if (rarely(r)) { - // change CMS merge parameters - MergeScheduler ms = c.getMergeScheduler(); - if (ms instanceof ConcurrentMergeScheduler cms) { - int maxThreadCount = TestUtil.nextInt(r, 1, 4); - int maxMergeCount = TestUtil.nextInt(r, maxThreadCount, maxThreadCount + 4); - boolean enableAutoIOThrottle = r.nextBoolean(); - if (enableAutoIOThrottle) { - cms.enableAutoIOThrottle(); - } else { - cms.disableAutoIOThrottle(); - } - cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount); - didChange = true; - } - } - - if (rarely(r)) { - MergePolicy mp = c.getMergePolicy(); - configureRandomCompoundFormat(r, c.getCodec().compoundFormat()); - if (mp instanceof LogMergePolicy logmp) { - logmp.setCalibrateSizeByDeletes(r.nextBoolean()); - if (rarely(r)) { - logmp.setMergeFactor(TestUtil.nextInt(r, 2, 9)); - } else { - logmp.setMergeFactor(TestUtil.nextInt(r, 10, 50)); - } - } else if (mp instanceof TieredMergePolicy tmp) { - if (rarely(r)) { - tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0); - } else { - tmp.setMaxMergedSegmentMB(r.nextDouble() * 100); - } - tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0); - tmp.setForceMergeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0); - if (rarely(r)) { - tmp.setSegmentsPerTier(TestUtil.nextInt(r, 2, 20)); - } else { - tmp.setSegmentsPerTier(TestUtil.nextInt(r, 10, 50)); - } - configureRandomCompoundFormat(r, c.getCodec().compoundFormat()); - tmp.setDeletesPctAllowed(20 + r.nextDouble() * 30); - } - didChange = true; - } - if (VERBOSE && didChange) { - String current = c.toString(); - String[] previousLines = previous.split("\n"); - String[] currentLines = current.split("\n"); - StringBuilder diff = new StringBuilder(); - - // this should always be the case, diff each line - if (previousLines.length == currentLines.length) { - for (int i = 0; i < previousLines.length; i++) { - if (!previousLines[i].equals(currentLines[i])) { - diff.append("- ").append(previousLines[i]).append("\n"); - diff.append("+ ").append(currentLines[i]).append("\n"); - } - } - } else { - // but just in case of something ridiculous... - diff.append(current); - } - - // its possible to be empty, if we "change" a value to what it had before. - if (diff.length() > 0) { - System.out.println("NOTE: LuceneTestCase: randomly changed IWC's live settings:"); - System.out.println(diff); - } - } - } - - /** - * Returns a new Directory instance. Use this when the test does not care about the specific - * Directory implementation (most tests). - * - *

The Directory is wrapped with {@link BaseDirectoryWrapper}. this means usually it will be - * picky, such as ensuring that you properly close it and all open files in your test. It will - * emulate some features of Windows, such as not allowing open files to be overwritten. - */ - public static BaseDirectoryWrapper newDirectory() { - return newDirectory(random()); - } - - /** Like {@link #newDirectory} except randomly the {@link VirusCheckingFS} may be installed */ - public static BaseDirectoryWrapper newMaybeVirusCheckingDirectory() { - if (random().nextInt(5) == 4) { - Path path = addVirusChecker(createTempDir()); - return newFSDirectory(path); - } else { - return newDirectory(random()); - } - } - - /** - * Returns a new Directory instance, using the specified random. See {@link #newDirectory()} for - * more information. - */ - public static BaseDirectoryWrapper newDirectory(Random r) { - return wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY), rarely(r), false); - } - - /** - * Returns a new Directory instance, using the specified random. See {@link #newDirectory()} for - * more information. - */ - public static BaseDirectoryWrapper newDirectory(Random r, LockFactory lf) { - return wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY, lf), rarely(r), false); - } - - public static MockDirectoryWrapper newMockDirectory() { - return newMockDirectory(random()); - } - - public static MockDirectoryWrapper newMockDirectory(Random r) { - return (MockDirectoryWrapper) - wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY), false, false); - } - - public static MockDirectoryWrapper newMockDirectory(Random r, LockFactory lf) { - return (MockDirectoryWrapper) - wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY, lf), false, false); - } - - public static MockDirectoryWrapper newMockFSDirectory(Path f) { - return (MockDirectoryWrapper) newFSDirectory(f, FSLockFactory.getDefault(), false); - } - - public static MockDirectoryWrapper newMockFSDirectory(Path f, LockFactory lf) { - return (MockDirectoryWrapper) newFSDirectory(f, lf, false); - } - - public static Path addVirusChecker(Path path) { - if (TestUtil.hasVirusChecker(path) == false) { - VirusCheckingFS fs = new VirusCheckingFS(path.getFileSystem(), random().nextLong()); - path = fs.wrapPath(path); - } - return path; - } - - /** - * Returns a new Directory instance, with contents copied from the provided directory. See {@link - * #newDirectory()} for more information. - */ - public static BaseDirectoryWrapper newDirectory(Directory d) throws IOException { - return newDirectory(random(), d); - } - - /** Returns a new FSDirectory instance over the given file, which must be a folder. */ - public static BaseDirectoryWrapper newFSDirectory(Path f) { - return newFSDirectory(f, FSLockFactory.getDefault()); - } - - /** Like {@link #newFSDirectory(Path)}, but randomly insert {@link VirusCheckingFS} */ - public static BaseDirectoryWrapper newMaybeVirusCheckingFSDirectory(Path f) { - if (random().nextInt(5) == 4) { - f = addVirusChecker(f); - } - return newFSDirectory(f, FSLockFactory.getDefault()); - } - - /** Returns a new FSDirectory instance over the given file, which must be a folder. */ - public static BaseDirectoryWrapper newFSDirectory(Path f, LockFactory lf) { - return newFSDirectory(f, lf, rarely()); - } - - private static BaseDirectoryWrapper newFSDirectory(Path f, LockFactory lf, boolean bare) { - String fsdirClass = TEST_DIRECTORY; - if (fsdirClass.equals("random")) { - fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES); - } - - Class clazz; - try { - try { - clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass); - } catch (ClassCastException _) { - // TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random - fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES); - clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass); - } - - Directory fsdir = newFSDirectoryImpl(clazz, f, lf); - return wrapDirectory(random(), fsdir, bare, true); - } catch (Exception e) { - Rethrow.rethrow(e); - throw null; // dummy to prevent compiler failure - } - } - - private static Directory newFileSwitchDirectory(Random random, Directory dir1, Directory dir2) { - List fileExtensions = - Arrays.asList( - "fdt", "fdx", "tim", "tip", "si", "fnm", "pos", "dii", "dim", "nvm", "nvd", "dvm", - "dvd"); - Collections.shuffle(fileExtensions, random); - fileExtensions = fileExtensions.subList(0, 1 + random.nextInt(fileExtensions.size())); - return new FileSwitchDirectory(new HashSet<>(fileExtensions), dir1, dir2, true); - } - - /** - * Returns a new Directory instance, using the specified random with contents copied from the - * provided directory. See {@link #newDirectory()} for more information. - */ - public static BaseDirectoryWrapper newDirectory(Random r, Directory d) throws IOException { - Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); - for (String file : d.listAll()) { - if (file.startsWith(IndexFileNames.SEGMENTS) - || IndexFileNames.CODEC_FILE_PATTERN.matcher(file).matches()) { - impl.copyFrom(d, file, file, newIOContext(r)); - } - } - return wrapDirectory(r, impl, rarely(r), false); - } - - private static BaseDirectoryWrapper wrapDirectory( - Random random, Directory directory, boolean bare, boolean filesystem) { - // IOContext randomization might make NRTCachingDirectory make bad decisions, so avoid - // using it if the user requested a filesystem directory. - if (rarely(random) && !bare && filesystem == false) { - directory = new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble()); - } - - if (bare) { - BaseDirectoryWrapper base = new RawDirectoryWrapper(directory); - closeAfterSuite(new CloseableDirectory(base, suiteFailureMarker)); - return base; - } else { - MockDirectoryWrapper mock = new MockDirectoryWrapper(random, directory); - - mock.setThrottling(TEST_THROTTLING); - closeAfterSuite(new CloseableDirectory(mock, suiteFailureMarker)); - return mock; - } - } - - public static Field newStringField(String name, String value, Store stored) { - return newField( - random(), - name, - value, - stored == Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); - } - - public static Field newStringField(String name, BytesRef value, Store stored) { - return newField( - random(), - name, - value, - stored == Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); - } - - public static Field newTextField(String name, String value, Store stored) { - return newField( - random(), - name, - value, - stored == Store.YES ? TextField.TYPE_STORED : TextField.TYPE_NOT_STORED); - } - - public static Field newStringField(Random random, String name, String value, Store stored) { - return newField( - random, - name, - value, - stored == Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); - } - - public static Field newStringField(Random random, String name, BytesRef value, Store stored) { - return newField( - random, - name, - value, - stored == Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); - } - - public static Field newTextField(Random random, String name, String value, Store stored) { - return newField( - random, - name, - value, - stored == Store.YES ? TextField.TYPE_STORED : TextField.TYPE_NOT_STORED); - } - - public static Field newField(String name, String value, FieldType type) { - return newField(random(), name, value, type); - } - - // TODO: if we can pull out the "make term vector options - // consistent across all instances of the same field name" - // write-once schema sort of helper class then we can - // remove the sync here. We can also fold the random - // "enable norms" (now commented out, below) into that: - public static synchronized Field newField( - Random random, String name, Object value, FieldType type) { - - // Defeat any consumers that illegally rely on intern'd - // strings (we removed this from Lucene a while back): - name = new String(name); - - FieldType prevType = fieldToType.get(name); - if (prevType != null) { - // always use the same fieldType for the same field name - return createField(name, value, prevType); - } - - // TODO: once all core & test codecs can index - // offsets, sometimes randomly turn on offsets if we are - // already indexing positions... - - FieldType newType = new FieldType(type); - if (!newType.stored() && random.nextBoolean()) { - newType.setStored(true); // randomly store it - } - if (newType.indexOptions() != IndexOptions.NONE - && newType.indexOptions() != IndexOptions.DOCS_AND_CUSTOM_FREQS) { - if (!newType.storeTermVectors() && random.nextBoolean()) { - newType.setStoreTermVectors(true); - if (!newType.storeTermVectorPositions()) { - newType.setStoreTermVectorPositions(random.nextBoolean()); - if (newType.storeTermVectorPositions()) { - if (!newType.storeTermVectorPayloads()) { - newType.setStoreTermVectorPayloads(random.nextBoolean()); - } - } - } - // Check for strings as offsets are disallowed on binary fields - if (value instanceof String && !newType.storeTermVectorOffsets()) { - newType.setStoreTermVectorOffsets(random.nextBoolean()); - } - if (VERBOSE) { - System.out.println("NOTE: LuceneTestCase: upgrade name=" + name + " type=" + newType); - } - } - } - newType.freeze(); - fieldToType.put(name, newType); - - // TODO: we need to do this, but smarter, ie, most of - // the time we set the same value for a given field but - // sometimes (rarely) we change it up: - /* - if (newType.omitNorms()) { - newType.setOmitNorms(random.nextBoolean()); - } - */ - - return createField(name, value, newType); - } - - private static Field createField(String name, Object value, FieldType fieldType) { - if (value instanceof String) { - return new Field(name, (String) value, fieldType); - } else if (value instanceof BytesRef) { - return new Field(name, (BytesRef) value, fieldType); - } else { - throw new IllegalArgumentException("value must be String or BytesRef"); - } - } - - private static final String[] availableLanguageTags = - Arrays.stream(Locale.getAvailableLocales()) - .map(Locale::toLanguageTag) - .sorted() - .distinct() - .toArray(String[]::new); - - /** - * Return a random Locale from the available locales on the system. - * - * @see LUCENE-4020 - */ - public static Locale randomLocale(Random random) { - return localeForLanguageTag( - availableLanguageTags[random.nextInt(availableLanguageTags.length)]); - } - - /** Time zone IDs that cause a deprecation warning in JDK 25. */ - private static final Set DEPRECATED_TIME_ZONE_IDS_JDK25 = - Set.of( - "ACT", "AET", "AGT", "ART", "AST", "BET", "BST", "CAT", "CNT", "CST", "CTT", "EAT", "ECT", - "EST", "HST", "IET", "IST", "JST", "MIT", "MST", "NET", "NST", "PLT", "PNT", "PRT", "PST", - "SST", "VST"); - - /** - * Return a random TimeZone from the available timezones on the system - * - * @see LUCENE-4020 - */ - public static TimeZone randomTimeZone(Random random) { - List tzIds = Arrays.asList(TimeZone.getAvailableIDs()); - // Remove time zones that cause deprecation warnings as these can break - // certain tests that expect exact output. - if (Runtime.version().feature() >= 25) { - tzIds = tzIds.stream().filter(id -> !DEPRECATED_TIME_ZONE_IDS_JDK25.contains(id)).toList(); - } - return TimeZone.getTimeZone(RandomPicks.randomFrom(random, tzIds)); - } - - /** return a Locale object equivalent to its programmatic name */ - public static Locale localeForLanguageTag(String languageTag) { - return new Locale.Builder().setLanguageTag(languageTag).build(); - } - - private static Directory newFSDirectoryImpl( - Class clazz, Path path, LockFactory lf) throws IOException { - FSDirectory d = null; - try { - d = CommandLineUtil.newFSDirectory(clazz, path, lf); - } catch (ReflectiveOperationException e) { - Rethrow.rethrow(e); - } - return d; - } - - static Directory newDirectoryImpl(Random random, String clazzName) { - return newDirectoryImpl(random, clazzName, FSLockFactory.getDefault()); - } - - static Directory newDirectoryImpl(Random random, String clazzName, LockFactory lf) { - if (clazzName.equals("random")) { - if (rarely(random)) { - clazzName = RandomPicks.randomFrom(random, CORE_DIRECTORIES); - } else if (rarely(random)) { - String clazzName1 = - rarely(random) - ? RandomPicks.randomFrom(random, CORE_DIRECTORIES) - : ByteBuffersDirectory.class.getName(); - String clazzName2 = - rarely(random) - ? RandomPicks.randomFrom(random, CORE_DIRECTORIES) - : ByteBuffersDirectory.class.getName(); - return newFileSwitchDirectory( - random, - newDirectoryImpl(random, clazzName1, lf), - newDirectoryImpl(random, clazzName2, lf)); - } else { - clazzName = ByteBuffersDirectory.class.getName(); - } - } - - try { - final Class clazz = CommandLineUtil.loadDirectoryClass(clazzName); - // If it is a FSDirectory type, try its ctor(Path) - if (FSDirectory.class.isAssignableFrom(clazz)) { - final Path dir = createTempDir("index-" + clazzName); - return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), dir, lf); - } - - // See if it has a Path/LockFactory ctor even though it's not an - // FSDir subclass: - try { - Constructor pathCtor = - clazz.getConstructor(Path.class, LockFactory.class); - final Path dir = createTempDir("index"); - return pathCtor.newInstance(dir, lf); - } catch (NoSuchMethodException _) { - // Ignore - } - - // the remaining dirs are no longer filesystem based, so we must check that the - // passedLockFactory is not file based: - if (!(lf instanceof FSLockFactory)) { - // try ctor with only LockFactory - try { - return clazz.getConstructor(LockFactory.class).newInstance(lf); - } catch (NoSuchMethodException _) { - // Ignore - } - } - - // try empty ctor - return clazz.getConstructor().newInstance(); - } catch (Exception e) { - Rethrow.rethrow(e); - throw null; // dummy to prevent compiler failure - } - } - - public static IndexReader wrapReader(IndexReader r) throws IOException { - Random random = random(); - - for (int i = 0, c = random.nextInt(6) + 1; i < c; i++) { - switch (random.nextInt(5)) { - case 0: - // will create no FC insanity in atomic case, as ParallelLeafReader has own cache key: - if (VERBOSE) { - System.out.println( - "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" - + r - + " with ParallelLeaf/CompositeReader"); - } - r = - (r instanceof LeafReader) - ? new ParallelLeafReader((LeafReader) r) - : new ParallelCompositeReader((CompositeReader) r); - break; - case 1: - if (r instanceof LeafReader ar) { - final List allFields = new ArrayList<>(); - for (FieldInfo fi : ar.getFieldInfos()) { - allFields.add(fi.name); - } - Collections.shuffle(allFields, random); - final int end = allFields.isEmpty() ? 0 : random.nextInt(allFields.size()); - final Set fields = new HashSet<>(allFields.subList(0, end)); - // will create no FC insanity as ParallelLeafReader has own cache key: - if (VERBOSE) { - System.out.println( - "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" - + r - + " with ParallelLeafReader"); - } - r = - new ParallelLeafReader( - new FieldFilterLeafReader(ar, fields, false), - new FieldFilterLeafReader(ar, fields, true)); - } - break; - case 2: - // Häckidy-Hick-Hack: a standard Reader will cause FC insanity, so we use - // QueryUtils' reader with a fake cache key, so insanity checker cannot walk - // along our reader: - if (VERBOSE) { - System.out.println( - "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" - + r - + " with AssertingLeaf/DirectoryReader"); - } - if (r instanceof LeafReader) { - r = new AssertingLeafReader((LeafReader) r); - } else if (r instanceof DirectoryReader) { - r = new AssertingDirectoryReader((DirectoryReader) r); - } - break; - case 3: - if (VERBOSE) { - System.out.println( - "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" - + r - + " with MismatchedLeaf/Directory/CodecReader"); - } - if (r instanceof LeafReader) { - r = new MismatchedLeafReader((LeafReader) r, random); - } else if (r instanceof DirectoryReader) { - r = new MismatchedDirectoryReader((DirectoryReader) r, random); - } else if (r instanceof CodecReader) { - r = new MismatchedCodecReader((CodecReader) r, random); - } - break; - case 4: - if (VERBOSE) { - System.out.println( - "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" - + r - + " with MergingCodecReader"); - } - if (r instanceof CodecReader) { - r = new MergingCodecReader((CodecReader) r); - } else if (r instanceof DirectoryReader) { - boolean allLeavesAreCodecReaders = true; - for (LeafReaderContext ctx : r.leaves()) { - if (ctx.reader() instanceof CodecReader == false) { - allLeavesAreCodecReaders = false; - break; - } - } - if (allLeavesAreCodecReaders) { - r = new MergingDirectoryReaderWrapper((DirectoryReader) r); - } - } - break; - default: - fail("should not get here"); - } - } - - if (VERBOSE) { - System.out.println("wrapReader wrapped: " + r); - } - - return r; - } - - /** Sometimes wrap the IndexReader as slow, parallel or filter reader (or combinations of that) */ - public static IndexReader maybeWrapReader(IndexReader r) throws IOException { - if (rarely()) { - r = wrapReader(r); - } - return r; - } - - /** TODO: javadoc */ - public static IOContext newIOContext(Random random) { - return newIOContext(random, IOContext.DEFAULT); - } - - /** TODO: javadoc */ - public static IOContext newIOContext(Random random, IOContext oldContext) { - if (oldContext.hints().contains(ReadOnceHint.INSTANCE)) { - return oldContext; // just return as-is - } - final int randomNumDocs = random.nextInt(4192); - final int size = random.nextInt(512) * randomNumDocs; - if (oldContext.flushInfo() != null) { - // Always return at least the estimatedSegmentSize of - // the incoming IOContext: - return IOContext.flush( - new FlushInfo( - randomNumDocs, Math.max(oldContext.flushInfo().estimatedSegmentSize(), size))); - } else if (oldContext.mergeInfo() != null) { - // Always return at least the estimatedMergeBytes of - // the incoming IOContext: - return IOContext.merge( - new MergeInfo( - randomNumDocs, - Math.max(oldContext.mergeInfo().estimatedMergeBytes(), size), - random.nextBoolean(), - TestUtil.nextInt(random, 1, 100))); - } else { - // Make a totally random IOContext - final IOContext context; - switch (random.nextInt(3)) { - case 0: - context = IOContext.DEFAULT; - break; - case 1: - context = IOContext.merge(new MergeInfo(randomNumDocs, size, true, -1)); - break; - case 2: - context = IOContext.flush(new FlushInfo(randomNumDocs, size)); - break; - default: - context = IOContext.DEFAULT; - } - return context; - } - } - - private static final QueryCache DEFAULT_QUERY_CACHE = IndexSearcher.getDefaultQueryCache(); - private static final QueryCachingPolicy DEFAULT_CACHING_POLICY = - IndexSearcher.getDefaultQueryCachingPolicy(); - private static final List queryCacheList = new ArrayList<>(); - - @Before - public void overrideTestDefaultQueryCache() { - // Make sure each test method has its own cache - overrideDefaultQueryCache(); - } - - @BeforeClass - public static void overrideDefaultQueryCache() { - // we need to reset the query cache in an @BeforeClass so that tests that - // instantiate an IndexSearcher in an @BeforeClass method use a fresh new cache - LRUQueryCache queryCacheTemp = - new LRUQueryCache(10000, 1 << 25, _ -> true, Float.POSITIVE_INFINITY); - queryCacheList.add(queryCacheTemp); - IndexSearcher.setDefaultQueryCache(queryCacheTemp); - IndexSearcher.setDefaultQueryCachingPolicy(MAYBE_CACHE_POLICY); - } - - @AfterClass - public static void resetDefaultQueryCache() { - IndexSearcher.setDefaultQueryCache(DEFAULT_QUERY_CACHE); - IndexSearcher.setDefaultQueryCachingPolicy(DEFAULT_CACHING_POLICY); - for (int i = 0; i < queryCacheList.size(); i++) { - try { - queryCacheList.get(i).close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - @BeforeClass - public static void setupCPUCoreCount() { - // Randomize core count so CMS varies its dynamic defaults, and this also "fixes" core - // count from the master seed so it will always be the same on reproduce: - int numCores = TestUtil.nextInt(random(), 1, 4); - System.setProperty( - ConcurrentMergeScheduler.DEFAULT_CPU_CORE_COUNT_PROPERTY, Integer.toString(numCores)); - } - - @AfterClass - public static void restoreCPUCoreCount() { - System.clearProperty(ConcurrentMergeScheduler.DEFAULT_CPU_CORE_COUNT_PROPERTY); - } - - private static ExecutorService executor; - - @BeforeClass - public static void setUpExecutorService() { - int threads = TestUtil.nextInt(random(), 1, 2); - executor = - new ThreadPoolExecutor( - threads, - threads, - 0L, - TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), - new NamedThreadFactory("LuceneTestCase")); - // uncomment to intensify LUCENE-3840 - // executor.prestartAllCoreThreads(); - if (VERBOSE) { - System.out.println("NOTE: Created shared ExecutorService with " + threads + " threads"); - } - } - - @AfterClass - public static void shutdownExecutorService() { - TestUtil.shutdownExecutorService(executor); - executor = null; - } - - /** Create a new searcher over the reader. This searcher might randomly use threads. */ - public static IndexSearcher newSearcher(IndexReader r) { - return newSearcher(r, true); - } - - /** Create a new searcher over the reader. This searcher might randomly use threads. */ - public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) { - return newSearcher(r, maybeWrap, true); - } - - /** - * Create a new searcher over the reader. This searcher might randomly use threads. if - * maybeWrap is true, this searcher might wrap the reader with one that returns null for - * getSequentialSubReaders. If wrapWithAssertions is true, this searcher might be an - * {@link AssertingIndexSearcher} instance. - */ - public static IndexSearcher newSearcher( - IndexReader r, boolean maybeWrap, boolean wrapWithAssertions) { - return newSearcher(r, maybeWrap, wrapWithAssertions, randomBoolean()); - } - - /** - * Create a new searcher over the reader. If - * maybeWrap is true, this searcher might wrap the reader with one that returns null for - * getSequentialSubReaders. If wrapWithAssertions is true, this searcher might be an - * {@link AssertingIndexSearcher} instance. The searcher will use threads if useThreads - * is set to true. - */ - public static IndexSearcher newSearcher( - IndexReader r, boolean maybeWrap, boolean wrapWithAssertions, boolean useThreads) { - if (useThreads) { - return newSearcher(r, maybeWrap, wrapWithAssertions, Concurrency.INTRA_SEGMENT); - } - return newSearcher(r, maybeWrap, wrapWithAssertions, Concurrency.NONE); - } - - /** What level of concurrency is supported by the searcher being created */ - public enum Concurrency { - /** No concurrency, meaning an executor won't be provided to the searcher */ - NONE, - /** - * Inter-segment concurrency, meaning an executor will be provided to the searcher and slices - * will be randomly created to concurrently search entire segments - */ - INTER_SEGMENT, - /** - * Intra-segment concurrency, meaning an executor will be provided to the searcher and slices - * will be randomly created to concurrently search segment partitions - */ - INTRA_SEGMENT - } - - public static IndexSearcher newSearcher( - IndexReader r, boolean maybeWrap, boolean wrapWithAssertions, Concurrency concurrency) { - Random random = random(); - if (concurrency == Concurrency.NONE) { - if (maybeWrap) { - try { - r = maybeWrapReader(r); - } catch (IOException e) { - Rethrow.rethrow(e); - } - } - // TODO: this whole check is a coverage hack, we should move it to tests for various - // filterreaders. - // ultimately whatever you do will be checkIndex'd at the end anyway. - if (random.nextInt(500) == 0 && r instanceof LeafReader) { - // TODO: not useful to check DirectoryReader (redundant with checkindex) - // but maybe sometimes run this on the other crazy readers maybeWrapReader creates? - try { - TestUtil.checkReader(r); - } catch (IOException e) { - Rethrow.rethrow(e); - } - } - final IndexSearcher ret; - if (wrapWithAssertions) { - ret = - random.nextBoolean() - ? new AssertingIndexSearcher(random, r) - : new AssertingIndexSearcher(random, r.getContext()); - } else { - ret = random.nextBoolean() ? new IndexSearcher(r) : new IndexSearcher(r.getContext()); - } - ret.setSimilarity(classEnvRule.similarity); - return ret; - } else { - final ExecutorService ex; - if (random.nextBoolean()) { - ex = null; - } else { - ex = executor; - if (VERBOSE) { - System.out.println("NOTE: newSearcher using shared ExecutorService"); - } - } - IndexSearcher ret; - int maxDocPerSlice = random.nextBoolean() ? 1 : 1 + random.nextInt(1000); - int maxSegmentsPerSlice = random.nextBoolean() ? 1 : 1 + random.nextInt(10); - if (wrapWithAssertions) { - if (random.nextBoolean()) { - ret = - new AssertingIndexSearcher(random, r, ex) { - @Override - protected LeafSlice[] slices(List leaves) { - return LuceneTestCase.slices( - leaves, maxDocPerSlice, maxSegmentsPerSlice, concurrency); - } - }; - } else { - ret = - new AssertingIndexSearcher(random, r.getContext(), ex) { - @Override - protected LeafSlice[] slices(List leaves) { - return LuceneTestCase.slices( - leaves, maxDocPerSlice, maxSegmentsPerSlice, concurrency); - } - }; - } - } else { - ret = - new IndexSearcher(r, ex) { - @Override - protected LeafSlice[] slices(List leaves) { - return LuceneTestCase.slices( - leaves, maxDocPerSlice, maxSegmentsPerSlice, concurrency); - } - }; - } - ret.setSimilarity(classEnvRule.similarity); - ret.setQueryCachingPolicy(MAYBE_CACHE_POLICY); - if (random().nextBoolean()) { - ret.setTimeout(() -> false); - } - return ret; - } - } - - /** - * Creates leaf slices according to the concurrency argument, that optionally leverage - * intra-segment concurrency by splitting segments into multiple partitions according to the - * maxDocsPerSlice argument. - */ - private static LeafSlice[] slices( - List leaves, - int maxDocsPerSlice, - int maxSegmentsPerSlice, - Concurrency concurrency) { - assert concurrency != Concurrency.NONE; - // Rarely test slices without partitions even though intra-segment concurrency is supported - return IndexSearcher.slices( - leaves, - maxDocsPerSlice, - maxSegmentsPerSlice, - concurrency == Concurrency.INTRA_SEGMENT && frequently()); - } - - /** - * Gets a resource from the test's classpath as {@link Path}. This method should only be used, if - * a real file is needed. To get a stream, code should prefer {@link #getDataInputStream(String)}. - */ - protected Path getDataPath(String name) throws IOException { - try { - return Paths.get( - IOUtils.requireResourceNonNull(this.getClass().getResource(name), name).toURI()); - } catch (URISyntaxException e) { - throw new AssertionError(e); - } - } - - /** Gets a resource from the test's classpath as {@link InputStream}. */ - protected InputStream getDataInputStream(String name) throws IOException { - return IOUtils.requireResourceNonNull(this.getClass().getResourceAsStream(name), name); - } - - // these hide the deprecated Assert.assertThat method - public static void assertThat(T actual, Matcher matcher) { - MatcherAssert.assertThat(actual, matcher); - } - - public static void assertThat(String reason, T actual, Matcher matcher) { - MatcherAssert.assertThat(reason, actual, matcher); - } - - public void assertReaderEquals(String info, IndexReader leftReader, IndexReader rightReader) - throws IOException { - assertReaderStatisticsEquals(info, leftReader, rightReader); - assertTermsEquals(info, leftReader, rightReader, true); - assertNormsEquals(info, leftReader, rightReader); - assertStoredFieldsEquals(info, leftReader, rightReader); - assertTermVectorsEquals(info, leftReader, rightReader); - assertDocValuesEquals(info, leftReader, rightReader); - assertDeletedDocsEquals(info, leftReader, rightReader); - assertFieldInfosEquals(info, leftReader, rightReader); - assertPointsEquals(info, leftReader, rightReader); - } - - /** checks that reader-level statistics are the same */ - public void assertReaderStatisticsEquals( - String info, IndexReader leftReader, IndexReader rightReader) throws IOException { - // Somewhat redundant: we never delete docs - assertEquals(info, leftReader.maxDoc(), rightReader.maxDoc()); - assertEquals(info, leftReader.numDocs(), rightReader.numDocs()); - assertEquals(info, leftReader.numDeletedDocs(), rightReader.numDeletedDocs()); - assertEquals(info, leftReader.hasDeletions(), rightReader.hasDeletions()); - } - - /** Fields api equivalency */ - public void assertTermsEquals( - String info, IndexReader leftReader, IndexReader rightReader, boolean deep) - throws IOException { - Set leftFields = new HashSet<>(FieldInfos.getIndexedFields(leftReader)); - Set rightFields = new HashSet<>(FieldInfos.getIndexedFields(rightReader)); - assertEquals(info, leftFields, rightFields); - - for (String field : leftFields) { - assertTermsEquals( - info, - leftReader, - MultiTerms.getTerms(leftReader, field), - MultiTerms.getTerms(rightReader, field), - deep); - } - } - - /** Terms api equivalency */ - public void assertTermsEquals( - String info, IndexReader leftReader, Terms leftTerms, Terms rightTerms, boolean deep) - throws IOException { - if (leftTerms == null || rightTerms == null) { - assertNull(info, leftTerms); - assertNull(info, rightTerms); - return; - } - assertTermsStatisticsEquals(info, leftTerms, rightTerms); - assertEquals("hasOffsets", leftTerms.hasOffsets(), rightTerms.hasOffsets()); - assertEquals("hasPositions", leftTerms.hasPositions(), rightTerms.hasPositions()); - assertEquals("hasPayloads", leftTerms.hasPayloads(), rightTerms.hasPayloads()); - - TermsEnum leftTermsEnum = leftTerms.iterator(); - TermsEnum rightTermsEnum = rightTerms.iterator(); - assertTermsEnumEquals(info, leftReader, leftTermsEnum, rightTermsEnum, true); - - assertTermsSeekingEquals(info, leftTerms, rightTerms); - - if (deep) { - int numIntersections = atLeast(3); - for (int i = 0; i < numIntersections; i++) { - String re = AutomatonTestUtil.randomRegexp(random()); - Automaton a = new RegExp(re, RegExp.NONE).toAutomaton(); - a = Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); - CompiledAutomaton automaton = new CompiledAutomaton(a); - if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) { - // TODO: test start term too - TermsEnum leftIntersection = leftTerms.intersect(automaton, null); - TermsEnum rightIntersection = rightTerms.intersect(automaton, null); - assertTermsEnumEquals(info, leftReader, leftIntersection, rightIntersection, rarely()); - } - } - } - } - - /** checks collection-level statistics on Terms */ - public void assertTermsStatisticsEquals(String info, Terms leftTerms, Terms rightTerms) - throws IOException { - assertEquals(info, leftTerms.getDocCount(), rightTerms.getDocCount()); - assertEquals(info, leftTerms.getSumDocFreq(), rightTerms.getSumDocFreq()); - assertEquals(info, leftTerms.getSumTotalTermFreq(), rightTerms.getSumTotalTermFreq()); - if (leftTerms.size() != -1 && rightTerms.size() != -1) { - assertEquals(info, leftTerms.size(), rightTerms.size()); - } - } - - /** - * checks the terms enum sequentially if deep is false, it does a 'shallow' test that doesnt go - * down to the docsenums - */ - public void assertTermsEnumEquals( - String info, - IndexReader leftReader, - TermsEnum leftTermsEnum, - TermsEnum rightTermsEnum, - boolean deep) - throws IOException { - BytesRef term; - PostingsEnum leftPositions = null; - PostingsEnum rightPositions = null; - PostingsEnum leftDocs = null; - PostingsEnum rightDocs = null; - - while ((term = leftTermsEnum.next()) != null) { - assertEquals(info, term, rightTermsEnum.next()); - assertTermStatsEquals(info, leftTermsEnum, rightTermsEnum); - if (deep) { - assertDocsAndPositionsEnumEquals( - info, - leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL)); - - assertPositionsSkippingEquals( - info, - leftReader, - leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL)); - - // with freqs: - assertDocsEnumEquals( - info, - leftDocs = leftTermsEnum.postings(leftDocs), - rightDocs = rightTermsEnum.postings(rightDocs), - true); - - // w/o freqs: - assertDocsEnumEquals( - info, - leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE), - rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE), - false); - - // with freqs: - assertDocsSkippingEquals( - info, - leftReader, - leftTermsEnum.docFreq(), - leftDocs = leftTermsEnum.postings(leftDocs), - rightDocs = rightTermsEnum.postings(rightDocs), - true); - - // w/o freqs: - assertDocsSkippingEquals( - info, - leftReader, - leftTermsEnum.docFreq(), - leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE), - rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE), - false); - } - } - assertNull(info, rightTermsEnum.next()); - } - - /** checks docs + freqs + positions + payloads, sequentially */ - public void assertDocsAndPositionsEnumEquals( - String info, PostingsEnum leftDocs, PostingsEnum rightDocs) throws IOException { - assertNotNull(leftDocs); - assertNotNull(rightDocs); - assertEquals(info, -1, leftDocs.docID()); - assertEquals(info, -1, rightDocs.docID()); - int docid; - while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - assertEquals(info, docid, rightDocs.nextDoc()); - int freq = leftDocs.freq(); - assertEquals(info, freq, rightDocs.freq()); - for (int i = 0; i < freq; i++) { - assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition()); - assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload()); - assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset()); - assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset()); - } - } - assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc()); - } - - /** checks docs + freqs, sequentially */ - public void assertDocsEnumEquals( - String info, PostingsEnum leftDocs, PostingsEnum rightDocs, boolean hasFreqs) - throws IOException { - if (leftDocs == null) { - assertNull(rightDocs); - return; - } - assertEquals(info, -1, leftDocs.docID()); - assertEquals(info, -1, rightDocs.docID()); - int docid; - while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - assertEquals(info, docid, rightDocs.nextDoc()); - if (hasFreqs) { - assertEquals(info, leftDocs.freq(), rightDocs.freq()); - } - } - assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc()); - } - - /** checks advancing docs */ - public void assertDocsSkippingEquals( - String info, - IndexReader leftReader, - int docFreq, - PostingsEnum leftDocs, - PostingsEnum rightDocs, - boolean hasFreqs) - throws IOException { - if (leftDocs == null) { - assertNull(rightDocs); - return; - } - int docid = -1; - int averageGap = leftReader.maxDoc() / (1 + docFreq); - int skipInterval = 16; - - while (true) { - if (random().nextBoolean()) { - // nextDoc() - docid = leftDocs.nextDoc(); - assertEquals(info, docid, rightDocs.nextDoc()); - } else { - // advance() - int skip = - docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap)); - docid = leftDocs.advance(skip); - assertEquals(info, docid, rightDocs.advance(skip)); - } - - if (docid == DocIdSetIterator.NO_MORE_DOCS) { - return; - } - if (hasFreqs) { - assertEquals(info, leftDocs.freq(), rightDocs.freq()); - } - } - } - - /** checks advancing docs + positions */ - public void assertPositionsSkippingEquals( - String info, - IndexReader leftReader, - int docFreq, - PostingsEnum leftDocs, - PostingsEnum rightDocs) - throws IOException { - if (leftDocs == null || rightDocs == null) { - assertNull(leftDocs); - assertNull(rightDocs); - return; - } - - int docid = -1; - int averageGap = leftReader.maxDoc() / (1 + docFreq); - int skipInterval = 16; - - while (true) { - if (random().nextBoolean()) { - // nextDoc() - docid = leftDocs.nextDoc(); - assertEquals(info, docid, rightDocs.nextDoc()); - } else { - // advance() - int skip = - docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap)); - docid = leftDocs.advance(skip); - assertEquals(info, docid, rightDocs.advance(skip)); - } - - if (docid == DocIdSetIterator.NO_MORE_DOCS) { - return; - } - int freq = leftDocs.freq(); - assertEquals(info, freq, rightDocs.freq()); - for (int i = 0; i < freq; i++) { - assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition()); - assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload()); - } - } - } - - private void assertTermsSeekingEquals(String info, Terms leftTerms, Terms rightTerms) - throws IOException { - - // just an upper bound - int numTests = atLeast(20); - Random random = random(); - - TermsEnum leftEnum = null; - - // collect this number of terms from the left side - HashSet tests = new HashSet<>(); - int numPasses = 0; - while (numPasses < 10 && tests.size() < numTests) { - leftEnum = leftTerms.iterator(); - BytesRef term; - while ((term = leftEnum.next()) != null) { - int code = random.nextInt(10); - if (code == 0) { - // the term - tests.add(BytesRef.deepCopyOf(term)); - } else if (code == 1) { - // truncated subsequence of term - term = BytesRef.deepCopyOf(term); - if (term.length > 0) { - // truncate it - term.length = random.nextInt(term.length); - } - } else if (code == 2) { - // term, but ensure a non-zero offset - byte[] newbytes = new byte[term.length + 5]; - System.arraycopy(term.bytes, term.offset, newbytes, 5, term.length); - tests.add(new BytesRef(newbytes, 5, term.length)); - } else if (code == 3) { - switch (random().nextInt(3)) { - case 0: - tests.add(new BytesRef()); // before the first term - break; - case 1: - tests.add(new BytesRef(new byte[] {(byte) 0xFF, (byte) 0xFF})); // past the last term - break; - case 2: - tests.add(new BytesRef(TestUtil.randomSimpleString(random()))); // random term - break; - default: - throw new AssertionError(); - } - } - } - numPasses++; - } - - TermsEnum rightEnum = rightTerms.iterator(); - - ArrayList shuffledTests = new ArrayList<>(tests); - Collections.shuffle(shuffledTests, random); - - for (BytesRef b : shuffledTests) { - if (rarely()) { - // make new enums - leftEnum = leftTerms.iterator(); - rightEnum = rightTerms.iterator(); - } - - final boolean seekExact = random().nextBoolean(); - - if (seekExact) { - assertEquals(info, leftEnum.seekExact(b), rightEnum.seekExact(b)); - } else { - SeekStatus leftStatus = leftEnum.seekCeil(b); - SeekStatus rightStatus = rightEnum.seekCeil(b); - assertEquals(info, leftStatus, rightStatus); - if (leftStatus != SeekStatus.END) { - assertEquals(info, leftEnum.term(), rightEnum.term()); - assertTermStatsEquals(info, leftEnum, rightEnum); - } - } - } - } - - /** checks term-level statistics */ - public void assertTermStatsEquals(String info, TermsEnum leftTermsEnum, TermsEnum rightTermsEnum) - throws IOException { - assertEquals(info, leftTermsEnum.docFreq(), rightTermsEnum.docFreq()); - assertEquals(info, leftTermsEnum.totalTermFreq(), rightTermsEnum.totalTermFreq()); - } - - /** checks that norms are the same across all fields */ - public void assertNormsEquals(String info, IndexReader leftReader, IndexReader rightReader) - throws IOException { - Set leftFields = new HashSet<>(FieldInfos.getIndexedFields(leftReader)); - Set rightFields = new HashSet<>(FieldInfos.getIndexedFields(rightReader)); - assertEquals(info, leftFields, rightFields); - - for (String field : leftFields) { - NumericDocValues leftNorms = MultiDocValues.getNormValues(leftReader, field); - NumericDocValues rightNorms = MultiDocValues.getNormValues(rightReader, field); - if (leftNorms != null && rightNorms != null) { - assertDocValuesEquals(info, leftReader.maxDoc(), leftNorms, rightNorms); - } else { - assertNull(info, leftNorms); - assertNull(info, rightNorms); - } - } - } - - /** checks that stored fields of all documents are the same */ - public void assertStoredFieldsEquals(String info, IndexReader leftReader, IndexReader rightReader) - throws IOException { - assert leftReader.maxDoc() == rightReader.maxDoc(); - StoredFields leftStoredFields = leftReader.storedFields(); - StoredFields rightStoredFields = rightReader.storedFields(); - for (int i = 0; i < leftReader.maxDoc(); i++) { - Document leftDoc = leftStoredFields.document(i); - Document rightDoc = rightStoredFields.document(i); - - // TODO: I think this is bogus because we don't document what the order should be - // from these iterators, etc. I think the codec/IndexReader should be free to order this stuff - // in whatever way it wants (e.g. maybe it packs related fields together or something) - // To fix this, we sort the fields in both documents by name, but - // we still assume that all instances with same name are in order: - Comparator comp = Comparator.comparing(IndexableField::name); - List leftFields = new ArrayList<>(leftDoc.getFields()); - List rightFields = new ArrayList<>(rightDoc.getFields()); - leftFields.sort(comp); - rightFields.sort(comp); - - Iterator leftIterator = leftFields.iterator(); - Iterator rightIterator = rightFields.iterator(); - while (leftIterator.hasNext()) { - assertTrue(info, rightIterator.hasNext()); - assertStoredFieldEquals(info, leftIterator.next(), rightIterator.next()); - } - assertFalse(info, rightIterator.hasNext()); - } - } - - /** checks that two stored fields are equivalent */ - public void assertStoredFieldEquals( - String info, IndexableField leftField, IndexableField rightField) { - assertEquals(info, leftField.name(), rightField.name()); - assertEquals(info, leftField.binaryValue(), rightField.binaryValue()); - assertEquals(info, leftField.stringValue(), rightField.stringValue()); - assertEquals(info, leftField.numericValue(), rightField.numericValue()); - // TODO: should we check the FT at all? - } - - /** checks that term vectors across all fields are equivalent */ - public void assertTermVectorsEquals(String info, IndexReader leftReader, IndexReader rightReader) - throws IOException { - assert leftReader.maxDoc() == rightReader.maxDoc(); - TermVectors leftVectors = leftReader.termVectors(); - TermVectors rightVectors = rightReader.termVectors(); - for (int i = 0; i < leftReader.maxDoc(); i++) { - Fields leftFields = leftVectors.get(i); - Fields rightFields = rightVectors.get(i); - - // Fields could be null if there are no postings, - // but then it must be null for both - if (leftFields == null || rightFields == null) { - assertNull(info, leftFields); - assertNull(info, rightFields); - return; - } - if (leftFields.size() != -1 && rightFields.size() != -1) { - assertEquals(info, leftFields.size(), rightFields.size()); - } - - Iterator leftEnum = leftFields.iterator(); - Iterator rightEnum = rightFields.iterator(); - while (leftEnum.hasNext()) { - String field = leftEnum.next(); - assertEquals(info, field, rightEnum.next()); - assertTermsEquals( - info, leftReader, leftFields.terms(field), rightFields.terms(field), rarely()); - } - assertFalse(rightEnum.hasNext()); - } - } - - private static Set getDVFields(IndexReader reader) { - Set fields = new HashSet<>(); - for (FieldInfo fi : FieldInfos.getMergedFieldInfos(reader)) { - if (fi.getDocValuesType() != DocValuesType.NONE) { - fields.add(fi.name); - } - } - - return fields; - } - - /** checks that docvalues across all fields are equivalent */ - public void assertDocValuesEquals(String info, IndexReader leftReader, IndexReader rightReader) - throws IOException { - Set leftFields = getDVFields(leftReader); - Set rightFields = getDVFields(rightReader); - assertEquals(info, leftFields, rightFields); - - for (String field : leftFields) { - // TODO: clean this up... very messy - { - NumericDocValues leftValues = MultiDocValues.getNumericValues(leftReader, field); - NumericDocValues rightValues = MultiDocValues.getNumericValues(rightReader, field); - if (leftValues != null && rightValues != null) { - assertDocValuesEquals(info, leftReader.maxDoc(), leftValues, rightValues); - } else { - assertTrue( - info + ": left numeric doc values for field=\"" + field + "\" are not null", - leftValues == null || leftValues.nextDoc() == NO_MORE_DOCS); - assertTrue( - info + ": right numeric doc values for field=\"" + field + "\" are not null", - rightValues == null || rightValues.nextDoc() == NO_MORE_DOCS); - } - } - - { - BinaryDocValues leftValues = MultiDocValues.getBinaryValues(leftReader, field); - BinaryDocValues rightValues = MultiDocValues.getBinaryValues(rightReader, field); - if (leftValues != null && rightValues != null) { - while (true) { - int docID = leftValues.nextDoc(); - assertEquals(docID, rightValues.nextDoc()); - if (docID == NO_MORE_DOCS) { - break; - } - assertEquals(leftValues.binaryValue(), rightValues.binaryValue()); - } - } else { - assertTrue(info, leftValues == null || leftValues.nextDoc() == NO_MORE_DOCS); - assertTrue(info, rightValues == null || rightValues.nextDoc() == NO_MORE_DOCS); - } - } - - { - SortedDocValues leftValues = MultiDocValues.getSortedValues(leftReader, field); - SortedDocValues rightValues = MultiDocValues.getSortedValues(rightReader, field); - if (leftValues != null && rightValues != null) { - // numOrds - assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount()); - // ords - for (int i = 0; i < leftValues.getValueCount(); i++) { - final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(i)); - final BytesRef right = rightValues.lookupOrd(i); - assertEquals(info, left, right); - } - // bytes - while (true) { - int docID = leftValues.nextDoc(); - assertEquals(docID, rightValues.nextDoc()); - if (docID == NO_MORE_DOCS) { - break; - } - final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(leftValues.ordValue())); - final BytesRef right = rightValues.lookupOrd(rightValues.ordValue()); - assertEquals(info, left, right); - } - } else { - assertNull(info, leftValues); - assertNull(info, rightValues); - } - } - - { - SortedSetDocValues leftValues = MultiDocValues.getSortedSetValues(leftReader, field); - SortedSetDocValues rightValues = MultiDocValues.getSortedSetValues(rightReader, field); - if (leftValues != null && rightValues != null) { - // numOrds - assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount()); - // ords - for (int i = 0; i < leftValues.getValueCount(); i++) { - final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(i)); - final BytesRef right = rightValues.lookupOrd(i); - assertEquals(info, left, right); - } - // ord lists - while (true) { - int docID = leftValues.nextDoc(); - assertEquals(docID, rightValues.nextDoc()); - if (docID == NO_MORE_DOCS) { - break; - } - assertEquals(info, leftValues.docValueCount(), rightValues.docValueCount()); - for (int i = 0; i < leftValues.docValueCount(); i++) { - assertEquals(info, leftValues.nextOrd(), rightValues.nextOrd()); - } - } - } else { - assertNull(info, leftValues); - assertNull(info, rightValues); - } - } - - { - SortedNumericDocValues leftValues = - MultiDocValues.getSortedNumericValues(leftReader, field); - SortedNumericDocValues rightValues = - MultiDocValues.getSortedNumericValues(rightReader, field); - if (leftValues != null && rightValues != null) { - while (true) { - int docID = leftValues.nextDoc(); - assertEquals(docID, rightValues.nextDoc()); - if (docID == NO_MORE_DOCS) { - break; - } - assertEquals(info, leftValues.docValueCount(), rightValues.docValueCount()); - for (int j = 0; j < leftValues.docValueCount(); j++) { - assertEquals(info, leftValues.nextValue(), rightValues.nextValue()); - } - } - } else { - assertNull(info, leftValues); - assertNull(info, rightValues); - } - } - } - } - - public void assertDocValuesEquals( - String info, int num, NumericDocValues leftDocValues, NumericDocValues rightDocValues) - throws IOException { - assertNotNull(info, leftDocValues); - assertNotNull(info, rightDocValues); - while (true) { - int leftDocID = leftDocValues.nextDoc(); - int rightDocID = rightDocValues.nextDoc(); - assertEquals(leftDocID, rightDocID); - if (leftDocID == NO_MORE_DOCS) { - return; - } - assertEquals(leftDocValues.longValue(), rightDocValues.longValue()); - } - } - - // TODO: this is kinda stupid, we don't delete documents in the test. - public void assertDeletedDocsEquals(String info, IndexReader leftReader, IndexReader rightReader) - throws IOException { - assert leftReader.numDeletedDocs() == rightReader.numDeletedDocs(); - Bits leftBits = MultiBits.getLiveDocs(leftReader); - Bits rightBits = MultiBits.getLiveDocs(rightReader); - - if (leftBits == null || rightBits == null) { - assertNull(info, leftBits); - assertNull(info, rightBits); - return; - } - - assert leftReader.maxDoc() == rightReader.maxDoc(); - assertEquals(info, leftBits.length(), rightBits.length()); - for (int i = 0; i < leftReader.maxDoc(); i++) { - assertEquals(info, leftBits.get(i), rightBits.get(i)); - } - } - - public void assertFieldInfosEquals(String info, IndexReader leftReader, IndexReader rightReader) - throws IOException { - FieldInfos leftInfos = FieldInfos.getMergedFieldInfos(leftReader); - FieldInfos rightInfos = FieldInfos.getMergedFieldInfos(rightReader); - - // TODO: would be great to verify more than just the names of the fields! - TreeSet left = new TreeSet<>(); - TreeSet right = new TreeSet<>(); - - for (FieldInfo fi : leftInfos) { - left.add(fi.name); - } - - for (FieldInfo fi : rightInfos) { - right.add(fi.name); - } - - assertEquals(info, left, right); - } - - // naive silly memory heavy uninversion!! maps docID -> packed values (a Set because a given doc - // can be multi-valued) - private Map> uninvert(String fieldName, IndexReader reader) - throws IOException { - final Map> docValues = new HashMap<>(); - for (LeafReaderContext ctx : reader.leaves()) { - - PointValues points = ctx.reader().getPointValues(fieldName); - if (points == null) { - continue; - } - - points.intersect( - new PointValues.IntersectVisitor() { - @Override - public void visit(int docID) { - throw new UnsupportedOperationException(); - } - - @Override - public void visit(int docID, byte[] packedValue) throws IOException { - int topDocID = ctx.docBase + docID; - if (docValues.containsKey(topDocID) == false) { - docValues.put(topDocID, new HashSet<>()); - } - docValues.get(topDocID).add(new BytesRef(packedValue.clone())); - } - - @Override - public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - // We pretend our query shape is so hairy that it crosses every single cell: - return PointValues.Relation.CELL_CROSSES_QUERY; - } - }); - } - - return docValues; - } - - public void assertPointsEquals(String info, IndexReader leftReader, IndexReader rightReader) - throws IOException { - FieldInfos fieldInfos1 = FieldInfos.getMergedFieldInfos(leftReader); - FieldInfos fieldInfos2 = FieldInfos.getMergedFieldInfos(rightReader); - for (FieldInfo fieldInfo1 : fieldInfos1) { - if (fieldInfo1.getPointDimensionCount() != 0) { - FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldInfo1.name); - // same data dimension count? - assertEquals( - info, fieldInfo2.getPointDimensionCount(), fieldInfo2.getPointDimensionCount()); - // same index dimension count? - assertEquals( - info, - fieldInfo2.getPointIndexDimensionCount(), - fieldInfo2.getPointIndexDimensionCount()); - // same bytes per dimension? - assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo2.getPointNumBytes()); - - assertEquals( - info + " field=" + fieldInfo1.name, - uninvert(fieldInfo1.name, leftReader), - uninvert(fieldInfo1.name, rightReader)); - } - } - - // make sure FieldInfos2 doesn't have any point fields that FieldInfo1 didn't have - for (FieldInfo fieldInfo2 : fieldInfos2) { - if (fieldInfo2.getPointDimensionCount() != 0) { - FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldInfo2.name); - // same data dimension count? - assertEquals( - info, fieldInfo2.getPointDimensionCount(), fieldInfo1.getPointDimensionCount()); - // same index dimension count? - assertEquals( - info, - fieldInfo2.getPointIndexDimensionCount(), - fieldInfo1.getPointIndexDimensionCount()); - // same bytes per dimension? - assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo1.getPointNumBytes()); - - // we don't need to uninvert and compare here ... we did that in the first loop above - } - } - } - - private static final StackWalker SW_NO_METHODS = StackWalker.getInstance(Option.DROP_METHOD_INFO), - SW_WITH_METHODS = StackWalker.getInstance(); - - /** Inspects stack trace to figure out if a method of a specific class called us. */ - public static boolean callStackContains(Class clazz, String methodName) { - final String className = clazz.getName(); - return SW_WITH_METHODS.walk( - s -> - s.skip(1) // exclude this utility method - .anyMatch( - f -> - className.equals(f.getClassName()) - && methodName.equals(f.getMethodName()))); - } - - /** - * Inspects stack trace to figure out if one of the given method names (no class restriction) - * called us. - */ - public static boolean callStackContainsAnyOf(String... methodNames) { - return SW_WITH_METHODS.walk( - s -> - s.skip(1) // exclude this utility method - .map(StackFrame::getMethodName) - .anyMatch(Set.of(methodNames)::contains)); - } - - /** Inspects stack trace if the given class called us. */ - public static boolean callStackContains(Class clazz) { - return SW_NO_METHODS.walk( - s -> - s.skip(1) // exclude this utility method - .map(StackFrame::getClassName) - .anyMatch(clazz.getName()::equals)); - } - - /** A runnable that can throw any checked exception. */ - @FunctionalInterface - public interface ThrowingRunnable { - void run() throws Throwable; - } - - /** A {@link java.util.function.Consumer} that can throw any checked exception. */ - @FunctionalInterface - public interface ThrowingConsumer { - void accept(T t) throws Exception; - } - - /** Checks a specific exception class is thrown by the given runnable, and returns it. */ - public static T expectThrows( - Class expectedType, ThrowingRunnable runnable) { - return expectThrows( - expectedType, - "Expected exception " + expectedType.getSimpleName() + " but no exception was thrown", - runnable); - } - - /** Checks a specific exception class is thrown by the given runnable, and returns it. */ - public static T expectThrows( - Class expectedType, String noExceptionMessage, ThrowingRunnable runnable) { - final Throwable thrown = _expectThrows(Collections.singletonList(expectedType), runnable); - if (expectedType.isInstance(thrown)) { - return expectedType.cast(thrown); - } - if (null == thrown) { - throw new AssertionFailedError(noExceptionMessage); - } - AssertionFailedError assertion = - new AssertionFailedError( - "Unexpected exception type, expected " - + expectedType.getSimpleName() - + " but got " - + thrown); - assertion.initCause(thrown); - throw assertion; - } - - /** Checks a specific exception class is thrown by the given runnable, and returns it. */ - public static T expectThrowsAnyOf( - List> expectedTypes, ThrowingRunnable runnable) { - if (expectedTypes.isEmpty()) { - throw new AssertionError("At least one expected exception type is required?"); - } - - final Throwable thrown = _expectThrows(expectedTypes, runnable); - if (null != thrown) { - for (Class expectedType : expectedTypes) { - if (expectedType.isInstance(thrown)) { - return expectedType.cast(thrown); - } - } - } - - List exceptionTypes = expectedTypes.stream().map(Class::getSimpleName).toList(); - - if (thrown != null) { - AssertionFailedError assertion = - new AssertionFailedError( - "Unexpected exception type, expected any of " - + exceptionTypes - + " but got: " - + thrown); - assertion.initCause(thrown); - throw assertion; - } else { - throw new AssertionFailedError( - "Expected any of the following exception types: " - + exceptionTypes - + " but no exception was thrown."); - } - } - - /** - * Checks that specific wrapped and outer exception classes are thrown by the given runnable, and - * returns the wrapped exception. - */ - public static TW expectThrows( - Class expectedOuterType, Class expectedWrappedType, ThrowingRunnable runnable) { - final Throwable thrown = _expectThrows(Collections.singletonList(expectedOuterType), runnable); - if (null == thrown) { - throw new AssertionFailedError( - "Expected outer exception " - + expectedOuterType.getSimpleName() - + " but no exception was thrown."); - } - if (expectedOuterType.isInstance(thrown)) { - Throwable cause = thrown.getCause(); - if (expectedWrappedType.isInstance(cause)) { - return expectedWrappedType.cast(cause); - } else { - AssertionFailedError assertion = - new AssertionFailedError( - "Unexpected wrapped exception type, expected " - + expectedWrappedType.getSimpleName() - + " but got: " - + cause); - assertion.initCause(thrown); - throw assertion; - } - } - AssertionFailedError assertion = - new AssertionFailedError( - "Unexpected outer exception type, expected " - + expectedOuterType.getSimpleName() - + " but got: " - + thrown); - assertion.initCause(thrown); - throw assertion; - } - - /** - * Checks that one of the specified wrapped and outer exception classes are thrown by the given - * runnable, and returns the outer exception. - * - *

This method accepts outer exceptions with no wrapped exception; an empty list of expected - * wrapped exception types indicates no wrapped exception. - */ - public static TO expectThrowsAnyOf( - LinkedHashMap, List>> expectedOuterToWrappedTypes, - ThrowingRunnable runnable) { - final List> outerClasses = - new ArrayList<>(expectedOuterToWrappedTypes.keySet()); - final Throwable thrown = _expectThrows(outerClasses, runnable); - - if (null == thrown) { - List outerTypes = outerClasses.stream().map(Class::getSimpleName).toList(); - throw new AssertionFailedError( - "Expected any of the following outer exception types: " - + outerTypes - + " but no exception was thrown."); - } - for (Map.Entry, List>> entry : - expectedOuterToWrappedTypes.entrySet()) { - Class expectedOuterType = entry.getKey(); - List> expectedWrappedTypes = entry.getValue(); - Throwable cause = thrown.getCause(); - if (expectedOuterType.isInstance(thrown)) { - if (expectedWrappedTypes.isEmpty()) { - return null; // no wrapped exception - } else { - for (Class expectedWrappedType : expectedWrappedTypes) { - if (expectedWrappedType.isInstance(cause)) { - return expectedOuterType.cast(thrown); - } - } - List wrappedTypes = - expectedWrappedTypes.stream().map(Class::getSimpleName).toList(); - AssertionFailedError assertion = - new AssertionFailedError( - "Unexpected wrapped exception type, expected one of " - + wrappedTypes - + " but got: " - + cause); - assertion.initCause(thrown); - throw assertion; - } - } - } - List outerTypes = outerClasses.stream().map(Class::getSimpleName).toList(); - AssertionFailedError assertion = - new AssertionFailedError( - "Unexpected outer exception type, expected one of " - + outerTypes - + " but got: " - + thrown); - assertion.initCause(thrown); - throw assertion; - } - - /** - * Helper method for {@link #expectThrows} and {@link #expectThrowsAnyOf} that takes care of - * propagating any {@link AssertionError} or {@link AssumptionViolatedException} instances thrown - * if and only if they are super classes of the expectedTypes. Otherwise simply - * returns any {@link Throwable} thrown, regardless of type, or null if the runnable - * completed w/o error. - */ - private static Throwable _expectThrows( - List> expectedTypes, ThrowingRunnable runnable) { - - try { - runnable.run(); - } catch (AssertionError | AssumptionViolatedException ae) { - for (Class expectedType : expectedTypes) { - if (expectedType.isInstance(ae)) { // user is expecting this type explicitly - return ae; - } - } - throw ae; - } catch (Throwable e) { - return e; - } - return null; - } - - /** - * Returns true if the file exists (can be opened), false if it cannot be opened, and (unlike - * Java's File.exists) throws IOException if there's some unexpected error. - */ - public static boolean slowFileExists(Directory dir, String fileName) throws IOException { - try { - dir.openInput(fileName, IOContext.READONCE).close(); - return true; - } catch (NoSuchFileException | FileNotFoundException _) { - return false; - } - } - - /** - * Creates an empty, temporary folder (when the name of the folder is of no importance). - * - * @see #createTempDir(String) - */ - public static Path createTempDir() { - return createTempDir("tempDir"); - } - - /** - * Creates an empty, temporary folder with the given name prefix. - * - *

The folder will be automatically removed after the test class completes successfully. The - * test should close any file handles that would prevent the folder from being removed. - */ - public static Path createTempDir(String prefix) { - return tempFilesCleanupRule.createTempDir(prefix); - } - - /** - * Creates an empty file with the given prefix and suffix. - * - *

The file will be automatically removed after the test class completes successfully. The test - * should close any file handles that would prevent the folder from being removed. - */ - public static Path createTempFile(String prefix, String suffix) throws IOException { - return tempFilesCleanupRule.createTempFile(prefix, suffix); - } - - /** - * Creates an empty temporary file. - * - * @see #createTempFile(String, String) - */ - public static Path createTempFile() throws IOException { - return createTempFile("tempFile", ".tmp"); - } - - /** - * Returns a set of JVM arguments to fork a JVM with the same class or module path (including any - * associated JVM options). The returned value may be empty. This method may throw an assertion - * error if fork options cannot be reliably acquired (at the moment they are collected and passed - * as an external file in gradle scripts). - * - *

JVM forking is strongly discouraged as it makes test slower and more resource-hungry. - * Consider all alternatives first. - */ - public static List getJvmForkArguments() throws IOException { - String forkArgsFile = System.getProperty("tests.jvmForkArgsFile"); - Path forkArgsPath; - if (forkArgsFile == null || !Files.isRegularFile(forkArgsPath = Paths.get(forkArgsFile))) { - throw new AssertionError("JVM fork arguments are not present."); - } - - return Files.readAllLines(forkArgsPath, StandardCharsets.UTF_8); - } - - /** - * Compares two strings with a collator, also looking to see if the strings are impacted by jdk - * bugs. may not avoid all jdk bugs in tests. see https://bugs.openjdk.java.net/browse/JDK-8071862 - */ - @SuppressForbidden(reason = "dodges JDK-8071862") - public static int collate(Collator collator, String s1, String s2) { - int v1 = collator.compare(s1, s2); - int v2 = collator.getCollationKey(s1).compareTo(collator.getCollationKey(s2)); - // if collation keys don't really respect collation order, things are screwed. - assumeTrue("hit JDK collator bug", Integer.signum(v1) == Integer.signum(v2)); - return v1; - } - - /** Ensures that the MergePolicy has sane values for tests that test with lots of documents. */ - protected static IndexWriterConfig ensureSaneIWCOnNightly(IndexWriterConfig conf) { - if (LuceneTestCase.TEST_NIGHTLY) { - // newIWConfig makes smallish max seg size, which - // results in tons and tons of segments for this test - // when run nightly: - MergePolicy mp = conf.getMergePolicy(); - if (mp instanceof TieredMergePolicy) { - ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.); - } else if (mp instanceof LogByteSizeMergePolicy) { - ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.); - } else if (mp instanceof LogMergePolicy) { - ((LogMergePolicy) mp).setMaxMergeDocs(100000); - } - // when running nightly, merging can still have crazy parameters, - // and might use many per-field codecs. turn on CFS for IW flushes - // and ensure CFS ratio is reasonable to keep it contained. - conf.setUseCompoundFile(true); - } - return conf; - } - - /** - * Creates a {@link BytesRef} holding UTF-8 bytes for the incoming String, that sometimes uses a - * non-zero {@code offset}, and non-zero end-padding, to tickle latent bugs that fail to look at - * {@code BytesRef.offset}. - */ - public static BytesRef newBytesRef(String s) { - return newBytesRef(s.getBytes(StandardCharsets.UTF_8)); - } - - /** - * Creates a copy of the incoming {@link BytesRef} that sometimes uses a non-zero {@code offset}, - * and non-zero end-padding, to tickle latent bugs that fail to look at {@code BytesRef.offset}. - */ - public static BytesRef newBytesRef(BytesRef b) { - assert b.isValid(); - return newBytesRef(b.bytes, b.offset, b.length); - } - - /** - * Creates a random BytesRef from the incoming bytes that sometimes uses a non-zero {@code - * offset}, and non-zero end-padding, to tickle latent bugs that fail to look at {@code - * BytesRef.offset}. - */ - public static BytesRef newBytesRef(byte[] b) { - return newBytesRef(b, 0, b.length); - } - - /** - * Creates a random empty BytesRef that sometimes uses a non-zero {@code offset}, and non-zero - * end-padding, to tickle latent bugs that fail to look at {@code BytesRef.offset}. - */ - public static BytesRef newBytesRef() { - return newBytesRef(new byte[0], 0, 0); - } - - /** - * Creates a random empty BytesRef, with at least the requested length of bytes free, that - * sometimes uses a non-zero {@code offset}, and non-zero end-padding, to tickle latent bugs that - * fail to look at {@code BytesRef.offset}. - */ - public static BytesRef newBytesRef(int byteLength) { - return newBytesRef(new byte[byteLength], 0, byteLength); - } - - /** - * Creates a copy of the incoming bytes slice that sometimes uses a non-zero {@code offset}, and - * non-zero end-padding, to tickle latent bugs that fail to look at {@code BytesRef.offset}. - */ - public static BytesRef newBytesRef(byte[] bytesIn, int offset, int length) { - // System.out.println("LTC.newBytesRef! bytesIn.length=" + bytesIn.length + " offset=" + offset - // + " length=" + length); - - assert bytesIn.length >= offset + length - : "got offset=" + offset + " length=" + length + " bytesIn.length=" + bytesIn.length; - - // randomly set a non-zero offset - int startOffset; - if (random().nextBoolean()) { - startOffset = RandomNumbers.randomIntBetween(random(), 1, 20); - } else { - startOffset = 0; - } - - // also randomly set an end padding: - int endPadding; - if (random().nextBoolean()) { - endPadding = RandomNumbers.randomIntBetween(random(), 1, 20); - } else { - endPadding = 0; - } - - byte[] bytes = new byte[startOffset + length + endPadding]; - - System.arraycopy(bytesIn, offset, bytes, startOffset, length); - // System.out.println("LTC: return bytes.length=" + bytes.length + " startOffset=" + - // startOffset + " length=" + length); - - BytesRef it = new BytesRef(bytes, startOffset, length); - assert it.isValid(); - - if (RandomNumbers.randomIntBetween(random(), 1, 17) == 7) { - // try to ferret out bugs in this method too! - return newBytesRef(it.bytes, it.offset, it.length); - } - - return it; - } - - private static boolean supportsVectorEncoding( - KnnVectorsFormat format, VectorEncoding vectorEncoding) { - if (format instanceof HnswBitVectorsFormat) { - // special case, this only supports BYTE - return vectorEncoding == VectorEncoding.BYTE; - } - return true; - } - - private static boolean supportsVectorSearch(KnnVectorsFormat format) { - return (format instanceof FlatVectorsFormat) == false; - } - - protected static KnnVectorsFormat randomVectorFormat(VectorEncoding vectorEncoding) { - List availableFormats = - KnnVectorsFormat.availableKnnVectorsFormats().stream() - .map(KnnVectorsFormat::forName) - .filter(format -> supportsVectorEncoding(format, vectorEncoding)) - .filter(format -> supportsVectorSearch(format)) - .toList(); - return RandomPicks.randomFrom(random(), availableFormats); - } - - /** - * This is a test merge scheduler that will always use the intra merge executor to ensure we test - * it. - */ - static class TestConcurrentMergeScheduler extends ConcurrentMergeScheduler { - @Override - public Executor getIntraMergeExecutor(MergePolicy.OneMerge merge) { - assert intraMergeExecutor != null : "scaledExecutor is not initialized"; - // Always do the intra merge executor to ensure we test it - return intraMergeExecutor; - } - } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCaseJupiter.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCaseJupiter.java new file mode 100644 index 000000000000..e0be5b494d97 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCaseJupiter.java @@ -0,0 +1,633 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.tests.util; + +import com.carrotsearch.randomizedtesting.jupiter.DetectThreadLeaks; +import com.carrotsearch.randomizedtesting.jupiter.Randomized; +import com.carrotsearch.randomizedtesting.jupiter.RandomizedContext; +import com.carrotsearch.randomizedtesting.jupiter.SystemThreadFilter; +import java.io.Closeable; +import java.io.PrintStream; +import java.util.ArrayDeque; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.util.IOUtils; +import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.DynamicTestInvocationContext; +import org.junit.jupiter.api.extension.ExecutableInvoker; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.InvocationInterceptor; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.extension.TestWatcher; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.platform.commons.support.AnnotationSupport; +import org.junit.platform.commons.support.HierarchyTraversalMode; +import org.junit.platform.commons.support.ModifierSupport; +import org.junit.platform.commons.support.ReflectionSupport; +import org.opentest4j.TestAbortedException; + +/// Base class for all Lucene unit tests (JUnit5/ Jupiter variant). +/// +/// ## Class and instance setup +/// +/// The preferred way to specify class (suite-level) setup/cleanup is to use static methods +/// annotated with [org.junit.jupiter.api.BeforeAll] and [org.junit.jupiter.api.AfterAll]. +/// **Do not use static initializers (including complex final field initializers).** +/// +/// For instance-level setup, use [org.junit.jupiter.api.BeforeEach] and +/// [org.junit.jupiter.api.AfterEach] annotated methods. +/// +/// ## Specifying test cases +/// +/// Any method of specifying JUnit jupiter tests will work. The most common way would therefore be: +/// ```java +/// @Test +/// public void testMethod(Random random) {} +/// ``` +/// +/// Note the (optional) [Random] argument - this is automatically populated for each test. +/// +/// ## Randomized execution and test facilities +/// +/// [LuceneTestCaseJupiter] uses the [Randomized] extension to support component randomization. +/// A [Random] can be automatically injected in the test (or any junit5 callback) as a parameter. +/// Tests should be fully reproducible for the same initial seed +/// (assuming no race conditions between threads +/// etc.). The initial seed for a test case is reported in many ways: +/// +/// - logged from the gradle build, +/// - inserted as a synthetic stack frame in any exceptions. +/// +/* +TODO: port the remaining infrastructure bits from LuceneTestCase: +- there are class rules and test rules that are still only on junit4 side +TestRuleRestoreSystemProperties +TestRuleLimitSysouts +NoInstanceHooksOverridesRule (?) +ignoreAfterMaxFailures +*/ +@Randomized +@DetectThreadLeaks(scope = DetectThreadLeaks.Scope.SUITE) +@DetectThreadLeaks.LingerTime(millis = 20_000) +@DetectThreadLeaks.ExcludeThreads({SystemThreadFilter.class, IsSystemThread.class}) +@Timeout(value = 2, unit = TimeUnit.HOURS) +@Execution( + value = ExecutionMode.SAME_THREAD, + reason = "single-threaded for backward compatibility.") +@ExtendWith(EnsureSequentialExecution.class) +@TestMethodOrder(CustomMethodOrderer.class) +public abstract non-sealed class LuceneTestCaseJupiter extends LuceneTestCaseParent { + + static final class PerThreadRandom implements BeforeAfterCallback { + private final ConcurrentHashMap perThreadRandoms = new ConcurrentHashMap<>(); + private final Supplier supplier; + + PerThreadRandom(Supplier randomSupplier) { + this.supplier = randomSupplier; + } + + @Override + public void after() throws Exception { + IOUtils.close( + perThreadRandoms.values().stream() + .filter(v -> v instanceof Closeable) + .map(v -> (Closeable) v) + .toList()); + } + + Random get() { + return perThreadRandoms.computeIfAbsent(Thread.currentThread(), _ -> supplier.get()); + } + } + + static final class OrderedBeforeAfterCallbacks implements BeforeAfterCallback { + final List callbacks; + final ArrayDeque executed = new ArrayDeque<>(); + + OrderedBeforeAfterCallbacks(List callbacks) { + this.callbacks = callbacks; + } + + @Override + public void before() throws Exception { + assert executed.isEmpty(); + for (var c : callbacks) { + c.before(); + executed.addLast(c); + } + } + + @Override + public void after() throws Exception { + Throwable t = null; + while (!executed.isEmpty()) { + var c = executed.removeLast(); + try { + c.after(); + } catch (Throwable ex) { + if (t == null) { + t = ex; + } else { + t.addSuppressed(ex); + } + } + } + + if (t != null) { + if (t instanceof Exception ex) { + throw ex; + } else if (t instanceof Error err) { + throw err; + } else /* only theoretically possible? */ { + throw new RuntimeException(t); + } + } + } + } + + /// Tracks whether any test in the current suite had a failure. + /// Registered before [ClassLevelCallbackChain] so its state is available during suite teardown. + /// We plug into multiple jupiter extensions, hoping they will be sufficient to detect failure + // state. + static final class SuiteFailureTracker + implements AfterAllCallback, + BeforeAllCallback, + TestWatcher, + SuiteFailureState, + InvocationInterceptor { + private static final ExtensionContext.Namespace NAMESPACE = + ExtensionContext.Namespace.create(SuiteFailureTracker.class); + + private volatile boolean hadFailures; + + @Override + public void beforeAll(ExtensionContext context) { + hadFailures = false; + // Register a Closeable invoked after all the other callbacks have been called: + // @BeforeAll/@AfterAll lifecycle methods and peer extension afterAll callbacks + // (e.g. DetectThreadLeaks). + context + .getStore(NAMESPACE) + .put("failureCheck", (AutoCloseable) () -> checkContextException(context)); + } + + @Override + public void afterAll(ExtensionContext context) { + checkContextException(context); + } + + private void checkContextException(ExtensionContext context) { + if (context.getExecutionException().isPresent()) { + if (!isFailedAssumption(context.getExecutionException().get())) { + hadFailures = true; + } + } + } + + public static boolean isFailedAssumption(Throwable t) { + return t instanceof TestAbortedException + || t.getClass().getName().equals("org.junit.AssumptionViolatedException"); + } + + @Override + public void testFailed(ExtensionContext context, Throwable cause) { + hadFailures = true; + } + + @Override + public void testAborted(ExtensionContext context, @Nullable Throwable cause) { + // This means the test threw an assumption exception. Ignored. + } + + @Override + public void testDisabled(ExtensionContext context, Optional reason) { + // This means the test is ignored for some other reason. + } + + // required to detect failures in dynamic tests. + @Override + public void interceptDynamicTest( + Invocation<@Nullable Void> invocation, + DynamicTestInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + try { + invocation.proceed(); + } catch (Throwable t) { + if (!isFailedAssumption(t)) { + hadFailures = true; + } + throw t; + } + } + + @Override + public boolean wasSuccessful() { + return !hadFailures; + } + } + + /// This extension sets up junit-jupiter implementations of the + /// test framework-dependent infrastructure in [LuceneTestCaseParent]. + /// + /// It tries to simulate before-after rules as they are implemented in junit4 + /// (call order, unwinding in case of failures, etc.). + static class ClassLevelCallbackChain + implements BeforeAllCallback, AfterAllCallback, AfterEachCallback { + private final SuiteFailureTracker suiteFailureTracker; + private final PrintReproduceInfoExtension printReproduceInfo; + + private TestFrameworkInfra jupiterFrameworkInfra; + private OrderedBeforeAfterCallbacks beforeAfters; + + private FieldToType fieldToType; + + ClassLevelCallbackChain( + SuiteFailureTracker suiteFailureTracker, + PrintReproduceInfoExtension printReproduceInfoExtension) { + this.suiteFailureTracker = suiteFailureTracker; + this.printReproduceInfo = printReproduceInfoExtension; + } + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + // touch LuceneTestCase to trigger static initializers. + LuceneTestCase.ensureInitialized(); + + var activeTestClass = context.getRequiredTestClass(); + + // these hooks need to run in the right order of before-after calls, + // with the expected nesting. + var classEnvRule = + new SetupAndRestoreStaticEnv(LuceneTestCaseJupiter::random, () -> activeTestClass); + var tempFileSupplier = + new TemporaryFilesSupplier( + suiteFailureTracker, LuceneTestCaseJupiter::random, () -> activeTestClass); + var perThreadRandom = new PerThreadRandom(getRandomSupplier(context.getExecutableInvoker())); + + fieldToType = new FieldToType(); + + this.jupiterFrameworkInfra = + new TestFrameworkInfra() { + @Override + public Random threadRandom() { + return perThreadRandom.get(); + } + + @Override + public SetupAndRestoreStaticEnv getClassEnv() { + return classEnvRule; + } + + @Override + public TemporaryFilesSupplier getTempFilesSupplier() { + return tempFileSupplier; + } + + @Override + public SuiteFailureState getSuiteFailureState() { + return suiteFailureTracker; + } + + @Override + public Field newField(Random random, String name, Object value, FieldType type) { + return fieldToType.newField(random, name, value, type); + } + }; + + var installFrameworkInfraSupport = + new BeforeAfterCallback() { + @Override + public void before() { + LuceneTestCaseParent.setTestFrameworkInfra(null, jupiterFrameworkInfra); + } + + @Override + public void after() { + LuceneTestCaseParent.setTestFrameworkInfra(jupiterFrameworkInfra, null); + } + }; + + var installEnvInfo = + new BeforeAfterCallback() { + @Override + public void before() throws Exception { + printReproduceInfo.testEnvInfo = + new TestEnvInfo( + classEnvRule.codec, + classEnvRule.similarity, + classEnvRule.locale, + classEnvRule.timeZone); + } + }; + + // This is the chain of before-after callbacks that must be called in the right order for + // compatibility + // with junit4 implementation. + this.beforeAfters = + new OrderedBeforeAfterCallbacks( + List.of( + perThreadRandom, + installFrameworkInfraSupport, + classEnvRule, + fieldToType, + installEnvInfo, + tempFileSupplier)); + + beforeAfters.before(); + } + + @Override + public void afterEach(ExtensionContext context) throws Exception { + fieldToType.after(); + } + + @Override + public void afterAll(ExtensionContext context) throws Exception { + beforeAfters.after(); + } + + // This trick is needed to get the Supplier injected by a parameter resolved + // of the randomized testing framework. + private Supplier getRandomSupplier(ExecutableInvoker executableInvoker) + throws Exception { + var hack = + new Object() { + @SuppressWarnings("unused") + public Supplier captureParameter(Supplier rnd) { + return rnd; + } + }; + + @SuppressWarnings("unchecked") + Supplier rnd = + (Supplier) + Objects.requireNonNull( + executableInvoker.invoke( + hack.getClass().getMethod("captureParameter", Supplier.class), hack)); + return rnd; + } + } + + static class PrintReproduceInfoExtension + implements BeforeAllCallback, TestWatcher, InvocationInterceptor { + public static final CharSequence TEST_REPRO_LEAD = TestEnvInfo.TEST_REPRO_LEAD; + public static final CharSequence TEST_ENV_LEAD = TestEnvInfo.TEST_ENV_LEAD; + + // Used for tests only to replace syserrs. + @SuppressWarnings("NonFinalStaticField") + public static PrintStream debugStream; + + TestEnvInfo testEnvInfo; + + private String rootSeed; + private boolean somethingFailed; + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + this.rootSeed = getRandomSupplier(context.getExecutableInvoker()).getRootSeed().toString(); + + // Register a Closeable invoked after all the other callbacks have been called: + // @BeforeAll/@AfterAll lifecycle methods and peer extension afterAll callbacks + // (e.g. DetectThreadLeaks). + context + .getStore(ExtensionContext.Namespace.create(PrintReproduceInfoExtension.class)) + .put( + "failureCheck", + (AutoCloseable) + () -> { + if (context.getExecutionException().isPresent()) { + if (!SuiteFailureTracker.isFailedAssumption( + context.getExecutionException().get())) { + printReproduceInfo(context); + somethingFailed = true; + } + } + + if (somethingFailed && testEnvInfo != null) { + println(testEnvInfo.getDebuggingInformation()); + } + }); + } + + @Override + public void testFailed(ExtensionContext context, @Nullable Throwable cause) { + somethingFailed = true; + printReproduceInfo(context); + } + + // Required to detect failures in dynamic tests. + @Override + public void interceptDynamicTest( + Invocation<@Nullable Void> invocation, + DynamicTestInvocationContext invocationContext, + ExtensionContext extensionContext) + throws Throwable { + try { + invocation.proceed(); + } catch (Throwable t) { + if (!SuiteFailureTracker.isFailedAssumption(t)) { + somethingFailed = true; + printReproduceInfo(extensionContext); + } + throw t; + } + } + + private void printReproduceInfo(ExtensionContext context) { + if (testEnvInfo == null || rootSeed == null) { + println( + "NOTE: test failed but no environment information is present to construct the reproduce-line."); + } else { + println( + testEnvInfo.getAdditionalFailureInfo( + rootSeed, + b -> { + // TODO: add gradle infrastructure to rerun tests based on their uniqueid + // instead of their class/method. This would allow dynamic tests to be + // repeatable. + + if (context.getTestClass().isPresent()) { + b.append("--tests "); + b.append(context.getRequiredTestClass().getName()); + if (context.getTestMethod().isPresent()) { + b.append(".").append(context.getRequiredTestMethod().getName()); + } + } + })); + } + } + + private void println(String msg) { + if (debugStream != null) { + debugStream.println(msg); + } else { + System.err.println(msg); + } + } + + // This trick is needed to get the Supplier injected by a parameter resolved + // of the randomized testing framework. + private RandomizedContext getRandomSupplier(ExecutableInvoker executableInvoker) + throws Exception { + var hack = + new Object() { + @SuppressWarnings("unused") + public RandomizedContext captureParameter(RandomizedContext ctx) { + return ctx; + } + }; + + return (RandomizedContext) + Objects.requireNonNull( + executableInvoker.invoke( + hack.getClass().getMethod("captureParameter", RandomizedContext.class), hack)); + } + } + + static class TestLevelCallbackChain implements BeforeEachCallback, AfterEachCallback { + private OrderedBeforeAfterCallbacks callbacks; + + @Override + public void beforeEach(ExtensionContext context) throws Exception { + this.callbacks = + new OrderedBeforeAfterCallbacks(List.of(new TestRuleSetupAndRestoreInstanceEnv())); + callbacks.before(); + } + + @Override + public void afterEach(ExtensionContext context) throws Exception { + callbacks.after(); + } + } + + @RegisterExtension + @Order(0) + static final SuiteFailureTracker suiteFailureTracker = new SuiteFailureTracker(); + + @RegisterExtension + @Order(1) + static final PrintReproduceInfoExtension printReproduceInfoExtension = + new PrintReproduceInfoExtension(); + + @RegisterExtension + @Order(2) + static final ClassLevelCallbackChain classLevelCallbackChain = + new ClassLevelCallbackChain( + Objects.requireNonNull(suiteFailureTracker), + Objects.requireNonNull(printReproduceInfoExtension)); + + @RegisterExtension + @Order(3) + static final TestLevelCallbackChain testLevelCallbackChain = new TestLevelCallbackChain(); + + // + // Deprecated or removed methods (LuceneTestCase) and other backward-compatibility + // infrastructure. + // + + /// [BeforeEach] isn't inherited in junit5, so call this method explicitly, even if overridden. + @BeforeEach + void callSetUp() throws Exception { + setUp(); + } + + /// a [BeforeEach] callback. In subclasses, if you override, you must call `super.setUp` too. + public void setUp() throws Exception {} + + /// [AfterEach] isn't inherited in junit5, so call this method explicitly, even if overridden. + @AfterEach + public void callTearDown() throws Exception { + tearDown(); + } + + /// a [AfterEach] callback. In subclasses, if you override, you must call `super.setUp` too. + public void tearDown() throws Exception {} + + /** + * Use explicit, injected {@link Random} or {@code Supplier} parameters on junit jupiter + * test methods (or callbacks). + */ + @Deprecated + public static Random random() { + return LuceneTestCaseParent.random(); + } + + @Test + void verifyTestAssertionStatus() throws Exception { + TestRuleAssertionsRequired.checkAssertionStatus(); + } + + /** Enforce test class naming convention. */ + @Test + void enforceClassNamingConvention(TestInfo testInfo) { + new VerifyTestClassNamingConvention( + "org.apache.lucene", Pattern.compile("(.+\\.)(Test)([^.]+)")) + .check(testInfo.getTestClass().orElseThrow()); + } + + /** + * Unfortunately there is no easy way to implement custom test providers in jupiter so we just + * enforce annotations on {@code test*} methods (so that they're not silently ignored). + * + *

A dynamic test factory would almost work but dynamic tests skip all the + * before-after hooks so they're not a direct substitute. + */ + @Test + void allTestMethodsAreAnnotated(TestInfo testInfo) { + var testMethodsWithoutAnnotations = + ReflectionSupport.findMethods( + testInfo.getTestClass().orElseThrow(), + m -> { + return m.getName().startsWith("test") + && !ModifierSupport.isStatic(m) + && !AnnotationSupport.isAnnotated(m, Test.class); + }, + HierarchyTraversalMode.BOTTOM_UP); + + if (!testMethodsWithoutAnnotations.isEmpty()) { + throw new AssertionError( + "test* methods must be annotated with @Test in junit5/jupiter, the following are not: " + + testMethodsWithoutAnnotations.stream() + .map(m -> "\n - " + m.getDeclaringClass().getName() + "#" + m.getName()) + .collect(Collectors.joining())); + } + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCaseParent.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCaseParent.java new file mode 100644 index 000000000000..9fcfb9e259d7 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCaseParent.java @@ -0,0 +1,2834 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.tests.util; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.frequently; +import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; +import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsInt; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +import com.carrotsearch.randomizedtesting.RandomizedContext; +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.Xoroshiro128PlusRandom; +import com.carrotsearch.randomizedtesting.annotations.TestGroup; +import com.carrotsearch.randomizedtesting.generators.RandomNumbers; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.lang.reflect.Constructor; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.Collator; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.Set; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import junit.framework.AssertionFailedError; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.bitvectors.HnswBitVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.CompositeReader; +import org.apache.lucene.index.ConcurrentMergeScheduler; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.LiveIndexWriterConfig; +import org.apache.lucene.index.LogByteSizeMergePolicy; +import org.apache.lucene.index.LogDocMergePolicy; +import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.MergeScheduler; +import org.apache.lucene.index.MultiBits; +import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.index.MultiTerms; +import org.apache.lucene.index.NoDeletionPolicy; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.ParallelCompositeReader; +import org.apache.lucene.index.ParallelLeafReader; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.index.SimpleMergedSegmentWarmer; +import org.apache.lucene.index.SnapshotDeletionPolicy; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.StoredFields; +import org.apache.lucene.index.TermVectors; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.internal.tests.IndexPackageAccess; +import org.apache.lucene.internal.tests.TestSecrets; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LRUQueryCache; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryCache; +import org.apache.lucene.search.QueryCachingPolicy; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FSLockFactory; +import org.apache.lucene.store.FileSwitchDirectory; +import org.apache.lucene.store.FlushInfo; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.MergeInfo; +import org.apache.lucene.store.NRTCachingDirectory; +import org.apache.lucene.store.ReadOnceHint; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.index.AlcoholicMergePolicy; +import org.apache.lucene.tests.index.AssertingDirectoryReader; +import org.apache.lucene.tests.index.AssertingLeafReader; +import org.apache.lucene.tests.index.FieldFilterLeafReader; +import org.apache.lucene.tests.index.MergingCodecReader; +import org.apache.lucene.tests.index.MergingDirectoryReaderWrapper; +import org.apache.lucene.tests.index.MismatchedCodecReader; +import org.apache.lucene.tests.index.MismatchedDirectoryReader; +import org.apache.lucene.tests.index.MismatchedLeafReader; +import org.apache.lucene.tests.index.MockIndexWriterEventListener; +import org.apache.lucene.tests.index.MockRandomMergePolicy; +import org.apache.lucene.tests.mockfile.VirusCheckingFS; +import org.apache.lucene.tests.search.AssertingIndexSearcher; +import org.apache.lucene.tests.store.BaseDirectoryWrapper; +import org.apache.lucene.tests.store.MockDirectoryWrapper; +import org.apache.lucene.tests.store.RawDirectoryWrapper; +import org.apache.lucene.tests.util.automaton.AutomatonTestUtil; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CommandLineUtil; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.NamedThreadFactory; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; +import org.hamcrest.Matcher; +import org.hamcrest.MatcherAssert; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.internal.AssumptionViolatedException; + +/** + * Private parent class for junit4 ({@link LuceneTestCase} and junit jupiter ({@link + * LuceneTestCaseJupiter}). + */ +public abstract sealed class LuceneTestCaseParent extends Assert + permits LuceneTestCase, LuceneTestCaseJupiter { + + // -------------------------------------------------------------------- + // Test groups, system properties and other annotations modifying tests + // -------------------------------------------------------------------- + + public static final String SYSPROP_NIGHTLY = "tests.nightly"; + public static final String SYSPROP_WEEKLY = "tests.weekly"; + public static final String SYSPROP_MONSTER = "tests.monster"; + public static final String SYSPROP_AWAITSFIX = "tests.awaitsfix"; + public static final String SYSPROP_MAXFAILURES = "tests.maxfailures"; + public static final String SYSPROP_FAILFAST = "tests.failfast"; + + /** + * True if and only if tests are run in verbose mode. If this flag is false tests are not expected + * to print any messages. Enforced with {@link TestRuleLimitSysouts}. + */ + public static final boolean VERBOSE = systemPropertyAsBoolean("tests.verbose", false); + + /** Enables or disables dumping of {@link InfoStream} messages. */ + public static final boolean INFOSTREAM = systemPropertyAsBoolean("tests.infostream", VERBOSE); + + /** + * True if {@code tests.asserts} is enabled (either explicitly via the build option or, if not + * present, by the default assertion status on this class). + */ + public static final boolean TEST_ASSERTS_ENABLED = + systemPropertyAsBoolean("tests.asserts", LuceneTestCase.class.desiredAssertionStatus()); + + /** + * The default (embedded resource) lines file. + * + * @see #TEST_LINE_DOCS_FILE + */ + public static final String DEFAULT_LINE_DOCS_FILE = "europarl.lines.txt.gz"; + + /** + * Random sample from enwiki used in tests. See {@code help/tests.txt}. gradle task downloading + * this data set: {@code gradlew getEnWikiRandomLines}. + */ + public static final String JENKINS_LARGE_LINE_DOCS_FILE = "enwiki.random.lines.txt"; + + /** Gets the codec to run tests with. */ + public static final String TEST_CODEC = System.getProperty("tests.codec", "random"); + + /** Gets the postingsFormat to run tests with. */ + public static final String TEST_POSTINGSFORMAT = + System.getProperty("tests.postingsformat", "random"); + + /** Gets the docValuesFormat to run tests with */ + public static final String TEST_DOCVALUESFORMAT = + System.getProperty("tests.docvaluesformat", "random"); + + /** Gets the directory to run tests with */ + public static final String TEST_DIRECTORY = System.getProperty("tests.directory", "random"); + + /** The line file used in tests (by {@link LineFileDocs}). */ + public static final String TEST_LINE_DOCS_FILE = + System.getProperty("tests.linedocsfile", DEFAULT_LINE_DOCS_FILE); + + /** Whether or not {@link LuceneTestCase.Nightly} tests should run. */ + public static final boolean TEST_NIGHTLY = + systemPropertyAsBoolean( + SYSPROP_NIGHTLY, LuceneTestCase.Nightly.class.getAnnotation(TestGroup.class).enabled()); + + /** Whether or not {@link LuceneTestCase.Weekly} tests should run. */ + public static final boolean TEST_WEEKLY = + systemPropertyAsBoolean( + SYSPROP_WEEKLY, LuceneTestCase.Weekly.class.getAnnotation(TestGroup.class).enabled()); + + /** Whether or not {@link LuceneTestCase.Monster} tests should run. */ + public static final boolean TEST_MONSTER = + systemPropertyAsBoolean( + SYSPROP_MONSTER, LuceneTestCase.Monster.class.getAnnotation(TestGroup.class).enabled()); + + /** Whether or not {@link LuceneTestCase.AwaitsFix} tests should run. */ + public static final boolean TEST_AWAITSFIX = + systemPropertyAsBoolean( + SYSPROP_AWAITSFIX, + LuceneTestCase.AwaitsFix.class.getAnnotation(TestGroup.class).enabled()); + + /** + * Throttling, see {@link MockDirectoryWrapper#setThrottling(MockDirectoryWrapper.Throttling)}. + */ + public static final MockDirectoryWrapper.Throttling TEST_THROTTLING = + TEST_NIGHTLY + ? MockDirectoryWrapper.Throttling.SOMETIMES + : MockDirectoryWrapper.Throttling.NEVER; + + /** + * A random multiplier which you should use when writing random tests: multiply it by the number + * of iterations to scale your tests (for nightly builds). + */ + public static final int RANDOM_MULTIPLIER = + systemPropertyAsInt("tests.multiplier", defaultRandomMultiplier()); + + /** Compute the default value of the random multiplier (based on {@link #TEST_NIGHTLY}). */ + static int defaultRandomMultiplier() { + return TEST_NIGHTLY ? 2 : 1; + } + + /** Leave temporary files on disk, even on successful runs. */ + public static final boolean LEAVE_TEMPORARY; + + static { + boolean defaultValue = false; + for (String property : + Arrays.asList( + "tests.leaveTemporary" /* ANT tasks' (junit4) flag. */, + "tests.leavetemporary" /* lowercase */, + "tests.leavetmpdir" /* default */)) { + defaultValue |= systemPropertyAsBoolean(property, false); + } + LEAVE_TEMPORARY = defaultValue; + } + + /** Filesystem-based {@link Directory} implementations. */ + protected static final List FS_DIRECTORIES = + Arrays.asList("NIOFSDirectory", "MMapDirectory"); + + /** All {@link Directory} implementations. */ + protected static final List CORE_DIRECTORIES; + + static { + CORE_DIRECTORIES = new ArrayList<>(FS_DIRECTORIES); + CORE_DIRECTORIES.add(ByteBuffersDirectory.class.getSimpleName()); + } + + /** + * If specified, limits the number of method calls to each individual instance returned by {@link + * #random()}. + */ + public static final String SYSPROP_RANDOM_MAXCALLS = "tests.random.maxcalls"; + + /** If specified, limits the number of calls {@link #random()} itself. */ + public static final String SYSPROP_RANDOM_MAXACQUIRES = "tests.random.maxacquires"; + + /** + * Returns a Random instance based on the current state of another Random. The returned instance + * should be faster for thousands of consecutive calls because it doesn't assert that it isn't + * shared between threads or used within the correct {@link RandomizedContext}. + * + *

Use this method for local tight loops that generate a lot of random data. + */ + public static Random nonAssertingRandom(Random rnd) { + return new Xoroshiro128PlusRandom(rnd.nextLong()); + } + + // ----------------------------------------------------------------- + // This bit is for supporting all the static method calls we have all over the place. + // In order for both junit4 and junit5 to work, we need to make sure that each class + // runs in isolation (so no concurrent tests at the moment) and that it provides + // test-framework-dependent "infrastructure" like Random suppliers, test class names, etc. + // to this class. + // ----------------------------------------------------------------- + + /** Junit4/Junit5-dependent implementations. */ + protected interface TestFrameworkInfra { + Random threadRandom(); + + SetupAndRestoreStaticEnv getClassEnv(); + + TemporaryFilesSupplier getTempFilesSupplier(); + + SuiteFailureState getSuiteFailureState(); + + Field newField(Random random, String name, Object value, FieldType type); + } + + private static final AtomicReference testFrameworkInfra = + new AtomicReference<>(); + + protected static void setTestFrameworkInfra( + TestFrameworkInfra expected, TestFrameworkInfra newInfra) { + TestFrameworkInfra prev; + if ((prev = testFrameworkInfra.compareAndExchange(expected, newInfra)) != expected) { + throw new RuntimeException( + "Test framework infrastructure is not set to the expected value: " + + prev + + ", expected: " + + expected); + } + } + + protected static TestFrameworkInfra getTestFrameworkInfra() { + return Objects.requireNonNull( + testFrameworkInfra.get(), "Expected test framework not to be null."); + } + + // ----------------------------------------------------------------- + // Truly immutable fields and constants, initialized once and valid + // for all suites ever since. + // ----------------------------------------------------------------- + + /** A {@link org.apache.lucene.search.QueryCachingPolicy} that randomly caches. */ + public static final QueryCachingPolicy MAYBE_CACHE_POLICY = + new QueryCachingPolicy() { + @Override + public void onUse(Query query) {} + + @Override + public boolean shouldCache(Query query) { + return getTestFrameworkInfra().threadRandom().nextBoolean(); + } + }; + + public static Random random() { + return Objects.requireNonNull(testFrameworkInfra.get(), "Test framework not initialized?") + .threadRandom(); + } + + /** Gets a resource from the test's classpath as {@link InputStream}. */ + protected InputStream getDataInputStream(String name) throws IOException { + return IOUtils.requireResourceNonNull(this.getClass().getResourceAsStream(name), name); + } + + // these hide the deprecated Assert.assertThat method + public static void assertThat(T actual, Matcher matcher) { + MatcherAssert.assertThat(actual, matcher); + } + + public static void assertThat(String reason, T actual, Matcher matcher) { + MatcherAssert.assertThat(reason, actual, matcher); + } + + public void assertReaderEquals(String info, IndexReader leftReader, IndexReader rightReader) + throws IOException { + assertReaderStatisticsEquals(info, leftReader, rightReader); + assertTermsEquals(info, leftReader, rightReader, true); + assertNormsEquals(info, leftReader, rightReader); + assertStoredFieldsEquals(info, leftReader, rightReader); + assertTermVectorsEquals(info, leftReader, rightReader); + assertDocValuesEquals(info, leftReader, rightReader); + assertDeletedDocsEquals(info, leftReader, rightReader); + assertFieldInfosEquals(info, leftReader, rightReader); + assertPointsEquals(info, leftReader, rightReader); + } + + /** checks that reader-level statistics are the same */ + public void assertReaderStatisticsEquals( + String info, IndexReader leftReader, IndexReader rightReader) throws IOException { + // Somewhat redundant: we never delete docs + assertEquals(info, leftReader.maxDoc(), rightReader.maxDoc()); + assertEquals(info, leftReader.numDocs(), rightReader.numDocs()); + assertEquals(info, leftReader.numDeletedDocs(), rightReader.numDeletedDocs()); + assertEquals(info, leftReader.hasDeletions(), rightReader.hasDeletions()); + } + + /** Fields api equivalency */ + public void assertTermsEquals( + String info, IndexReader leftReader, IndexReader rightReader, boolean deep) + throws IOException { + Set leftFields = new HashSet<>(FieldInfos.getIndexedFields(leftReader)); + Set rightFields = new HashSet<>(FieldInfos.getIndexedFields(rightReader)); + assertEquals(info, leftFields, rightFields); + + for (String field : leftFields) { + assertTermsEquals( + info, + leftReader, + MultiTerms.getTerms(leftReader, field), + MultiTerms.getTerms(rightReader, field), + deep); + } + } + + /** Terms api equivalency */ + public void assertTermsEquals( + String info, IndexReader leftReader, Terms leftTerms, Terms rightTerms, boolean deep) + throws IOException { + if (leftTerms == null || rightTerms == null) { + assertNull(info, leftTerms); + assertNull(info, rightTerms); + return; + } + assertTermsStatisticsEquals(info, leftTerms, rightTerms); + assertEquals("hasOffsets", leftTerms.hasOffsets(), rightTerms.hasOffsets()); + assertEquals("hasPositions", leftTerms.hasPositions(), rightTerms.hasPositions()); + assertEquals("hasPayloads", leftTerms.hasPayloads(), rightTerms.hasPayloads()); + + TermsEnum leftTermsEnum = leftTerms.iterator(); + TermsEnum rightTermsEnum = rightTerms.iterator(); + assertTermsEnumEquals(info, leftReader, leftTermsEnum, rightTermsEnum, true); + + assertTermsSeekingEquals(info, leftTerms, rightTerms); + + if (deep) { + int numIntersections = atLeast(3); + for (int i = 0; i < numIntersections; i++) { + String re = AutomatonTestUtil.randomRegexp(random()); + Automaton a = new RegExp(re, RegExp.NONE).toAutomaton(); + a = Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + CompiledAutomaton automaton = new CompiledAutomaton(a); + if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) { + // TODO: test start term too + TermsEnum leftIntersection = leftTerms.intersect(automaton, null); + TermsEnum rightIntersection = rightTerms.intersect(automaton, null); + assertTermsEnumEquals(info, leftReader, leftIntersection, rightIntersection, rarely()); + } + } + } + } + + /** checks collection-level statistics on Terms */ + public void assertTermsStatisticsEquals(String info, Terms leftTerms, Terms rightTerms) + throws IOException { + assertEquals(info, leftTerms.getDocCount(), rightTerms.getDocCount()); + assertEquals(info, leftTerms.getSumDocFreq(), rightTerms.getSumDocFreq()); + assertEquals(info, leftTerms.getSumTotalTermFreq(), rightTerms.getSumTotalTermFreq()); + if (leftTerms.size() != -1 && rightTerms.size() != -1) { + assertEquals(info, leftTerms.size(), rightTerms.size()); + } + } + + /** + * checks the terms enum sequentially if deep is false, it does a 'shallow' test that doesnt go + * down to the docsenums + */ + public void assertTermsEnumEquals( + String info, + IndexReader leftReader, + TermsEnum leftTermsEnum, + TermsEnum rightTermsEnum, + boolean deep) + throws IOException { + BytesRef term; + PostingsEnum leftPositions = null; + PostingsEnum rightPositions = null; + PostingsEnum leftDocs = null; + PostingsEnum rightDocs = null; + + while ((term = leftTermsEnum.next()) != null) { + assertEquals(info, term, rightTermsEnum.next()); + assertTermStatsEquals(info, leftTermsEnum, rightTermsEnum); + if (deep) { + assertDocsAndPositionsEnumEquals( + info, + leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL), + rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL)); + + assertPositionsSkippingEquals( + info, + leftReader, + leftTermsEnum.docFreq(), + leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL), + rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL)); + + // with freqs: + assertDocsEnumEquals( + info, + leftDocs = leftTermsEnum.postings(leftDocs), + rightDocs = rightTermsEnum.postings(rightDocs), + true); + + // w/o freqs: + assertDocsEnumEquals( + info, + leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE), + rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE), + false); + + // with freqs: + assertDocsSkippingEquals( + info, + leftReader, + leftTermsEnum.docFreq(), + leftDocs = leftTermsEnum.postings(leftDocs), + rightDocs = rightTermsEnum.postings(rightDocs), + true); + + // w/o freqs: + assertDocsSkippingEquals( + info, + leftReader, + leftTermsEnum.docFreq(), + leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE), + rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE), + false); + } + } + assertNull(info, rightTermsEnum.next()); + } + + /** checks docs + freqs + positions + payloads, sequentially */ + public void assertDocsAndPositionsEnumEquals( + String info, PostingsEnum leftDocs, PostingsEnum rightDocs) throws IOException { + assertNotNull(leftDocs); + assertNotNull(rightDocs); + assertEquals(info, -1, leftDocs.docID()); + assertEquals(info, -1, rightDocs.docID()); + int docid; + while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + assertEquals(info, docid, rightDocs.nextDoc()); + int freq = leftDocs.freq(); + assertEquals(info, freq, rightDocs.freq()); + for (int i = 0; i < freq; i++) { + assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition()); + assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload()); + assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset()); + assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset()); + } + } + assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc()); + } + + /** checks docs + freqs, sequentially */ + public void assertDocsEnumEquals( + String info, PostingsEnum leftDocs, PostingsEnum rightDocs, boolean hasFreqs) + throws IOException { + if (leftDocs == null) { + assertNull(rightDocs); + return; + } + assertEquals(info, -1, leftDocs.docID()); + assertEquals(info, -1, rightDocs.docID()); + int docid; + while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + assertEquals(info, docid, rightDocs.nextDoc()); + if (hasFreqs) { + assertEquals(info, leftDocs.freq(), rightDocs.freq()); + } + } + assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc()); + } + + /** checks advancing docs */ + public void assertDocsSkippingEquals( + String info, + IndexReader leftReader, + int docFreq, + PostingsEnum leftDocs, + PostingsEnum rightDocs, + boolean hasFreqs) + throws IOException { + if (leftDocs == null) { + assertNull(rightDocs); + return; + } + int docid = -1; + int averageGap = leftReader.maxDoc() / (1 + docFreq); + int skipInterval = 16; + + while (true) { + if (random().nextBoolean()) { + // nextDoc() + docid = leftDocs.nextDoc(); + assertEquals(info, docid, rightDocs.nextDoc()); + } else { + // advance() + int skip = + docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap)); + docid = leftDocs.advance(skip); + assertEquals(info, docid, rightDocs.advance(skip)); + } + + if (docid == DocIdSetIterator.NO_MORE_DOCS) { + return; + } + if (hasFreqs) { + assertEquals(info, leftDocs.freq(), rightDocs.freq()); + } + } + } + + /** checks advancing docs + positions */ + public void assertPositionsSkippingEquals( + String info, + IndexReader leftReader, + int docFreq, + PostingsEnum leftDocs, + PostingsEnum rightDocs) + throws IOException { + if (leftDocs == null || rightDocs == null) { + assertNull(leftDocs); + assertNull(rightDocs); + return; + } + + int docid = -1; + int averageGap = leftReader.maxDoc() / (1 + docFreq); + int skipInterval = 16; + + while (true) { + if (random().nextBoolean()) { + // nextDoc() + docid = leftDocs.nextDoc(); + assertEquals(info, docid, rightDocs.nextDoc()); + } else { + // advance() + int skip = + docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap)); + docid = leftDocs.advance(skip); + assertEquals(info, docid, rightDocs.advance(skip)); + } + + if (docid == DocIdSetIterator.NO_MORE_DOCS) { + return; + } + int freq = leftDocs.freq(); + assertEquals(info, freq, rightDocs.freq()); + for (int i = 0; i < freq; i++) { + assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition()); + assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload()); + } + } + } + + private void assertTermsSeekingEquals(String info, Terms leftTerms, Terms rightTerms) + throws IOException { + + // just an upper bound + int numTests = atLeast(20); + Random random = random(); + + TermsEnum leftEnum = null; + + // collect this number of terms from the left side + HashSet tests = new HashSet<>(); + int numPasses = 0; + while (numPasses < 10 && tests.size() < numTests) { + leftEnum = leftTerms.iterator(); + BytesRef term; + while ((term = leftEnum.next()) != null) { + int code = random.nextInt(10); + if (code == 0) { + // the term + tests.add(BytesRef.deepCopyOf(term)); + } else if (code == 1) { + // truncated subsequence of term + term = BytesRef.deepCopyOf(term); + if (term.length > 0) { + // truncate it + term.length = random.nextInt(term.length); + } + } else if (code == 2) { + // term, but ensure a non-zero offset + byte[] newbytes = new byte[term.length + 5]; + System.arraycopy(term.bytes, term.offset, newbytes, 5, term.length); + tests.add(new BytesRef(newbytes, 5, term.length)); + } else if (code == 3) { + switch (random().nextInt(3)) { + case 0: + tests.add(new BytesRef()); // before the first term + break; + case 1: + tests.add(new BytesRef(new byte[] {(byte) 0xFF, (byte) 0xFF})); // past the last term + break; + case 2: + tests.add(new BytesRef(TestUtil.randomSimpleString(random()))); // random term + break; + default: + throw new AssertionError(); + } + } + } + numPasses++; + } + + TermsEnum rightEnum = rightTerms.iterator(); + + ArrayList shuffledTests = new ArrayList<>(tests); + Collections.shuffle(shuffledTests, random); + + for (BytesRef b : shuffledTests) { + if (rarely()) { + // make new enums + leftEnum = leftTerms.iterator(); + rightEnum = rightTerms.iterator(); + } + + final boolean seekExact = random().nextBoolean(); + + if (seekExact) { + assertEquals(info, leftEnum.seekExact(b), rightEnum.seekExact(b)); + } else { + TermsEnum.SeekStatus leftStatus = leftEnum.seekCeil(b); + TermsEnum.SeekStatus rightStatus = rightEnum.seekCeil(b); + assertEquals(info, leftStatus, rightStatus); + if (leftStatus != TermsEnum.SeekStatus.END) { + assertEquals(info, leftEnum.term(), rightEnum.term()); + assertTermStatsEquals(info, leftEnum, rightEnum); + } + } + } + } + + /** checks term-level statistics */ + public void assertTermStatsEquals(String info, TermsEnum leftTermsEnum, TermsEnum rightTermsEnum) + throws IOException { + assertEquals(info, leftTermsEnum.docFreq(), rightTermsEnum.docFreq()); + assertEquals(info, leftTermsEnum.totalTermFreq(), rightTermsEnum.totalTermFreq()); + } + + /** checks that norms are the same across all fields */ + public void assertNormsEquals(String info, IndexReader leftReader, IndexReader rightReader) + throws IOException { + Set leftFields = new HashSet<>(FieldInfos.getIndexedFields(leftReader)); + Set rightFields = new HashSet<>(FieldInfos.getIndexedFields(rightReader)); + assertEquals(info, leftFields, rightFields); + + for (String field : leftFields) { + NumericDocValues leftNorms = MultiDocValues.getNormValues(leftReader, field); + NumericDocValues rightNorms = MultiDocValues.getNormValues(rightReader, field); + if (leftNorms != null && rightNorms != null) { + assertDocValuesEquals(info, leftReader.maxDoc(), leftNorms, rightNorms); + } else { + assertNull(info, leftNorms); + assertNull(info, rightNorms); + } + } + } + + /** checks that stored fields of all documents are the same */ + public void assertStoredFieldsEquals(String info, IndexReader leftReader, IndexReader rightReader) + throws IOException { + assert leftReader.maxDoc() == rightReader.maxDoc(); + StoredFields leftStoredFields = leftReader.storedFields(); + StoredFields rightStoredFields = rightReader.storedFields(); + for (int i = 0; i < leftReader.maxDoc(); i++) { + Document leftDoc = leftStoredFields.document(i); + Document rightDoc = rightStoredFields.document(i); + + // TODO: I think this is bogus because we don't document what the order should be + // from these iterators, etc. I think the codec/IndexReader should be free to order this stuff + // in whatever way it wants (e.g. maybe it packs related fields together or something) + // To fix this, we sort the fields in both documents by name, but + // we still assume that all instances with same name are in order: + Comparator comp = Comparator.comparing(IndexableField::name); + List leftFields = new ArrayList<>(leftDoc.getFields()); + List rightFields = new ArrayList<>(rightDoc.getFields()); + leftFields.sort(comp); + rightFields.sort(comp); + + Iterator leftIterator = leftFields.iterator(); + Iterator rightIterator = rightFields.iterator(); + while (leftIterator.hasNext()) { + assertTrue(info, rightIterator.hasNext()); + assertStoredFieldEquals(info, leftIterator.next(), rightIterator.next()); + } + assertFalse(info, rightIterator.hasNext()); + } + } + + /** checks that two stored fields are equivalent */ + public void assertStoredFieldEquals( + String info, IndexableField leftField, IndexableField rightField) { + assertEquals(info, leftField.name(), rightField.name()); + assertEquals(info, leftField.binaryValue(), rightField.binaryValue()); + assertEquals(info, leftField.stringValue(), rightField.stringValue()); + assertEquals(info, leftField.numericValue(), rightField.numericValue()); + // TODO: should we check the FT at all? + } + + /** checks that term vectors across all fields are equivalent */ + public void assertTermVectorsEquals(String info, IndexReader leftReader, IndexReader rightReader) + throws IOException { + assert leftReader.maxDoc() == rightReader.maxDoc(); + TermVectors leftVectors = leftReader.termVectors(); + TermVectors rightVectors = rightReader.termVectors(); + for (int i = 0; i < leftReader.maxDoc(); i++) { + Fields leftFields = leftVectors.get(i); + Fields rightFields = rightVectors.get(i); + + // Fields could be null if there are no postings, + // but then it must be null for both + if (leftFields == null || rightFields == null) { + assertNull(info, leftFields); + assertNull(info, rightFields); + return; + } + if (leftFields.size() != -1 && rightFields.size() != -1) { + assertEquals(info, leftFields.size(), rightFields.size()); + } + + Iterator leftEnum = leftFields.iterator(); + Iterator rightEnum = rightFields.iterator(); + while (leftEnum.hasNext()) { + String field = leftEnum.next(); + assertEquals(info, field, rightEnum.next()); + assertTermsEquals( + info, leftReader, leftFields.terms(field), rightFields.terms(field), rarely()); + } + assertFalse(rightEnum.hasNext()); + } + } + + private static Set getDVFields(IndexReader reader) { + Set fields = new HashSet<>(); + for (FieldInfo fi : FieldInfos.getMergedFieldInfos(reader)) { + if (fi.getDocValuesType() != DocValuesType.NONE) { + fields.add(fi.name); + } + } + + return fields; + } + + /** checks that docvalues across all fields are equivalent */ + public void assertDocValuesEquals(String info, IndexReader leftReader, IndexReader rightReader) + throws IOException { + Set leftFields = getDVFields(leftReader); + Set rightFields = getDVFields(rightReader); + assertEquals(info, leftFields, rightFields); + + for (String field : leftFields) { + // TODO: clean this up... very messy + { + NumericDocValues leftValues = MultiDocValues.getNumericValues(leftReader, field); + NumericDocValues rightValues = MultiDocValues.getNumericValues(rightReader, field); + if (leftValues != null && rightValues != null) { + assertDocValuesEquals(info, leftReader.maxDoc(), leftValues, rightValues); + } else { + assertTrue( + info + ": left numeric doc values for field=\"" + field + "\" are not null", + leftValues == null || leftValues.nextDoc() == NO_MORE_DOCS); + assertTrue( + info + ": right numeric doc values for field=\"" + field + "\" are not null", + rightValues == null || rightValues.nextDoc() == NO_MORE_DOCS); + } + } + + { + BinaryDocValues leftValues = MultiDocValues.getBinaryValues(leftReader, field); + BinaryDocValues rightValues = MultiDocValues.getBinaryValues(rightReader, field); + if (leftValues != null && rightValues != null) { + while (true) { + int docID = leftValues.nextDoc(); + assertEquals(docID, rightValues.nextDoc()); + if (docID == NO_MORE_DOCS) { + break; + } + assertEquals(leftValues.binaryValue(), rightValues.binaryValue()); + } + } else { + assertTrue(info, leftValues == null || leftValues.nextDoc() == NO_MORE_DOCS); + assertTrue(info, rightValues == null || rightValues.nextDoc() == NO_MORE_DOCS); + } + } + + { + SortedDocValues leftValues = MultiDocValues.getSortedValues(leftReader, field); + SortedDocValues rightValues = MultiDocValues.getSortedValues(rightReader, field); + if (leftValues != null && rightValues != null) { + // numOrds + assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount()); + // ords + for (int i = 0; i < leftValues.getValueCount(); i++) { + final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(i)); + final BytesRef right = rightValues.lookupOrd(i); + assertEquals(info, left, right); + } + // bytes + while (true) { + int docID = leftValues.nextDoc(); + assertEquals(docID, rightValues.nextDoc()); + if (docID == NO_MORE_DOCS) { + break; + } + final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(leftValues.ordValue())); + final BytesRef right = rightValues.lookupOrd(rightValues.ordValue()); + assertEquals(info, left, right); + } + } else { + assertNull(info, leftValues); + assertNull(info, rightValues); + } + } + + { + SortedSetDocValues leftValues = MultiDocValues.getSortedSetValues(leftReader, field); + SortedSetDocValues rightValues = MultiDocValues.getSortedSetValues(rightReader, field); + if (leftValues != null && rightValues != null) { + // numOrds + assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount()); + // ords + for (int i = 0; i < leftValues.getValueCount(); i++) { + final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(i)); + final BytesRef right = rightValues.lookupOrd(i); + assertEquals(info, left, right); + } + // ord lists + while (true) { + int docID = leftValues.nextDoc(); + assertEquals(docID, rightValues.nextDoc()); + if (docID == NO_MORE_DOCS) { + break; + } + assertEquals(info, leftValues.docValueCount(), rightValues.docValueCount()); + for (int i = 0; i < leftValues.docValueCount(); i++) { + assertEquals(info, leftValues.nextOrd(), rightValues.nextOrd()); + } + } + } else { + assertNull(info, leftValues); + assertNull(info, rightValues); + } + } + + { + SortedNumericDocValues leftValues = + MultiDocValues.getSortedNumericValues(leftReader, field); + SortedNumericDocValues rightValues = + MultiDocValues.getSortedNumericValues(rightReader, field); + if (leftValues != null && rightValues != null) { + while (true) { + int docID = leftValues.nextDoc(); + assertEquals(docID, rightValues.nextDoc()); + if (docID == NO_MORE_DOCS) { + break; + } + assertEquals(info, leftValues.docValueCount(), rightValues.docValueCount()); + for (int j = 0; j < leftValues.docValueCount(); j++) { + assertEquals(info, leftValues.nextValue(), rightValues.nextValue()); + } + } + } else { + assertNull(info, leftValues); + assertNull(info, rightValues); + } + } + } + } + + public void assertDocValuesEquals( + String info, int num, NumericDocValues leftDocValues, NumericDocValues rightDocValues) + throws IOException { + assertNotNull(info, leftDocValues); + assertNotNull(info, rightDocValues); + while (true) { + int leftDocID = leftDocValues.nextDoc(); + int rightDocID = rightDocValues.nextDoc(); + assertEquals(leftDocID, rightDocID); + if (leftDocID == NO_MORE_DOCS) { + return; + } + assertEquals(leftDocValues.longValue(), rightDocValues.longValue()); + } + } + + // TODO: this is kinda stupid, we don't delete documents in the test. + public void assertDeletedDocsEquals(String info, IndexReader leftReader, IndexReader rightReader) + throws IOException { + assert leftReader.numDeletedDocs() == rightReader.numDeletedDocs(); + Bits leftBits = MultiBits.getLiveDocs(leftReader); + Bits rightBits = MultiBits.getLiveDocs(rightReader); + + if (leftBits == null || rightBits == null) { + assertNull(info, leftBits); + assertNull(info, rightBits); + return; + } + + assert leftReader.maxDoc() == rightReader.maxDoc(); + assertEquals(info, leftBits.length(), rightBits.length()); + for (int i = 0; i < leftReader.maxDoc(); i++) { + assertEquals(info, leftBits.get(i), rightBits.get(i)); + } + } + + public void assertFieldInfosEquals(String info, IndexReader leftReader, IndexReader rightReader) + throws IOException { + FieldInfos leftInfos = FieldInfos.getMergedFieldInfos(leftReader); + FieldInfos rightInfos = FieldInfos.getMergedFieldInfos(rightReader); + + // TODO: would be great to verify more than just the names of the fields! + TreeSet left = new TreeSet<>(); + TreeSet right = new TreeSet<>(); + + for (FieldInfo fi : leftInfos) { + left.add(fi.name); + } + + for (FieldInfo fi : rightInfos) { + right.add(fi.name); + } + + assertEquals(info, left, right); + } + + // naive silly memory heavy uninversion!! maps docID -> packed values (a Set because a given doc + // can be multi-valued) + private Map> uninvert(String fieldName, IndexReader reader) + throws IOException { + final Map> docValues = new HashMap<>(); + for (LeafReaderContext ctx : reader.leaves()) { + + PointValues points = ctx.reader().getPointValues(fieldName); + if (points == null) { + continue; + } + + points.intersect( + new PointValues.IntersectVisitor() { + @Override + public void visit(int docID) { + throw new UnsupportedOperationException(); + } + + @Override + public void visit(int docID, byte[] packedValue) throws IOException { + int topDocID = ctx.docBase + docID; + if (docValues.containsKey(topDocID) == false) { + docValues.put(topDocID, new HashSet<>()); + } + docValues.get(topDocID).add(new BytesRef(packedValue.clone())); + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + // We pretend our query shape is so hairy that it crosses every single cell: + return PointValues.Relation.CELL_CROSSES_QUERY; + } + }); + } + + return docValues; + } + + public void assertPointsEquals(String info, IndexReader leftReader, IndexReader rightReader) + throws IOException { + FieldInfos fieldInfos1 = FieldInfos.getMergedFieldInfos(leftReader); + FieldInfos fieldInfos2 = FieldInfos.getMergedFieldInfos(rightReader); + for (FieldInfo fieldInfo1 : fieldInfos1) { + if (fieldInfo1.getPointDimensionCount() != 0) { + FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldInfo1.name); + // same data dimension count? + assertEquals( + info, fieldInfo2.getPointDimensionCount(), fieldInfo2.getPointDimensionCount()); + // same index dimension count? + assertEquals( + info, + fieldInfo2.getPointIndexDimensionCount(), + fieldInfo2.getPointIndexDimensionCount()); + // same bytes per dimension? + assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo2.getPointNumBytes()); + + assertEquals( + info + " field=" + fieldInfo1.name, + uninvert(fieldInfo1.name, leftReader), + uninvert(fieldInfo1.name, rightReader)); + } + } + + // make sure FieldInfos2 doesn't have any point fields that FieldInfo1 didn't have + for (FieldInfo fieldInfo2 : fieldInfos2) { + if (fieldInfo2.getPointDimensionCount() != 0) { + FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldInfo2.name); + // same data dimension count? + assertEquals( + info, fieldInfo2.getPointDimensionCount(), fieldInfo1.getPointDimensionCount()); + // same index dimension count? + assertEquals( + info, + fieldInfo2.getPointIndexDimensionCount(), + fieldInfo1.getPointIndexDimensionCount()); + // same bytes per dimension? + assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo1.getPointNumBytes()); + + // we don't need to uninvert and compare here ... we did that in the first loop above + } + } + } + + /** + * Creates leaf slices according to the concurrency argument, that optionally leverage + * intra-segment concurrency by splitting segments into multiple partitions according to the + * maxDocsPerSlice argument. + */ + protected static IndexSearcher.LeafSlice[] slices( + List leaves, + int maxDocsPerSlice, + int maxSegmentsPerSlice, + LuceneTestCase.Concurrency concurrency) { + assert concurrency != Concurrency.NONE; + // Rarely test slices without partitions even though intra-segment concurrency is supported + return IndexSearcher.slices( + leaves, + maxDocsPerSlice, + maxSegmentsPerSlice, + concurrency == Concurrency.INTRA_SEGMENT && frequently()); + } + + /** + * Gets a resource from the test's classpath as {@link Path}. This method should only be used, if + * a real file is needed. To get a stream, code should prefer {@link #getDataInputStream(String)}. + */ + protected Path getDataPath(String name) throws IOException { + try { + return Paths.get( + IOUtils.requireResourceNonNull(this.getClass().getResource(name), name).toURI()); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + } + + /** + * Creates a {@link BytesRef} holding UTF-8 bytes for the incoming String, that sometimes uses a + * non-zero {@code offset}, and non-zero end-padding, to tickle latent bugs that fail to look at + * {@code BytesRef.offset}. + */ + public static BytesRef newBytesRef(String s) { + return newBytesRef(s.getBytes(StandardCharsets.UTF_8)); + } + + /** + * Creates a copy of the incoming {@link BytesRef} that sometimes uses a non-zero {@code offset}, + * and non-zero end-padding, to tickle latent bugs that fail to look at {@code BytesRef.offset}. + */ + public static BytesRef newBytesRef(BytesRef b) { + assert b.isValid(); + return newBytesRef(b.bytes, b.offset, b.length); + } + + /** + * Creates a random BytesRef from the incoming bytes that sometimes uses a non-zero {@code + * offset}, and non-zero end-padding, to tickle latent bugs that fail to look at {@code + * BytesRef.offset}. + */ + public static BytesRef newBytesRef(byte[] b) { + return newBytesRef(b, 0, b.length); + } + + /** + * Creates a random empty BytesRef that sometimes uses a non-zero {@code offset}, and non-zero + * end-padding, to tickle latent bugs that fail to look at {@code BytesRef.offset}. + */ + public static BytesRef newBytesRef() { + return newBytesRef(new byte[0], 0, 0); + } + + /** + * Creates a random empty BytesRef, with at least the requested length of bytes free, that + * sometimes uses a non-zero {@code offset}, and non-zero end-padding, to tickle latent bugs that + * fail to look at {@code BytesRef.offset}. + */ + public static BytesRef newBytesRef(int byteLength) { + return newBytesRef(new byte[byteLength], 0, byteLength); + } + + /** + * Creates a copy of the incoming bytes slice that sometimes uses a non-zero {@code offset}, and + * non-zero end-padding, to tickle latent bugs that fail to look at {@code BytesRef.offset}. + */ + public static BytesRef newBytesRef(byte[] bytesIn, int offset, int length) { + // System.out.println("LTC.newBytesRef! bytesIn.length=" + bytesIn.length + " offset=" + offset + // + " length=" + length); + + assert bytesIn.length >= offset + length + : "got offset=" + offset + " length=" + length + " bytesIn.length=" + bytesIn.length; + + // randomly set a non-zero offset + int startOffset; + if (random().nextBoolean()) { + startOffset = RandomNumbers.randomIntBetween(random(), 1, 20); + } else { + startOffset = 0; + } + + // also randomly set an end padding: + int endPadding; + if (random().nextBoolean()) { + endPadding = RandomNumbers.randomIntBetween(random(), 1, 20); + } else { + endPadding = 0; + } + + byte[] bytes = new byte[startOffset + length + endPadding]; + + System.arraycopy(bytesIn, offset, bytes, startOffset, length); + // System.out.println("LTC: return bytes.length=" + bytes.length + " startOffset=" + + // startOffset + " length=" + length); + + BytesRef it = new BytesRef(bytes, startOffset, length); + assert it.isValid(); + + if (RandomNumbers.randomIntBetween(random(), 1, 17) == 7) { + // try to ferret out bugs in this method too! + return newBytesRef(it.bytes, it.offset, it.length); + } + + return it; + } + + /** + * Returns a set of JVM arguments to fork a JVM with the same class or module path (including any + * associated JVM options). The returned value may be empty. This method may throw an assertion + * error if fork options cannot be reliably acquired (at the moment they are collected and passed + * as an external file in gradle scripts). + * + *

JVM forking is strongly discouraged as it makes test slower and more resource-hungry. + * Consider all alternatives first. + */ + public static List getJvmForkArguments() throws IOException { + String forkArgsFile = System.getProperty("tests.jvmForkArgsFile"); + Path forkArgsPath; + if (forkArgsFile == null || !Files.isRegularFile(forkArgsPath = Paths.get(forkArgsFile))) { + throw new AssertionError("JVM fork arguments are not present."); + } + + return Files.readAllLines(forkArgsPath, StandardCharsets.UTF_8); + } + + /** + * Compares two strings with a collator, also looking to see if the strings are impacted by jdk + * bugs. may not avoid all jdk bugs in tests. see https://bugs.openjdk.java.net/browse/JDK-8071862 + */ + @SuppressForbidden(reason = "dodges JDK-8071862") + public static int collate(Collator collator, String s1, String s2) { + int v1 = collator.compare(s1, s2); + int v2 = collator.getCollationKey(s1).compareTo(collator.getCollationKey(s2)); + // if collation keys don't really respect collation order, things are screwed. + assumeTrue("hit JDK collator bug", Integer.signum(v1) == Integer.signum(v2)); + return v1; + } + + /** + * Returns a number of at least i + * + *

The actual number returned will be influenced by whether {@link #TEST_NIGHTLY} is active and + * {@link #RANDOM_MULTIPLIER}, but also with some random fudge. + */ + public static int atLeast(Random random, int i) { + int min = i * RANDOM_MULTIPLIER; + int max = min + (min / 2); + return TestUtil.nextInt(random, min, max); + } + + public static int atLeast(int i) { + return atLeast(random(), i); + } + + /** + * Returns true if something should happen rarely, + * + *

The actual number returned will be influenced by whether {@link #TEST_NIGHTLY} is active and + * {@link #RANDOM_MULTIPLIER}. + */ + public static boolean rarely(Random random) { + int p = TEST_NIGHTLY ? 5 : 1; + p += (p * Math.log(RANDOM_MULTIPLIER)); + int min = 100 - Math.min(p, 20); // never more than 20 + return random.nextInt(100) >= min; + } + + public static boolean rarely() { + return rarely(random()); + } + + public static boolean usually(Random random) { + return !rarely(random); + } + + public static boolean usually() { + return usually(random()); + } + + public static void assumeTrue(String msg, boolean condition) { + RandomizedTest.assumeTrue(msg, condition); + } + + public static void assumeFalse(String msg, boolean condition) { + RandomizedTest.assumeFalse(msg, condition); + } + + public static void assumeNoException(String msg, Exception e) { + RandomizedTest.assumeNoException(msg, e); + } + + public static void assertFloatUlpEquals(final float x, final float y, final short maxUlps) { + assertTrue( + x + " and " + y + " are not within " + maxUlps + " ULPs of each other", + TestUtil.floatUlpEquals(x, y, maxUlps)); + } + + public static void assertDoubleUlpEquals(final double x, final double y, final int maxUlps) { + assertTrue( + x + " and " + y + " are not within " + maxUlps + " ULPs of each other", + TestUtil.doubleUlpEquals(x, y, maxUlps)); + } + + /** + * Return args as a {@link Set} instance. The order of elements is not preserved in + * iterators. + */ + @SafeVarargs + @SuppressWarnings("varargs") + public static Set asSet(T... args) { + return new HashSet<>(Arrays.asList(args)); + } + + /** + * Convenience method for logging an iterator. + * + * @param label String logged before/after the items in the iterator + * @param iter Each next() is toString()ed and logged on its own line. If iter is null this is + * logged differently then an empty iterator. + * @param stream Stream to log messages to. + */ + public static void dumpIterator(String label, Iterator iter, PrintStream stream) { + stream.println("*** BEGIN " + label + " ***"); + if (null == iter) { + stream.println(" ... NULL ..."); + } else { + while (iter.hasNext()) { + stream.println(iter.next().toString()); + } + } + stream.println("*** END " + label + " ***"); + } + + /** + * Convenience method for logging an array. Wraps the array in an iterator and delegates + * + * @see #dumpIterator(String, Iterator, PrintStream) + */ + public static void dumpArray(String label, Object[] objs, PrintStream stream) { + Iterator iter = (null == objs) ? null : Arrays.asList(objs).iterator(); + dumpIterator(label, iter, stream); + } + + /** + * Returns true if the file exists (can be opened), false if it cannot be opened, and (unlike + * Java's File.exists) throws IOException if there's some unexpected error. + */ + public static boolean slowFileExists(Directory dir, String fileName) throws IOException { + try { + dir.openInput(fileName, IOContext.READONCE).close(); + return true; + } catch (NoSuchFileException | FileNotFoundException _) { + return false; + } + } + + /** A runnable that can throw any checked exception. */ + @FunctionalInterface + public interface ThrowingRunnable { + void run() throws Throwable; + } + + /** A {@link java.util.function.Consumer} that can throw any checked exception. */ + @FunctionalInterface + public interface ThrowingConsumer { + void accept(T t) throws Exception; + } + + /** Checks a specific exception class is thrown by the given runnable, and returns it. */ + public static T expectThrows( + Class expectedType, ThrowingRunnable runnable) { + return expectThrows( + expectedType, + "Expected exception " + expectedType.getSimpleName() + " but no exception was thrown", + runnable); + } + + /** Checks a specific exception class is thrown by the given runnable, and returns it. */ + public static T expectThrows( + Class expectedType, String noExceptionMessage, ThrowingRunnable runnable) { + final Throwable thrown = _expectThrows(Collections.singletonList(expectedType), runnable); + if (expectedType.isInstance(thrown)) { + return expectedType.cast(thrown); + } + if (null == thrown) { + throw new AssertionFailedError(noExceptionMessage); + } + AssertionFailedError assertion = + new AssertionFailedError( + "Unexpected exception type, expected " + + expectedType.getSimpleName() + + " but got " + + thrown); + assertion.initCause(thrown); + throw assertion; + } + + /** Checks a specific exception class is thrown by the given runnable, and returns it. */ + public static T expectThrowsAnyOf( + List> expectedTypes, ThrowingRunnable runnable) { + if (expectedTypes.isEmpty()) { + throw new AssertionError("At least one expected exception type is required?"); + } + + final Throwable thrown = _expectThrows(expectedTypes, runnable); + if (null != thrown) { + for (Class expectedType : expectedTypes) { + if (expectedType.isInstance(thrown)) { + return expectedType.cast(thrown); + } + } + } + + List exceptionTypes = expectedTypes.stream().map(Class::getSimpleName).toList(); + + if (thrown != null) { + AssertionFailedError assertion = + new AssertionFailedError( + "Unexpected exception type, expected any of " + + exceptionTypes + + " but got: " + + thrown); + assertion.initCause(thrown); + throw assertion; + } else { + throw new AssertionFailedError( + "Expected any of the following exception types: " + + exceptionTypes + + " but no exception was thrown."); + } + } + + /** + * Checks that specific wrapped and outer exception classes are thrown by the given runnable, and + * returns the wrapped exception. + */ + public static TW expectThrows( + Class expectedOuterType, Class expectedWrappedType, ThrowingRunnable runnable) { + final Throwable thrown = _expectThrows(Collections.singletonList(expectedOuterType), runnable); + if (null == thrown) { + throw new AssertionFailedError( + "Expected outer exception " + + expectedOuterType.getSimpleName() + + " but no exception was thrown."); + } + if (expectedOuterType.isInstance(thrown)) { + Throwable cause = thrown.getCause(); + if (expectedWrappedType.isInstance(cause)) { + return expectedWrappedType.cast(cause); + } else { + AssertionFailedError assertion = + new AssertionFailedError( + "Unexpected wrapped exception type, expected " + + expectedWrappedType.getSimpleName() + + " but got: " + + cause); + assertion.initCause(thrown); + throw assertion; + } + } + AssertionFailedError assertion = + new AssertionFailedError( + "Unexpected outer exception type, expected " + + expectedOuterType.getSimpleName() + + " but got: " + + thrown); + assertion.initCause(thrown); + throw assertion; + } + + /** + * Checks that one of the specified wrapped and outer exception classes are thrown by the given + * runnable, and returns the outer exception. + * + *

This method accepts outer exceptions with no wrapped exception; an empty list of expected + * wrapped exception types indicates no wrapped exception. + */ + public static TO expectThrowsAnyOf( + LinkedHashMap, List>> expectedOuterToWrappedTypes, + ThrowingRunnable runnable) { + final List> outerClasses = + new ArrayList<>(expectedOuterToWrappedTypes.keySet()); + final Throwable thrown = _expectThrows(outerClasses, runnable); + + if (null == thrown) { + List outerTypes = outerClasses.stream().map(Class::getSimpleName).toList(); + throw new AssertionFailedError( + "Expected any of the following outer exception types: " + + outerTypes + + " but no exception was thrown."); + } + for (Map.Entry, List>> entry : + expectedOuterToWrappedTypes.entrySet()) { + Class expectedOuterType = entry.getKey(); + List> expectedWrappedTypes = entry.getValue(); + Throwable cause = thrown.getCause(); + if (expectedOuterType.isInstance(thrown)) { + if (expectedWrappedTypes.isEmpty()) { + return null; // no wrapped exception + } else { + for (Class expectedWrappedType : expectedWrappedTypes) { + if (expectedWrappedType.isInstance(cause)) { + return expectedOuterType.cast(thrown); + } + } + List wrappedTypes = + expectedWrappedTypes.stream().map(Class::getSimpleName).toList(); + AssertionFailedError assertion = + new AssertionFailedError( + "Unexpected wrapped exception type, expected one of " + + wrappedTypes + + " but got: " + + cause); + assertion.initCause(thrown); + throw assertion; + } + } + } + List outerTypes = outerClasses.stream().map(Class::getSimpleName).toList(); + AssertionFailedError assertion = + new AssertionFailedError( + "Unexpected outer exception type, expected one of " + + outerTypes + + " but got: " + + thrown); + assertion.initCause(thrown); + throw assertion; + } + + /** + * Helper method for {@link #expectThrows} and {@link #expectThrowsAnyOf} that takes care of + * propagating any {@link AssertionError} or {@link AssumptionViolatedException} instances thrown + * if and only if they are super classes of the expectedTypes. Otherwise simply + * returns any {@link Throwable} thrown, regardless of type, or null if the runnable + * completed w/o error. + */ + private static Throwable _expectThrows( + List> expectedTypes, ThrowingRunnable runnable) { + + try { + runnable.run(); + } catch (AssertionError | AssumptionViolatedException ae) { + for (Class expectedType : expectedTypes) { + if (expectedType.isInstance(ae)) { // user is expecting this type explicitly + return ae; + } + } + throw ae; + } catch (Throwable e) { + return e; + } + return null; + } + + private static final StackWalker + SW_NO_METHODS = StackWalker.getInstance(StackWalker.Option.DROP_METHOD_INFO), + SW_WITH_METHODS = StackWalker.getInstance(); + + /** Inspects stack trace to figure out if a method of a specific class called us. */ + public static boolean callStackContains(Class clazz, String methodName) { + final String className = clazz.getName(); + return SW_WITH_METHODS.walk( + s -> + s.skip(1) // exclude this utility method + .anyMatch( + f -> + className.equals(f.getClassName()) + && methodName.equals(f.getMethodName()))); + } + + /** + * Inspects stack trace to figure out if one of the given method names (no class restriction) + * called us. + */ + public static boolean callStackContainsAnyOf(String... methodNames) { + return SW_WITH_METHODS.walk( + s -> + s.skip(1) // exclude this utility method + .map(StackWalker.StackFrame::getMethodName) + .anyMatch(Set.of(methodNames)::contains)); + } + + /** Inspects stack trace if the given class called us. */ + public static boolean callStackContains(Class clazz) { + return SW_NO_METHODS.walk( + s -> + s.skip(1) // exclude this utility method + .map(StackWalker.StackFrame::getClassName) + .anyMatch(clazz.getName()::equals)); + } + + /** + * Tells {@link IndexWriter} to enforce the specified limit as the maximum number of documents in + * one index; call {@link #restoreIndexWriterMaxDocs} once your test is done. + */ + public void setIndexWriterMaxDocs(int limit) { + INDEX_PACKAGE_ACCESS.setIndexWriterMaxDocs(limit); + } + + /** Returns to the default {@link IndexWriter#MAX_DOCS} limit. */ + public void restoreIndexWriterMaxDocs() { + INDEX_PACKAGE_ACCESS.setIndexWriterMaxDocs(IndexWriter.MAX_DOCS); + } + + private static final IndexPackageAccess INDEX_PACKAGE_ACCESS = + TestSecrets.getIndexPackageAccess(); + + private static void configureRandomCompoundFormat(Random r, CompoundFormat compoundFormat) { + compoundFormat.setShouldUseCompoundFile(r.nextBoolean()); + + if (rarely(r)) { + compoundFormat.setMaxCFSSegmentSizeMB(0.2 + r.nextDouble() * 2.0); + } else { + compoundFormat.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY); + } + } + + /** + * Some tests expect the directory to contain a single segment, and want to do tests on that + * segment's reader. This is an utility method to help them. + */ + public static LeafReader getOnlyLeafReader(IndexReader reader) { + List subReaders = reader.leaves(); + if (subReaders.size() != 1) { + throw new IllegalArgumentException( + reader + " has " + subReaders.size() + " segments instead of exactly one"); + } + return subReaders.get(0).reader(); + } + + /** create a new index writer config with a snapshot deletion policy */ + public static IndexWriterConfig newSnapshotIndexWriterConfig(Analyzer analyzer) { + IndexWriterConfig c = newIndexWriterConfig(analyzer); + c.setIndexDeletionPolicy(new SnapshotDeletionPolicy(NoDeletionPolicy.INSTANCE)); + return c; + } + + /** create a new index writer config with random defaults */ + public static IndexWriterConfig newIndexWriterConfig() { + return newIndexWriterConfig(new MockAnalyzer(random())); + } + + /** create a new index writer config with random defaults */ + public static IndexWriterConfig newIndexWriterConfig(Analyzer a) { + return newIndexWriterConfig(random(), a); + } + + /** create a new index writer config with random defaults using the specified random */ + public static IndexWriterConfig newIndexWriterConfig(Random r, Analyzer a) { + IndexWriterConfig c = new IndexWriterConfig(a); + configureRandomCompoundFormat(r, c.getCodec().compoundFormat()); + c.setSimilarity(getTestFrameworkInfra().getClassEnv().similarity); + if (INFOSTREAM) { + // Even though TestRuleSetupAndRestoreClassEnv calls + // InfoStream.setDefault, we do it again here so that + // the PrintStreamInfoStream.messageID increments so + // that when there are separate instances of + // IndexWriter created we see "IW 0", "IW 1", "IW 2", + // ... instead of just always "IW 0": + c.setInfoStream( + new SetupAndRestoreStaticEnv.ThreadNameFixingPrintStreamInfoStream(System.out)); + } + + if (rarely(r)) { + c.setMergeScheduler(new SerialMergeScheduler()); + } else if (rarely(r)) { + ConcurrentMergeScheduler cms; + if (r.nextBoolean()) { + cms = new IntraMergeConcurrentMergeScheduler(); + } else { + cms = + new IntraMergeConcurrentMergeScheduler() { + @Override + protected synchronized boolean maybeStall(MergeSource mergeSource) { + return true; + } + }; + } + int maxThreadCount = TestUtil.nextInt(r, 1, 4); + int maxMergeCount = TestUtil.nextInt(r, maxThreadCount, maxThreadCount + 4); + cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount); + if (r.nextBoolean()) { + cms.disableAutoIOThrottle(); + assertFalse(cms.getAutoIOThrottle()); + } + cms.setForceMergeMBPerSec(10 + 10 * r.nextDouble()); + c.setMergeScheduler(cms); + } else { + // Always use consistent settings, else CMS's dynamic (SSD or not) + // defaults can change, hurting reproducibility: + ConcurrentMergeScheduler cms = + r.nextBoolean() + ? new IntraMergeConcurrentMergeScheduler() + : new ConcurrentMergeScheduler(); + + // Only 1 thread can run at once (should maybe help reproducibility), + // with up to 3 pending merges before segment-producing threads are + // stalled: + cms.setMaxMergesAndThreads(3, 1); + c.setMergeScheduler(cms); + } + + if (r.nextBoolean()) { + if (rarely(r)) { + // crazy value + c.setMaxBufferedDocs(TestUtil.nextInt(r, 2, 15)); + } else { + // reasonable value + c.setMaxBufferedDocs(TestUtil.nextInt(r, 16, 1000)); + } + } + + c.setMergePolicy(newMergePolicy(r)); + + if (rarely(r)) { + c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream())); + } + c.setUseCompoundFile(r.nextBoolean()); + c.setReaderPooling(r.nextBoolean()); + if (rarely(r)) { + c.setCheckPendingFlushUpdate(false); + } + + if (rarely(r)) { + c.setIndexWriterEventListener(new MockIndexWriterEventListener()); + } + switch (r.nextInt(3)) { + case 0: + // Disable merge on refresh + c.setMaxFullFlushMergeWaitMillis(0L); + break; + case 1: + // Very low timeout, merges will likely not be able to run in time + c.setMaxFullFlushMergeWaitMillis(1L); + break; + default: + // Very long timeout, merges will almost always be able to run in time + c.setMaxFullFlushMergeWaitMillis(1000L); + break; + } + + c.setMaxFullFlushMergeWaitMillis(rarely(r) ? atLeast(r, 1000) : atLeast(r, 200)); + return c; + } + + public static MergePolicy newMergePolicy(Random r) { + return newMergePolicy(r, true); + } + + public static MergePolicy newMergePolicy(Random r, boolean includeMockMP) { + if (includeMockMP && rarely(r)) { + return new MockRandomMergePolicy(r); + } else if (r.nextBoolean()) { + return newTieredMergePolicy(r); + } else if (rarely(r)) { + return newAlcoholicMergePolicy(r, getTestFrameworkInfra().getClassEnv().timeZone); + } + return newLogMergePolicy(r); + } + + public static MergePolicy newMergePolicy() { + return newMergePolicy(random()); + } + + public static LogMergePolicy newLogMergePolicy() { + return newLogMergePolicy(random()); + } + + public static TieredMergePolicy newTieredMergePolicy() { + return newTieredMergePolicy(random()); + } + + public static AlcoholicMergePolicy newAlcoholicMergePolicy() { + return newAlcoholicMergePolicy(random(), getTestFrameworkInfra().getClassEnv().timeZone); + } + + public static AlcoholicMergePolicy newAlcoholicMergePolicy(Random r, TimeZone tz) { + return new AlcoholicMergePolicy(tz, new Random(r.nextLong())); + } + + public static LogMergePolicy newLogMergePolicy(Random r) { + LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy(); + logmp.setCalibrateSizeByDeletes(r.nextBoolean()); + logmp.setTargetSearchConcurrency(TestUtil.nextInt(r, 1, 16)); + if (rarely(r)) { + logmp.setMergeFactor(TestUtil.nextInt(r, 2, 9)); + } else { + logmp.setMergeFactor(TestUtil.nextInt(r, 10, 50)); + } + return logmp; + } + + public static TieredMergePolicy newTieredMergePolicy(Random r) { + TieredMergePolicy tmp = new TieredMergePolicy(); + if (rarely(r)) { + tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0); + } else { + tmp.setMaxMergedSegmentMB(10 + r.nextDouble() * 100); + } + tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0); + tmp.setForceMergeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0); + if (rarely(r)) { + tmp.setSegmentsPerTier(TestUtil.nextInt(r, 2, 20)); + } else { + tmp.setSegmentsPerTier(TestUtil.nextInt(r, 10, 50)); + } + if (rarely(r)) { + tmp.setTargetSearchConcurrency(TestUtil.nextInt(r, 10, 50)); + } else { + tmp.setTargetSearchConcurrency(TestUtil.nextInt(r, 2, 20)); + } + + tmp.setDeletesPctAllowed(20 + r.nextDouble() * 30); + return tmp; + } + + public static LogMergePolicy newLogMergePolicy(int mergeFactor) { + LogMergePolicy logmp = newLogMergePolicy(); + logmp.setMergeFactor(mergeFactor); + return logmp; + } + + enum LiveIWCFlushMode { + BY_RAM, + BY_DOCS, + EITHER + } + + // if you want it in LiveIndexWriterConfig: it must and will be tested here. + public static void maybeChangeLiveIndexWriterConfig(Random r, LiveIndexWriterConfig c) { + boolean didChange = false; + + String previous = c.toString(); + + if (rarely(r)) { + // change flush parameters: + // this is complicated because the api requires you "invoke setters in a magical order!" + // LUCENE-5661: workaround for race conditions in the API + synchronized (c) { + boolean flushByRAM; + switch (getTestFrameworkInfra().getClassEnv().getLiveIWCFlushMode()) { + case BY_RAM: + flushByRAM = true; + break; + case BY_DOCS: + flushByRAM = false; + break; + case EITHER: + flushByRAM = r.nextBoolean(); + break; + default: + throw new AssertionError(); + } + if (flushByRAM) { + c.setRAMBufferSizeMB(TestUtil.nextInt(r, 1, 10)); + c.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); + } else { + if (rarely(r)) { + // crazy value + c.setMaxBufferedDocs(TestUtil.nextInt(r, 2, 15)); + } else { + // reasonable value + c.setMaxBufferedDocs(TestUtil.nextInt(r, 16, 1000)); + } + c.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + } + } + didChange = true; + } + + if (rarely(r)) { + IndexWriter.IndexReaderWarmer curWarmer = c.getMergedSegmentWarmer(); + if (curWarmer == null || curWarmer instanceof SimpleMergedSegmentWarmer) { + // change warmer parameters + if (r.nextBoolean()) { + c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream())); + } else { + c.setMergedSegmentWarmer(null); + } + } + didChange = true; + } + + if (rarely(r)) { + // change CFS flush parameters + c.setUseCompoundFile(r.nextBoolean()); + didChange = true; + } + + if (rarely(r)) { + // change CMS merge parameters + MergeScheduler ms = c.getMergeScheduler(); + if (ms instanceof ConcurrentMergeScheduler cms) { + int maxThreadCount = TestUtil.nextInt(r, 1, 4); + int maxMergeCount = TestUtil.nextInt(r, maxThreadCount, maxThreadCount + 4); + boolean enableAutoIOThrottle = r.nextBoolean(); + if (enableAutoIOThrottle) { + cms.enableAutoIOThrottle(); + } else { + cms.disableAutoIOThrottle(); + } + cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount); + didChange = true; + } + } + + if (rarely(r)) { + MergePolicy mp = c.getMergePolicy(); + configureRandomCompoundFormat(r, c.getCodec().compoundFormat()); + if (mp instanceof LogMergePolicy logmp) { + logmp.setCalibrateSizeByDeletes(r.nextBoolean()); + if (rarely(r)) { + logmp.setMergeFactor(TestUtil.nextInt(r, 2, 9)); + } else { + logmp.setMergeFactor(TestUtil.nextInt(r, 10, 50)); + } + } else if (mp instanceof TieredMergePolicy tmp) { + if (rarely(r)) { + tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0); + } else { + tmp.setMaxMergedSegmentMB(r.nextDouble() * 100); + } + tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0); + tmp.setForceMergeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0); + if (rarely(r)) { + tmp.setSegmentsPerTier(TestUtil.nextInt(r, 2, 20)); + } else { + tmp.setSegmentsPerTier(TestUtil.nextInt(r, 10, 50)); + } + configureRandomCompoundFormat(r, c.getCodec().compoundFormat()); + tmp.setDeletesPctAllowed(20 + r.nextDouble() * 30); + } + didChange = true; + } + if (VERBOSE && didChange) { + String current = c.toString(); + String[] previousLines = previous.split("\n"); + String[] currentLines = current.split("\n"); + StringBuilder diff = new StringBuilder(); + + // this should always be the case, diff each line + if (previousLines.length == currentLines.length) { + for (int i = 0; i < previousLines.length; i++) { + if (!previousLines[i].equals(currentLines[i])) { + diff.append("- ").append(previousLines[i]).append("\n"); + diff.append("+ ").append(currentLines[i]).append("\n"); + } + } + } else { + // but just in case of something ridiculous... + diff.append(current); + } + + // its possible to be empty, if we "change" a value to what it had before. + if (diff.length() > 0) { + System.out.println("NOTE: LuceneTestCase: randomly changed IWC's live settings:"); + System.out.println(diff); + } + } + } + + public static Field newStringField(String name, String value, Field.Store stored) { + return newField( + random(), + name, + value, + stored == Field.Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); + } + + public static Field newStringField(String name, BytesRef value, Field.Store stored) { + return newField( + random(), + name, + value, + stored == Field.Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); + } + + public static Field newTextField(String name, String value, Field.Store stored) { + return newField( + random(), + name, + value, + stored == Field.Store.YES ? TextField.TYPE_STORED : TextField.TYPE_NOT_STORED); + } + + public static Field newStringField(Random random, String name, String value, Field.Store stored) { + return newField( + random, + name, + value, + stored == Field.Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); + } + + public static Field newStringField( + Random random, String name, BytesRef value, Field.Store stored) { + return newField( + random, + name, + value, + stored == Field.Store.YES ? StringField.TYPE_STORED : StringField.TYPE_NOT_STORED); + } + + public static Field newTextField(Random random, String name, String value, Field.Store stored) { + return newField( + random, + name, + value, + stored == Field.Store.YES ? TextField.TYPE_STORED : TextField.TYPE_NOT_STORED); + } + + public static Field newField(String name, String value, FieldType type) { + return newField(random(), name, value, type); + } + + public static Field newField(Random random, String name, Object value, FieldType type) { + // TODO: if we can pull out the "make term vector options + // consistent across all instances of the same field name" + // write-once schema sort of helper class then we can + // remove the sync here. We can also fold the random + // "enable norms" (now commented out, below) into that: + return getTestFrameworkInfra().newField(random, name, value, type); + } + + private static final String[] availableLanguageTags = + Arrays.stream(Locale.getAvailableLocales()) + .map(Locale::toLanguageTag) + .sorted() + .distinct() + .toArray(String[]::new); + + /** + * Return a random Locale from the available locales on the system. + * + * @see LUCENE-4020 + */ + public static Locale randomLocale(Random random) { + return localeForLanguageTag( + availableLanguageTags[random.nextInt(availableLanguageTags.length)]); + } + + /** Time zone IDs that cause a deprecation warning in JDK 25. */ + private static final Set DEPRECATED_TIME_ZONE_IDS_JDK25 = + Set.of( + "ACT", "AET", "AGT", "ART", "AST", "BET", "BST", "CAT", "CNT", "CST", "CTT", "EAT", "ECT", + "EST", "HST", "IET", "IST", "JST", "MIT", "MST", "NET", "NST", "PLT", "PNT", "PRT", "PST", + "SST", "VST"); + + /** + * Return a random TimeZone from the available timezones on the system + * + * @see LUCENE-4020 + */ + public static TimeZone randomTimeZone(Random random) { + List tzIds = Arrays.asList(TimeZone.getAvailableIDs()); + // Remove time zones that cause deprecation warnings as these can break + // certain tests that expect exact output. + if (Runtime.version().feature() >= 25) { + tzIds = tzIds.stream().filter(id -> !DEPRECATED_TIME_ZONE_IDS_JDK25.contains(id)).toList(); + } + return TimeZone.getTimeZone(RandomPicks.randomFrom(random, tzIds)); + } + + /** return a Locale object equivalent to its programmatic name */ + public static Locale localeForLanguageTag(String languageTag) { + return new Locale.Builder().setLanguageTag(languageTag).build(); + } + + /** + * Returns a new Directory instance. Use this when the test does not care about the specific + * Directory implementation (most tests). + * + *

The Directory is wrapped with {@link BaseDirectoryWrapper}. this means usually it will be + * picky, such as ensuring that you properly close it and all open files in your test. It will + * emulate some features of Windows, such as not allowing open files to be overwritten. + */ + public static BaseDirectoryWrapper newDirectory() { + return newDirectory(random()); + } + + /** Like {@link #newDirectory} except randomly the {@link VirusCheckingFS} may be installed */ + public static BaseDirectoryWrapper newMaybeVirusCheckingDirectory() { + if (random().nextInt(5) == 4) { + Path path = addVirusChecker(createTempDir()); + return newFSDirectory(path); + } else { + return newDirectory(random()); + } + } + + /** + * Returns a new Directory instance, using the specified random. See {@link #newDirectory()} for + * more information. + */ + public static BaseDirectoryWrapper newDirectory(Random r) { + return wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY), rarely(r), false); + } + + /** + * Returns a new Directory instance, using the specified random. See {@link #newDirectory()} for + * more information. + */ + public static BaseDirectoryWrapper newDirectory(Random r, LockFactory lf) { + return wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY, lf), rarely(r), false); + } + + public static MockDirectoryWrapper newMockDirectory() { + return newMockDirectory(random()); + } + + public static MockDirectoryWrapper newMockDirectory(Random r) { + return (MockDirectoryWrapper) + wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY), false, false); + } + + public static MockDirectoryWrapper newMockDirectory(Random r, LockFactory lf) { + return (MockDirectoryWrapper) + wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY, lf), false, false); + } + + public static MockDirectoryWrapper newMockFSDirectory(Path f) { + return (MockDirectoryWrapper) newFSDirectory(f, FSLockFactory.getDefault(), false); + } + + public static MockDirectoryWrapper newMockFSDirectory(Path f, LockFactory lf) { + return (MockDirectoryWrapper) newFSDirectory(f, lf, false); + } + + public static Path addVirusChecker(Path path) { + if (TestUtil.hasVirusChecker(path) == false) { + VirusCheckingFS fs = new VirusCheckingFS(path.getFileSystem(), random().nextLong()); + path = fs.wrapPath(path); + } + return path; + } + + /** + * Returns a new Directory instance, with contents copied from the provided directory. See {@link + * #newDirectory()} for more information. + */ + public static BaseDirectoryWrapper newDirectory(Directory d) throws IOException { + return newDirectory(random(), d); + } + + /** Returns a new FSDirectory instance over the given file, which must be a folder. */ + public static BaseDirectoryWrapper newFSDirectory(Path f) { + return newFSDirectory(f, FSLockFactory.getDefault()); + } + + /** Like {@link #newFSDirectory(Path)}, but randomly insert {@link VirusCheckingFS} */ + public static BaseDirectoryWrapper newMaybeVirusCheckingFSDirectory(Path f) { + if (random().nextInt(5) == 4) { + f = addVirusChecker(f); + } + return newFSDirectory(f, FSLockFactory.getDefault()); + } + + /** Returns a new FSDirectory instance over the given file, which must be a folder. */ + public static BaseDirectoryWrapper newFSDirectory(Path f, LockFactory lf) { + return newFSDirectory(f, lf, rarely()); + } + + private static BaseDirectoryWrapper newFSDirectory(Path f, LockFactory lf, boolean bare) { + String fsdirClass = TEST_DIRECTORY; + if (fsdirClass.equals("random")) { + fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES); + } + + Class clazz; + try { + try { + clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass); + } catch (ClassCastException _) { + // TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random + fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES); + clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass); + } + + Directory fsdir = newFSDirectoryImpl(clazz, f, lf); + return wrapDirectory(random(), fsdir, bare, true); + } catch (Exception e) { + Rethrow.rethrow(e); + throw null; // dummy to prevent compiler failure + } + } + + private static Directory newFileSwitchDirectory(Random random, Directory dir1, Directory dir2) { + List fileExtensions = + Arrays.asList( + "fdt", "fdx", "tim", "tip", "si", "fnm", "pos", "dii", "dim", "nvm", "nvd", "dvm", + "dvd"); + Collections.shuffle(fileExtensions, random); + fileExtensions = fileExtensions.subList(0, 1 + random.nextInt(fileExtensions.size())); + return new FileSwitchDirectory(new HashSet<>(fileExtensions), dir1, dir2, true); + } + + /** + * Returns a new Directory instance, using the specified random with contents copied from the + * provided directory. See {@link #newDirectory()} for more information. + */ + public static BaseDirectoryWrapper newDirectory(Random r, Directory d) throws IOException { + Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); + for (String file : d.listAll()) { + if (file.startsWith(IndexFileNames.SEGMENTS) + || IndexFileNames.CODEC_FILE_PATTERN.matcher(file).matches()) { + impl.copyFrom(d, file, file, newIOContext(r)); + } + } + return wrapDirectory(r, impl, rarely(r), false); + } + + private static BaseDirectoryWrapper wrapDirectory( + Random random, Directory directory, boolean bare, boolean filesystem) { + // IOContext randomization might make NRTCachingDirectory make bad decisions, so avoid + // using it if the user requested a filesystem directory. + if (rarely(random) && !bare && filesystem == false) { + directory = new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble()); + } + + BaseDirectoryWrapper dir; + if (bare) { + dir = new RawDirectoryWrapper(directory); + } else { + MockDirectoryWrapper mock = new MockDirectoryWrapper(random, directory); + mock.setThrottling(TEST_THROTTLING); + dir = mock; + } + + getTestFrameworkInfra() + .getTempFilesSupplier() + .registerToCloseAfterSuite( + new AssertDirectoryClosed(dir, getTestFrameworkInfra().getSuiteFailureState())); + + return dir; + } + + private static Directory newFSDirectoryImpl( + Class clazz, Path path, LockFactory lf) throws IOException { + FSDirectory d = null; + try { + d = CommandLineUtil.newFSDirectory(clazz, path, lf); + } catch (ReflectiveOperationException e) { + Rethrow.rethrow(e); + } + return d; + } + + static Directory newDirectoryImpl(Random random, String clazzName) { + return newDirectoryImpl(random, clazzName, FSLockFactory.getDefault()); + } + + static Directory newDirectoryImpl(Random random, String clazzName, LockFactory lf) { + if (clazzName.equals("random")) { + if (rarely(random)) { + clazzName = RandomPicks.randomFrom(random, CORE_DIRECTORIES); + } else if (rarely(random)) { + String clazzName1 = + rarely(random) + ? RandomPicks.randomFrom(random, CORE_DIRECTORIES) + : ByteBuffersDirectory.class.getName(); + String clazzName2 = + rarely(random) + ? RandomPicks.randomFrom(random, CORE_DIRECTORIES) + : ByteBuffersDirectory.class.getName(); + return newFileSwitchDirectory( + random, + newDirectoryImpl(random, clazzName1, lf), + newDirectoryImpl(random, clazzName2, lf)); + } else { + clazzName = ByteBuffersDirectory.class.getName(); + } + } + + try { + final Class clazz = CommandLineUtil.loadDirectoryClass(clazzName); + // If it is a FSDirectory type, try its ctor(Path) + if (FSDirectory.class.isAssignableFrom(clazz)) { + final Path dir = createTempDir("index-" + clazzName); + return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), dir, lf); + } + + // See if it has a Path/LockFactory ctor even though it's not an + // FSDir subclass: + try { + Constructor pathCtor = + clazz.getConstructor(Path.class, LockFactory.class); + final Path dir = createTempDir("index"); + return pathCtor.newInstance(dir, lf); + } catch (NoSuchMethodException _) { + // Ignore + } + + // the remaining dirs are no longer filesystem based, so we must check that the + // passedLockFactory is not file based: + if (!(lf instanceof FSLockFactory)) { + // try ctor with only LockFactory + try { + return clazz.getConstructor(LockFactory.class).newInstance(lf); + } catch (NoSuchMethodException _) { + // Ignore + } + } + + // try empty ctor + return clazz.getConstructor().newInstance(); + } catch (Exception e) { + Rethrow.rethrow(e); + throw null; // dummy to prevent compiler failure + } + } + + public static IndexReader wrapReader(IndexReader r) throws IOException { + Random random = random(); + + for (int i = 0, c = random.nextInt(6) + 1; i < c; i++) { + switch (random.nextInt(5)) { + case 0: + // will create no FC insanity in atomic case, as ParallelLeafReader has own cache key: + if (VERBOSE) { + System.out.println( + "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + + r + + " with ParallelLeaf/CompositeReader"); + } + r = + (r instanceof LeafReader) + ? new ParallelLeafReader((LeafReader) r) + : new ParallelCompositeReader((CompositeReader) r); + break; + case 1: + if (r instanceof LeafReader ar) { + final List allFields = new ArrayList<>(); + for (FieldInfo fi : ar.getFieldInfos()) { + allFields.add(fi.name); + } + Collections.shuffle(allFields, random); + final int end = allFields.isEmpty() ? 0 : random.nextInt(allFields.size()); + final Set fields = new HashSet<>(allFields.subList(0, end)); + // will create no FC insanity as ParallelLeafReader has own cache key: + if (VERBOSE) { + System.out.println( + "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + + r + + " with ParallelLeafReader"); + } + r = + new ParallelLeafReader( + new FieldFilterLeafReader(ar, fields, false), + new FieldFilterLeafReader(ar, fields, true)); + } + break; + case 2: + // Häckidy-Hick-Hack: a standard Reader will cause FC insanity, so we use + // QueryUtils' reader with a fake cache key, so insanity checker cannot walk + // along our reader: + if (VERBOSE) { + System.out.println( + "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + + r + + " with AssertingLeaf/DirectoryReader"); + } + if (r instanceof LeafReader) { + r = new AssertingLeafReader((LeafReader) r); + } else if (r instanceof DirectoryReader) { + r = new AssertingDirectoryReader((DirectoryReader) r); + } + break; + case 3: + if (VERBOSE) { + System.out.println( + "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + + r + + " with MismatchedLeaf/Directory/CodecReader"); + } + if (r instanceof LeafReader) { + r = new MismatchedLeafReader((LeafReader) r, random); + } else if (r instanceof DirectoryReader) { + r = new MismatchedDirectoryReader((DirectoryReader) r, random); + } else if (r instanceof CodecReader) { + r = new MismatchedCodecReader((CodecReader) r, random); + } + break; + case 4: + if (VERBOSE) { + System.out.println( + "NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + + r + + " with MergingCodecReader"); + } + if (r instanceof CodecReader) { + r = new MergingCodecReader((CodecReader) r); + } else if (r instanceof DirectoryReader) { + boolean allLeavesAreCodecReaders = true; + for (LeafReaderContext ctx : r.leaves()) { + if (ctx.reader() instanceof CodecReader == false) { + allLeavesAreCodecReaders = false; + break; + } + } + if (allLeavesAreCodecReaders) { + r = new MergingDirectoryReaderWrapper((DirectoryReader) r); + } + } + break; + default: + fail("should not get here"); + } + } + + if (VERBOSE) { + System.out.println("wrapReader wrapped: " + r); + } + + return r; + } + + /** Sometimes wrap the IndexReader as slow, parallel or filter reader (or combinations of that) */ + public static IndexReader maybeWrapReader(IndexReader r) throws IOException { + if (rarely()) { + r = wrapReader(r); + } + return r; + } + + /** TODO: javadoc */ + public static IOContext newIOContext(Random random) { + return newIOContext(random, IOContext.DEFAULT); + } + + /** TODO: javadoc */ + public static IOContext newIOContext(Random random, IOContext oldContext) { + if (oldContext.hints().contains(ReadOnceHint.INSTANCE)) { + return oldContext; // just return as-is + } + final int randomNumDocs = random.nextInt(4192); + final int size = random.nextInt(512) * randomNumDocs; + if (oldContext.flushInfo() != null) { + // Always return at least the estimatedSegmentSize of + // the incoming IOContext: + return IOContext.flush( + new FlushInfo( + randomNumDocs, Math.max(oldContext.flushInfo().estimatedSegmentSize(), size))); + } else if (oldContext.mergeInfo() != null) { + // Always return at least the estimatedMergeBytes of + // the incoming IOContext: + return IOContext.merge( + new MergeInfo( + randomNumDocs, + Math.max(oldContext.mergeInfo().estimatedMergeBytes(), size), + random.nextBoolean(), + TestUtil.nextInt(random, 1, 100))); + } else { + // Make a totally random IOContext + final IOContext context; + switch (random.nextInt(3)) { + case 0: + context = IOContext.DEFAULT; + break; + case 1: + context = IOContext.merge(new MergeInfo(randomNumDocs, size, true, -1)); + break; + case 2: + context = IOContext.flush(new FlushInfo(randomNumDocs, size)); + break; + default: + context = IOContext.DEFAULT; + } + return context; + } + } + + private static final QueryCache DEFAULT_QUERY_CACHE = IndexSearcher.getDefaultQueryCache(); + private static final QueryCachingPolicy DEFAULT_CACHING_POLICY = + IndexSearcher.getDefaultQueryCachingPolicy(); + private static final List queryCacheList = new ArrayList<>(); + + @Before + public void overrideTestDefaultQueryCache() { + // Make sure each test method has its own cache + overrideDefaultQueryCache(); + } + + @BeforeClass + public static void overrideDefaultQueryCache() { + // we need to reset the query cache in an @BeforeClass so that tests that + // instantiate an IndexSearcher in an @BeforeClass method use a fresh new cache + LRUQueryCache queryCacheTemp = + new LRUQueryCache(10000, 1 << 25, _ -> true, Float.POSITIVE_INFINITY); + queryCacheList.add(queryCacheTemp); + IndexSearcher.setDefaultQueryCache(queryCacheTemp); + IndexSearcher.setDefaultQueryCachingPolicy(MAYBE_CACHE_POLICY); + } + + @AfterClass + public static void resetDefaultQueryCache() { + IndexSearcher.setDefaultQueryCache(DEFAULT_QUERY_CACHE); + IndexSearcher.setDefaultQueryCachingPolicy(DEFAULT_CACHING_POLICY); + for (int i = 0; i < queryCacheList.size(); i++) { + try { + queryCacheList.get(i).close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @BeforeClass + public static void setupCPUCoreCount() { + // Randomize core count so CMS varies its dynamic defaults, and this also "fixes" core + // count from the master seed so it will always be the same on reproduce: + int numCores = TestUtil.nextInt(random(), 1, 4); + System.setProperty( + ConcurrentMergeScheduler.DEFAULT_CPU_CORE_COUNT_PROPERTY, Integer.toString(numCores)); + } + + @AfterClass + public static void restoreCPUCoreCount() { + System.clearProperty(ConcurrentMergeScheduler.DEFAULT_CPU_CORE_COUNT_PROPERTY); + } + + private static ExecutorService executor; + + @BeforeClass + public static void setUpExecutorService() { + int threads = TestUtil.nextInt(random(), 1, 2); + executor = + new ThreadPoolExecutor( + threads, + threads, + 0L, + TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), + new NamedThreadFactory("LuceneTestCase")); + // uncomment to intensify LUCENE-3840 + // executor.prestartAllCoreThreads(); + if (VERBOSE) { + System.out.println("NOTE: Created shared ExecutorService with " + threads + " threads"); + } + } + + @AfterClass + public static void shutdownExecutorService() { + TestUtil.shutdownExecutorService(executor); + executor = null; + } + + /** Create a new searcher over the reader. This searcher might randomly use threads. */ + public static IndexSearcher newSearcher(IndexReader r) { + return newSearcher(r, true); + } + + /** Create a new searcher over the reader. This searcher might randomly use threads. */ + public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) { + return newSearcher(r, maybeWrap, true); + } + + /** + * Create a new searcher over the reader. This searcher might randomly use threads. if + * maybeWrap is true, this searcher might wrap the reader with one that returns null for + * getSequentialSubReaders. If wrapWithAssertions is true, this searcher might be an + * {@link AssertingIndexSearcher} instance. + */ + public static IndexSearcher newSearcher( + IndexReader r, boolean maybeWrap, boolean wrapWithAssertions) { + return newSearcher(r, maybeWrap, wrapWithAssertions, random().nextBoolean()); + } + + /** + * Create a new searcher over the reader. If + * maybeWrap is true, this searcher might wrap the reader with one that returns null for + * getSequentialSubReaders. If wrapWithAssertions is true, this searcher might be an + * {@link AssertingIndexSearcher} instance. The searcher will use threads if useThreads + * is set to true. + */ + public static IndexSearcher newSearcher( + IndexReader r, boolean maybeWrap, boolean wrapWithAssertions, boolean useThreads) { + if (useThreads) { + return newSearcher(r, maybeWrap, wrapWithAssertions, Concurrency.INTRA_SEGMENT); + } + return newSearcher(r, maybeWrap, wrapWithAssertions, Concurrency.NONE); + } + + /** What level of concurrency is supported by the searcher being created */ + public enum Concurrency { + /** No concurrency, meaning an executor won't be provided to the searcher */ + NONE, + /** + * Inter-segment concurrency, meaning an executor will be provided to the searcher and slices + * will be randomly created to concurrently search entire segments + */ + INTER_SEGMENT, + /** + * Intra-segment concurrency, meaning an executor will be provided to the searcher and slices + * will be randomly created to concurrently search segment partitions + */ + INTRA_SEGMENT + } + + public static IndexSearcher newSearcher( + IndexReader r, boolean maybeWrap, boolean wrapWithAssertions, Concurrency concurrency) { + Random random = random(); + if (concurrency == Concurrency.NONE) { + if (maybeWrap) { + try { + r = maybeWrapReader(r); + } catch (IOException e) { + Rethrow.rethrow(e); + } + } + // TODO: this whole check is a coverage hack, we should move it to tests for various + // filterreaders. + // ultimately whatever you do will be checkIndex'd at the end anyway. + if (random.nextInt(500) == 0 && r instanceof LeafReader) { + // TODO: not useful to check DirectoryReader (redundant with checkindex) + // but maybe sometimes run this on the other crazy readers maybeWrapReader creates? + try { + TestUtil.checkReader(r); + } catch (IOException e) { + Rethrow.rethrow(e); + } + } + final IndexSearcher ret; + if (wrapWithAssertions) { + ret = + random.nextBoolean() + ? new AssertingIndexSearcher(random, r) + : new AssertingIndexSearcher(random, r.getContext()); + } else { + ret = random.nextBoolean() ? new IndexSearcher(r) : new IndexSearcher(r.getContext()); + } + ret.setSimilarity(getTestFrameworkInfra().getClassEnv().similarity); + return ret; + } else { + final ExecutorService ex; + if (random.nextBoolean()) { + ex = null; + } else { + ex = executor; + if (VERBOSE) { + System.out.println("NOTE: newSearcher using shared ExecutorService"); + } + } + IndexSearcher ret; + int maxDocPerSlice = random.nextBoolean() ? 1 : 1 + random.nextInt(1000); + int maxSegmentsPerSlice = random.nextBoolean() ? 1 : 1 + random.nextInt(10); + if (wrapWithAssertions) { + if (random.nextBoolean()) { + ret = + new AssertingIndexSearcher(random, r, ex) { + @Override + protected LeafSlice[] slices(List leaves) { + return LuceneTestCaseParent.slices( + leaves, maxDocPerSlice, maxSegmentsPerSlice, concurrency); + } + }; + } else { + ret = + new AssertingIndexSearcher(random, r.getContext(), ex) { + @Override + protected LeafSlice[] slices(List leaves) { + return LuceneTestCaseParent.slices( + leaves, maxDocPerSlice, maxSegmentsPerSlice, concurrency); + } + }; + } + } else { + ret = + new IndexSearcher(r, ex) { + @Override + protected LeafSlice[] slices(List leaves) { + return LuceneTestCaseParent.slices( + leaves, maxDocPerSlice, maxSegmentsPerSlice, concurrency); + } + }; + } + ret.setSimilarity(getTestFrameworkInfra().getClassEnv().similarity); + ret.setQueryCachingPolicy(MAYBE_CACHE_POLICY); + if (random().nextBoolean()) { + ret.setTimeout(() -> false); + } + return ret; + } + } + + /** + * Creates an empty, temporary folder (when the name of the folder is of no importance). + * + * @see #createTempDir(String) + */ + public static Path createTempDir() { + return createTempDir("tempDir"); + } + + /** + * Creates an empty temporary file. + * + * @see #createTempFile(String, String) + */ + public static Path createTempFile() throws IOException { + return createTempFile("tempFile", ".tmp"); + } + + /** Ensures that the MergePolicy has sane values for tests that test with lots of documents. */ + protected static IndexWriterConfig ensureSaneIWCOnNightly(IndexWriterConfig conf) { + if (LuceneTestCase.TEST_NIGHTLY) { + // newIWConfig makes smallish max seg size, which + // results in tons and tons of segments for this test + // when run nightly: + MergePolicy mp = conf.getMergePolicy(); + if (mp instanceof TieredMergePolicy) { + ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.); + } else if (mp instanceof LogByteSizeMergePolicy) { + ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.); + } else if (mp instanceof LogMergePolicy) { + ((LogMergePolicy) mp).setMaxMergeDocs(100000); + } + // when running nightly, merging can still have crazy parameters, + // and might use many per-field codecs. turn on CFS for IW flushes + // and ensure CFS ratio is reasonable to keep it contained. + conf.setUseCompoundFile(true); + } + return conf; + } + + private static boolean supportsVectorEncoding( + KnnVectorsFormat format, VectorEncoding vectorEncoding) { + if (format instanceof HnswBitVectorsFormat) { + // special case, this only supports BYTE + return vectorEncoding == VectorEncoding.BYTE; + } + return true; + } + + private static boolean supportsVectorSearch(KnnVectorsFormat format) { + return (format instanceof FlatVectorsFormat) == false; + } + + protected static KnnVectorsFormat randomVectorFormat(VectorEncoding vectorEncoding) { + List availableFormats = + KnnVectorsFormat.availableKnnVectorsFormats().stream() + .map(KnnVectorsFormat::forName) + .filter(format -> supportsVectorEncoding(format, vectorEncoding)) + .filter(format -> supportsVectorSearch(format)) + .toList(); + return RandomPicks.randomFrom(random(), availableFormats); + } + + /** + * This is a test merge scheduler that will always use the intra merge executor to ensure we test + * it. + */ + private static class IntraMergeConcurrentMergeScheduler extends ConcurrentMergeScheduler { + @Override + public Executor getIntraMergeExecutor(MergePolicy.OneMerge merge) { + assert intraMergeExecutor != null : "scaledExecutor is not initialized"; + // Always do the intra merge executor to ensure we test it + return intraMergeExecutor; + } + } + + /** + * Creates an empty, temporary folder with the given name prefix. + * + *

The folder will be automatically removed after the test class completes successfully. The + * test should close any file handles that would prevent the folder from being removed. + */ + public static Path createTempDir(String prefix) { + return getTestFrameworkInfra().getTempFilesSupplier().createTempDir(prefix); + } + + /** + * Creates an empty file with the given prefix and suffix. + * + *

The file will be automatically removed after the test class completes successfully. The test + * should close any file handles that would prevent the folder from being removed. + */ + public static Path createTempFile(String prefix, String suffix) throws IOException { + return getTestFrameworkInfra().getTempFilesSupplier().createTempFile(prefix, suffix); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/QuickPatchThreadsFilter.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/QuickPatchThreadsFilter.java index 42dbe4a1c020..d7bb24c56a4a 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/QuickPatchThreadsFilter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/QuickPatchThreadsFilter.java @@ -17,43 +17,13 @@ package org.apache.lucene.tests.util; import com.carrotsearch.randomizedtesting.ThreadFilter; -import java.util.concurrent.ForkJoinWorkerThread; -import org.apache.lucene.util.Constants; /** Last minute patches. */ public class QuickPatchThreadsFilter implements ThreadFilter { - static final boolean isJ9; - - static { - isJ9 = Constants.JAVA_VENDOR.startsWith("IBM"); - } + private static final IsSystemThread delegate = new IsSystemThread(); @Override public boolean reject(Thread t) { - if (isJ9) { - // LUCENE-6518 - if ("ClassCache Reaper".equals(t.getName())) { - return true; - } - - // LUCENE-4736 - StackTraceElement[] stack = t.getStackTrace(); - if (stack.length > 0 - && stack[stack.length - 1].getClassName().equals("java.util.Timer$TimerImpl")) { - return true; - } - } - - if (t instanceof ForkJoinWorkerThread - && t.getName().startsWith("ForkJoinPool.commonPool-worker") - && t.isDaemon()) { - // GH-14066: filter out common pool's worker threads. Assume they have completed - // all background tasks and are idle. - return true; - } - - // Also filter out JNA Cleaner threads, which is static per-JVM and not under the - // control of a test suite. - return t.getName().equals("JNA Cleaner"); + return delegate.test(t); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/RemoveUponClose.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/RemoveUponClose.java deleted file mode 100644 index 3149bfd92090..000000000000 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/RemoveUponClose.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.tests.util; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import org.apache.lucene.util.IOUtils; - -/** A {@link Closeable} that attempts to remove a given file/folder. */ -final class RemoveUponClose implements Closeable { - private final Path path; - private final TestRuleMarkFailure failureMarker; - private final String creationStack; - - public RemoveUponClose(Path path, TestRuleMarkFailure failureMarker) { - this.path = path; - this.failureMarker = failureMarker; - - StringBuilder b = new StringBuilder(); - for (StackTraceElement e : Thread.currentThread().getStackTrace()) { - b.append('\t').append(e.toString()).append('\n'); - } - creationStack = b.toString(); - } - - @Override - public void close() throws IOException { - // only if there were no other test failures. - if (failureMarker.wasSuccessful()) { - if (Files.exists(path)) { - try { - IOUtils.rm(path); - } catch (IOException e) { - throw new IOException( - "Could not remove temporary location '" - + path.toAbsolutePath() - + "', created at stack trace:\n" - + creationStack, - e); - } - } - } - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/RunListenerPrintReproduceInfo.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/RunListenerPrintReproduceInfo.java index 7bdc3a710b8f..eda3a5a7d9d2 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/RunListenerPrintReproduceInfo.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/RunListenerPrintReproduceInfo.java @@ -16,47 +16,20 @@ */ package org.apache.lucene.tests.util; -import static org.apache.lucene.tests.util.LuceneTestCase.DEFAULT_LINE_DOCS_FILE; -import static org.apache.lucene.tests.util.LuceneTestCase.JENKINS_LARGE_LINE_DOCS_FILE; -import static org.apache.lucene.tests.util.LuceneTestCase.RANDOM_MULTIPLIER; -import static org.apache.lucene.tests.util.LuceneTestCase.SYSPROP_AWAITSFIX; -import static org.apache.lucene.tests.util.LuceneTestCase.SYSPROP_MONSTER; -import static org.apache.lucene.tests.util.LuceneTestCase.SYSPROP_NIGHTLY; -import static org.apache.lucene.tests.util.LuceneTestCase.SYSPROP_WEEKLY; -import static org.apache.lucene.tests.util.LuceneTestCase.TEST_AWAITSFIX; -import static org.apache.lucene.tests.util.LuceneTestCase.TEST_CODEC; -import static org.apache.lucene.tests.util.LuceneTestCase.TEST_DIRECTORY; -import static org.apache.lucene.tests.util.LuceneTestCase.TEST_DOCVALUESFORMAT; -import static org.apache.lucene.tests.util.LuceneTestCase.TEST_LINE_DOCS_FILE; -import static org.apache.lucene.tests.util.LuceneTestCase.TEST_MONSTER; -import static org.apache.lucene.tests.util.LuceneTestCase.TEST_NIGHTLY; -import static org.apache.lucene.tests.util.LuceneTestCase.TEST_POSTINGSFORMAT; -import static org.apache.lucene.tests.util.LuceneTestCase.TEST_WEEKLY; -import static org.apache.lucene.tests.util.LuceneTestCase.classEnvRule; - import com.carrotsearch.randomizedtesting.LifecycleScope; import com.carrotsearch.randomizedtesting.RandomizedContext; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Pattern; -import org.apache.lucene.util.Constants; +import java.util.Optional; import org.junit.runner.Description; import org.junit.runner.Result; import org.junit.runner.notification.Failure; import org.junit.runner.notification.RunListener; /** - * A suite listener printing a "reproduce string". This ensures test result events are always - * captured properly even if exceptions happen at initialization or suite/ hooks level. + * A suite listener printing a "reproduce string" (junit4/ randomizedtesting). This ensures test + * result events are always captured properly even if exceptions happen at initialization or suite/ + * hooks level. */ public final class RunListenerPrintReproduceInfo extends RunListener { - /** - * A list of all test suite classes executed so far in this JVM (ehm, under this class's - * classloader). - */ - private static final List testClassesRun = new ArrayList<>(); - /** The currently executing scope. */ private LifecycleScope scope; @@ -66,22 +39,27 @@ public final class RunListenerPrintReproduceInfo extends RunListener { /** Suite-level code (initialization, rule, hook) failed. */ private boolean suiteFailed; - /** A marker to print full env. diagnostics after the suite. */ - private boolean printDiagnosticsAfterClass; + /** Either a test or something else has failed. */ + private boolean somethingFailed; /** true if we should skip the reproduce string (diagnostics are independent) */ private boolean suppressReproduceLine; + /** Environment settings for the run. Set by {@link LuceneTestCase}. */ + @SuppressWarnings("NonFinalStaticField") + static TestEnvInfo envInfoJunit4; + @Override public void testRunStarted(Description description) throws Exception { suiteFailed = false; testFailed = false; scope = LifecycleScope.SUITE; + envInfoJunit4 = null; - Class targetClass = RandomizedContext.current().getTargetClass(); suppressReproduceLine = - targetClass.isAnnotationPresent(LuceneTestCase.SuppressReproduceLine.class); - testClassesRun.add(targetClass.getSimpleName()); + RandomizedContext.current() + .getTargetClass() + .isAnnotationPresent(LuceneTestCase.SuppressReproduceLine.class); } @Override @@ -97,18 +75,34 @@ public void testFailure(Failure failure) throws Exception { } else { suiteFailed = true; } - printDiagnosticsAfterClass = true; + somethingFailed = true; } @Override public void testFinished(Description description) throws Exception { - if (testFailed) { - reportAdditionalFailureInfo(stripTestNameAugmentations(description.getMethodName())); + if (testFailed && !suppressReproduceLine) { + System.err.println( + envInfoJunit4.getAdditionalFailureInfo( + RandomizedContext.current().getRunnerSeedAsString(), + b -> { + appendSelectorArguments( + b, Optional.of(stripTestNameAugmentations(description.getMethodName()))); + })); } scope = LifecycleScope.SUITE; testFailed = false; } + private void appendSelectorArguments(StringBuilder b, Optional testMethod) { + // Figure out the test case name and method, if any. + String testClass = RandomizedContext.current().getTargetClass().getSimpleName(); + b.append("--tests "); + b.append(testClass); + if (testMethod.isPresent()) { + b.append(".").append(testMethod.get()); + } + } + /** * The {@link Description} object in JUnit does not expose the actual test method, instead it has * the concept of a unique "name" of a test. To run the same method (tests) repeatedly, @@ -124,123 +118,16 @@ private String stripTestNameAugmentations(String methodName) { @Override public void testRunFinished(Result result) throws Exception { - if (printDiagnosticsAfterClass || LuceneTestCase.VERBOSE) { - RunListenerPrintReproduceInfo.printDebuggingInformation(); - } - - if (suiteFailed) { - reportAdditionalFailureInfo(null); - } - } - - /** print some useful debugging information about the environment */ - private static void printDebuggingInformation() { - if (classEnvRule != null && classEnvRule.isInitialized()) { - System.err.println( - ("NOTE: test params are: codec=" + classEnvRule.codec) - + (", sim=" + classEnvRule.similarity) - + (", locale=" + classEnvRule.locale.toLanguageTag()) - + (", timezone=" - + (classEnvRule.timeZone == null ? "(null)" : classEnvRule.timeZone.getID()))); - } - System.err.println( - "NOTE: " - + (System.getProperty("os.name") + " ") - + (System.getProperty("os.version") + " ") - + (System.getProperty("os.arch") + "/" + System.getProperty("java.vendor")) - + (" " + System.getProperty("java.version")) - + (" " - + (Constants.JRE_IS_64BIT ? "(64-bit)" : "(32-bit)") - + "/" - + "cpus=" - + Runtime.getRuntime().availableProcessors() - + ",") - + ("threads=" + Thread.activeCount() + ",") - + ("free=" + Runtime.getRuntime().freeMemory() + ",") - + ("total=" + Runtime.getRuntime().totalMemory())); - System.err.println( - "NOTE: All tests run in this JVM: " + Arrays.toString(testClassesRun.toArray())); - } - - private void reportAdditionalFailureInfo(final String testName) { - if (suppressReproduceLine) { - return; - } - if (TEST_LINE_DOCS_FILE.endsWith(JENKINS_LARGE_LINE_DOCS_FILE)) { + if (suiteFailed && !suppressReproduceLine) { System.err.println( - "NOTE: large line-docs file was used in this run. You have to download " - + "it manually ('gradlew getEnWikiRandomLines') and use -P" - + TEST_LINE_DOCS_FILE - + "=... property to point to it."); + envInfoJunit4.getAdditionalFailureInfo( + RandomizedContext.current().getRunnerSeedAsString(), + b -> { + appendSelectorArguments(b, Optional.empty()); + })); } - - final StringBuilder b = new StringBuilder(); - b.append("NOTE: reproduce with: gradlew test "); - - // Figure out the test case name and method, if any. - String testClass = RandomizedContext.current().getTargetClass().getSimpleName(); - b.append("--tests "); - b.append(testClass); - if (testName != null) { - b.append(".").append(testName); - } - - // Pass the master seed. - addVmOpt(b, "tests.seed", RandomizedContext.current().getRunnerSeedAsString()); - - addVmOpt(b, "tests.jvmargs", System.getProperty("tests.jvmargs")); - - // Test groups and multipliers. - if (RANDOM_MULTIPLIER != LuceneTestCase.defaultRandomMultiplier()) - addVmOpt(b, "tests.multiplier", RANDOM_MULTIPLIER); - if (TEST_NIGHTLY) addVmOpt(b, SYSPROP_NIGHTLY, TEST_NIGHTLY); - if (TEST_WEEKLY) addVmOpt(b, SYSPROP_WEEKLY, TEST_WEEKLY); - if (TEST_MONSTER) addVmOpt(b, SYSPROP_MONSTER, TEST_MONSTER); - if (TEST_AWAITSFIX) addVmOpt(b, SYSPROP_AWAITSFIX, TEST_AWAITSFIX); - - // Codec, postings, directories. - if (!TEST_CODEC.equals("random")) addVmOpt(b, "tests.codec", TEST_CODEC); - if (!TEST_POSTINGSFORMAT.equals("random")) - addVmOpt(b, "tests.postingsformat", TEST_POSTINGSFORMAT); - if (!TEST_DOCVALUESFORMAT.equals("random")) - addVmOpt(b, "tests.docvaluesformat", TEST_DOCVALUESFORMAT); - if (!TEST_DIRECTORY.equals("random")) addVmOpt(b, "tests.directory", TEST_DIRECTORY); - - // Environment. - if (!TEST_LINE_DOCS_FILE.equals(DEFAULT_LINE_DOCS_FILE)) - addVmOpt(b, "tests.linedocsfile", TEST_LINE_DOCS_FILE); - if (classEnvRule != null && classEnvRule.isInitialized()) { - addVmOpt(b, "tests.locale", classEnvRule.locale.toLanguageTag()); - if (classEnvRule.timeZone != null) { - addVmOpt(b, "tests.timezone", classEnvRule.timeZone.getID()); - } - } - - if (LuceneTestCase.TEST_ASSERTS_ENABLED) { - addVmOpt(b, "tests.asserts", "true"); - } else { - addVmOpt(b, "tests.asserts", "false"); - } - - addVmOpt(b, "tests.file.encoding", System.getProperty("file.encoding")); - - System.err.println(b.toString()); - } - - /** - * Append a VM option (-Dkey=value) to a {@link StringBuilder}. Add quotes if spaces or other - * funky characters are detected. - */ - static void addVmOpt(StringBuilder b, String key, Object value) { - if (value == null) return; - - b.append(" -D").append(key).append("="); - String v = value.toString(); - // Add simplistic quoting. This varies a lot from system to system and between - // shells... ANT should have some code for doing it properly. - if (Pattern.compile("[\\s=']").matcher(v).find()) { - v = '"' + v + '"'; + if (somethingFailed || LuceneTestCase.VERBOSE) { + System.err.println(envInfoJunit4.getDebuggingInformation()); } - b.append(v); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/SetupAndRestoreStaticEnv.java similarity index 89% rename from lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java rename to lucene/test-framework/src/java/org/apache/lucene/tests/util/SetupAndRestoreStaticEnv.java index f5ab9b3a1737..347c5c754e45 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/SetupAndRestoreStaticEnv.java @@ -23,11 +23,9 @@ import static org.apache.lucene.tests.util.LuceneTestCase.VERBOSE; import static org.apache.lucene.tests.util.LuceneTestCase.assumeFalse; import static org.apache.lucene.tests.util.LuceneTestCase.localeForLanguageTag; -import static org.apache.lucene.tests.util.LuceneTestCase.random; import static org.apache.lucene.tests.util.LuceneTestCase.randomLocale; import static org.apache.lucene.tests.util.LuceneTestCase.randomTimeZone; -import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import java.io.PrintStream; import java.util.Arrays; @@ -35,6 +33,7 @@ import java.util.Locale; import java.util.Random; import java.util.TimeZone; +import java.util.function.Supplier; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; @@ -50,18 +49,22 @@ import org.apache.lucene.tests.index.RandomCodec; import org.apache.lucene.tests.search.similarities.AssertingSimilarity; import org.apache.lucene.tests.search.similarities.RandomSimilarity; -import org.apache.lucene.tests.util.LuceneTestCase.LiveIWCFlushMode; import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; +import org.apache.lucene.tests.util.LuceneTestCaseParent.LiveIWCFlushMode; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.PrintStreamInfoStream; import org.junit.internal.AssumptionViolatedException; -/** Setup and restore suite-level environment (fine grained junk that doesn't fit anywhere else). */ -final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { +/** Setup and restore suite-level environment (fine-grained junk that doesn't fit anywhere else). */ +final class SetupAndRestoreStaticEnv implements BeforeAfterCallback { + private final Supplier randomSupplier; + private final Supplier> targetClassSupplier; + private Codec savedCodec; private Locale savedLocale; private TimeZone savedTimeZone; private InfoStream savedInfoStream; + private LiveIWCFlushMode flushMode; Locale locale; TimeZone timeZone; @@ -76,6 +79,16 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule { */ HashSet avoidCodecs; + SetupAndRestoreStaticEnv( + Supplier randomSupplier, Supplier> targetClassSupplier) { + this.randomSupplier = randomSupplier; + this.targetClassSupplier = targetClassSupplier; + } + + LuceneTestCaseParent.LiveIWCFlushMode getLiveIWCFlushMode() { + return flushMode; + } + static class ThreadNameFixingPrintStreamInfoStream extends PrintStreamInfoStream { public ThreadNameFixingPrintStreamInfoStream(PrintStream out) { super(out); @@ -104,7 +117,7 @@ public boolean isInitialized() { } @Override - protected void before() throws Exception { + public void before() throws Exception { // if verbose: print some debugging stuff about which codecs are loaded. if (VERBOSE) { System.out.println("Loaded codecs: " + Codec.availableCodecs()); @@ -112,7 +125,7 @@ protected void before() throws Exception { } savedInfoStream = InfoStream.getDefault(); - final Random random = RandomizedContext.current().getRandom(); + final Random random = randomSupplier.get(); final boolean v = random.nextBoolean(); if (INFOSTREAM) { InfoStream.setDefault(new ThreadNameFixingPrintStreamInfoStream(System.out)); @@ -120,7 +133,7 @@ protected void before() throws Exception { InfoStream.setDefault(new NullInfoStream()); } - Class targetClass = RandomizedContext.current().getTargetClass(); + Class targetClass = targetClassSupplier.get(); avoidCodecs = new HashSet<>(); if (targetClass.isAnnotationPresent(SuppressCodecs.class)) { SuppressCodecs a = targetClass.getAnnotation(SuppressCodecs.class); @@ -211,10 +224,10 @@ public String toString() { Locale.setDefault(locale); savedTimeZone = TimeZone.getDefault(); - TimeZone randomTimeZone = randomTimeZone(random()); + TimeZone randomTimeZone = randomTimeZone(random); timeZone = testTimeZone.equals("random") ? randomTimeZone : TimeZone.getTimeZone(testTimeZone); TimeZone.setDefault(timeZone); - similarity = new AssertingSimilarity(new RandomSimilarity(random())); + similarity = new AssertingSimilarity(new RandomSimilarity(random)); // Check codec restrictions once at class level. try { @@ -232,22 +245,13 @@ public String toString() { // just the doc count to flush by, else both. // This way the assertMemory in DocumentsWriterFlushControl sometimes runs (when we always flush // by RAM). - LiveIWCFlushMode flushMode; - switch (random().nextInt(3)) { - case 0: - flushMode = LiveIWCFlushMode.BY_RAM; - break; - case 1: - flushMode = LiveIWCFlushMode.BY_DOCS; - break; - case 2: - flushMode = LiveIWCFlushMode.EITHER; - break; - default: - throw new AssertionError(); - } - - LuceneTestCase.setLiveIWCFlushMode(flushMode); + this.flushMode = + switch (random.nextInt(3)) { + case 0 -> LiveIWCFlushMode.BY_RAM; + case 1 -> LiveIWCFlushMode.BY_DOCS; + case 2 -> LiveIWCFlushMode.EITHER; + default -> throw new AssertionError(); + }; initialized = true; } @@ -281,7 +285,7 @@ private void checkCodecRestrictions(Codec codec) { /** After suite cleanup (always invoked). */ @Override - protected void after() throws Exception { + public void after() throws Exception { Codec.setDefault(savedCodec); InfoStream.setDefault(savedInfoStream); if (savedLocale != null) Locale.setDefault(savedLocale); diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/SuiteFailureState.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/SuiteFailureState.java new file mode 100644 index 000000000000..b6b4fe829c6b --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/SuiteFailureState.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.tests.util; + +/** Signals whether the current test suite had any failures. */ +interface SuiteFailureState { + boolean wasSuccessful(); +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TagState.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TagState.java new file mode 100644 index 000000000000..a0db4564e039 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TagState.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.tests.util; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.platform.commons.support.AnnotationSupport; + +/** + * Control the default state (enabled/ disabled) of a test tag meta-annotation. The state can be + * switched using a system property. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +@ExtendWith(TagState.Condition.class) +public @interface TagState { + boolean enabled(); + + String sysProperty(); + + /** Execution condition to ignore a tagged test depending on the tag's system property state. */ + final class Condition implements ExecutionCondition { + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + var opt = context.getElement(); + if (opt.isPresent()) { + var element = opt.get(); + var stateAnn_ = AnnotationSupport.findAnnotation(element, TagState.class); + if (stateAnn_.isPresent()) { + var stateAnn = stateAnn_.get(); + var tagAnns = AnnotationSupport.findRepeatableAnnotations(element, Tag.class); + if (tagAnns.size() != 1) { + throw new RuntimeException("Expected exactly one @Tag meta-annotation on: " + element); + } + + String tag = tagAnns.getFirst().value(); + var enabled = + context + .getConfigurationParameter(stateAnn.sysProperty(), Boolean::parseBoolean) + .orElse(stateAnn.enabled()); + return enabled + ? ConditionEvaluationResult.enabled("@" + tag + " enabled") + : ConditionEvaluationResult.disabled( + "@" + tag + " disabled; use -D" + stateAnn.sysProperty() + "=true to enable."); + } + } + + return ConditionEvaluationResult.enabled("no @TestGroupTag found"); + } + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleTemporaryFilesCleanup.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TemporaryFilesSupplier.java similarity index 72% rename from lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleTemporaryFilesCleanup.java rename to lucene/test-framework/src/java/org/apache/lucene/tests/util/TemporaryFilesSupplier.java index 8cd38b6bdc90..32669aa3ed44 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleTemporaryFilesCleanup.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TemporaryFilesSupplier.java @@ -16,8 +16,7 @@ */ package org.apache.lucene.tests.util; -import com.carrotsearch.randomizedtesting.RandomizedContext; -import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; +import java.io.Closeable; import java.io.IOException; import java.net.URI; import java.nio.file.FileSystem; @@ -27,12 +26,11 @@ import java.nio.file.spi.FileSystemProvider; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Random; import java.util.Set; +import java.util.function.Supplier; import org.apache.lucene.tests.mockfile.DisableFsyncFS; import org.apache.lucene.tests.mockfile.ExtrasFS; import org.apache.lucene.tests.mockfile.HandleLimitFS; @@ -51,7 +49,10 @@ * @see LuceneTestCase#createTempDir() * @see LuceneTestCase#createTempFile() */ -final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter { +final class TemporaryFilesSupplier implements BeforeAfterCallback { + private final Supplier randomSupplier; + private final Supplier> targetClassSupplier; + /** Retry to create temporary file name this many times. */ private static final int TEMP_NAME_RETRY_THRESHOLD = 9999; @@ -65,21 +66,40 @@ final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter { private FileSystem fileSystem; /** Suite failure marker. */ - private final TestRuleMarkFailure failureMarker; + private final SuiteFailureState failureMarker; + + /** + * A queue of paths to be removed after the suite completes. + * + * @see #registerToCloseAfterSuite(Path) + */ + private static final ArrayList pathQueue = new ArrayList<>(); /** - * A queue of temporary resources to be removed after the suite completes. + * A queue of closeables to be closed after the suite completes. * - * @see #registerToRemoveAfterSuite(Path) + * @see #registerToCloseAfterSuite(Closeable) */ - private static final List cleanupQueue = new ArrayList<>(); + private static final ArrayList closeableQueue = new ArrayList<>(); - public TestRuleTemporaryFilesCleanup(TestRuleMarkFailure failureMarker) { + public TemporaryFilesSupplier( + SuiteFailureState failureMarker, + Supplier randomSupplier, + Supplier> targetClassSupplier) { this.failureMarker = failureMarker; + this.randomSupplier = randomSupplier; + this.targetClassSupplier = targetClassSupplier; } /** Register temporary folder for removal after the suite completes. */ - void registerToRemoveAfterSuite(Path f) { + void registerToCloseAfterSuite(Closeable c) { + synchronized (closeableQueue) { + closeableQueue.addLast(c); + } + } + + /** Register temporary folder for removal after the suite completes. */ + void registerToCloseAfterSuite(Path f) { assert f != null; if (LuceneTestCase.LEAVE_TEMPORARY) { @@ -87,17 +107,21 @@ void registerToRemoveAfterSuite(Path f) { return; } - synchronized (cleanupQueue) { - cleanupQueue.add(f); + synchronized (pathQueue) { + pathQueue.addLast(f); } } @Override - protected void before() throws Throwable { - super.before(); - + public void before() throws Exception { assert tempDirBase == null; + assert pathQueue.isEmpty() && closeableQueue.isEmpty(); + fileSystem = initializeFileSystem(); + if (fileSystem != FileSystems.getDefault()) { + registerToCloseAfterSuite(fileSystem); + } + javaTempDir = initializeJavaTempDir(); } @@ -110,7 +134,7 @@ private boolean allowed(Set avoid, Class c } private FileSystem initializeFileSystem() { - Class targetClass = RandomizedContext.current().getTargetClass(); + Class targetClass = targetClassSupplier.get(); Set avoid = new HashSet<>(); if (targetClass.isAnnotationPresent(SuppressFileSystems.class)) { SuppressFileSystems a = targetClass.getAnnotation(SuppressFileSystems.class); @@ -121,12 +145,11 @@ private FileSystem initializeFileSystem() { fs = new VerboseFS( fs, - new TestRuleSetupAndRestoreClassEnv.ThreadNameFixingPrintStreamInfoStream( - System.out)) + new SetupAndRestoreStaticEnv.ThreadNameFixingPrintStreamInfoStream(System.out)) .getFileSystem(null); } - Random random = RandomizedContext.current().getRandom(); + Random random = randomSupplier.get(); // speed up tests by omitting actual fsync calls to the hardware most of the time. if (targetClass.isAnnotationPresent(SuppressFsync.class) || random.nextInt(100) > 0) { @@ -182,53 +205,57 @@ private Path initializeJavaTempDir() throws IOException { } @Override - protected void afterAlways(List errors) throws Throwable { + public void after() throws Exception { + final String tempDirBasePath = + (tempDirBase != null ? tempDirBase.toAbsolutePath().toString() : null); + tempDirBase = null; + // Drain cleanup queue and clear it. - final Path[] everything; - final String tempDirBasePath; - synchronized (cleanupQueue) { - tempDirBasePath = (tempDirBase != null ? tempDirBase.toAbsolutePath().toString() : null); - tempDirBase = null; - - Collections.reverse(cleanupQueue); - everything = new Path[cleanupQueue.size()]; - cleanupQueue.toArray(everything); - cleanupQueue.clear(); + final Path[] paths; + synchronized (pathQueue) { + paths = pathQueue.reversed().toArray(Path[]::new); + pathQueue.clear(); } // Only check and throw an IOException on un-removable files if the test - // was successful. Otherwise just report the path of temporary files + // was successful. Otherwise, just report the path of temporary files // and leave them there. if (failureMarker.wasSuccessful()) { - - try { - IOUtils.rm(everything); - } catch (IOException e) { - Class suiteClass = RandomizedContext.current().getTargetClass(); - if (suiteClass.isAnnotationPresent(SuppressTempFileChecks.class)) { - System.err.println( - "WARNING: Leftover undeleted temporary files (bugUrl: " - + suiteClass.getAnnotation(SuppressTempFileChecks.class).bugUrl() - + "): " - + e.getMessage()); - return; - } - throw e; - } - if (fileSystem != FileSystems.getDefault()) { - fileSystem.close(); - } + registerToCloseAfterSuite( + () -> { + try { + IOUtils.rm(paths); + } catch (IOException e) { + Class suiteClass = targetClassSupplier.get(); + if (suiteClass.isAnnotationPresent(SuppressTempFileChecks.class)) { + System.err.println( + "WARNING: Leftover undeleted temporary files (bugUrl: " + + suiteClass.getAnnotation(SuppressTempFileChecks.class).bugUrl() + + "): " + + e.getMessage()); + return; + } + throw e; + } + }); } else { if (tempDirBasePath != null) { System.err.println("NOTE: leaving temporary files on disk at: " + tempDirBasePath); } } + + Closeable[] toClose; + synchronized (closeableQueue) { + toClose = closeableQueue.reversed().toArray(Closeable[]::new); + closeableQueue.clear(); + } + + IOUtils.close(toClose); } Path getPerTestClassTempDir() { if (tempDirBase == null) { - RandomizedContext ctx = RandomizedContext.current(); - Class clazz = ctx.getTargetClass(); + Class clazz = targetClassSupplier.get(); String prefix = clazz.getName(); prefix = prefix.replaceFirst("^org.apache.lucene.", "lucene."); @@ -241,13 +268,7 @@ Path getPerTestClassTempDir() { "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: " + javaTempDir.toAbsolutePath()); } - f = - javaTempDir.resolve( - prefix - + "_" - + ctx.getRunnerSeedAsString() - + "-" - + String.format(Locale.ENGLISH, "%03d", attempt)); + f = javaTempDir.resolve(prefix + "_" + String.format(Locale.ENGLISH, "%03d", attempt)); try { Files.createDirectory(f); success = true; @@ -256,7 +277,7 @@ Path getPerTestClassTempDir() { } while (!success); tempDirBase = f; - registerToRemoveAfterSuite(tempDirBase); + registerToCloseAfterSuite(tempDirBase); } return tempDirBase; } @@ -284,7 +305,7 @@ public Path createTempDir(String prefix) { } } while (!success); - registerToRemoveAfterSuite(f); + registerToCloseAfterSuite(f); return f; } @@ -311,7 +332,7 @@ public Path createTempFile(String prefix, String suffix) throws IOException { } } while (!success); - registerToRemoveAfterSuite(f); + registerToCloseAfterSuite(f); return f; } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestEnvInfo.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestEnvInfo.java new file mode 100644 index 000000000000..9518e39b8479 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestEnvInfo.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.tests.util; + +import static org.apache.lucene.tests.util.LuceneTestCaseParent.DEFAULT_LINE_DOCS_FILE; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.JENKINS_LARGE_LINE_DOCS_FILE; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.RANDOM_MULTIPLIER; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.SYSPROP_AWAITSFIX; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.SYSPROP_MONSTER; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.SYSPROP_NIGHTLY; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.SYSPROP_WEEKLY; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_ASSERTS_ENABLED; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_AWAITSFIX; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_CODEC; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_DIRECTORY; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_DOCVALUESFORMAT; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_LINE_DOCS_FILE; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_MONSTER; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_NIGHTLY; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_POSTINGSFORMAT; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.TEST_WEEKLY; +import static org.apache.lucene.tests.util.LuceneTestCaseParent.defaultRandomMultiplier; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.TimeZone; +import java.util.function.Consumer; +import java.util.regex.Pattern; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.Constants; + +/** Test environment information. */ +record TestEnvInfo(String codec, String similarity, String locale, String timeZone) { + static final String TEST_ENV_LEAD = "NOTE: test environment is:"; + static final String TEST_REPRO_LEAD = "NOTE: reproduce with:"; + + TestEnvInfo(Codec codec, Similarity similarity, Locale locale, TimeZone timeZone) { + this( + Objects.toString(codec), + Objects.toString(similarity), + Optional.ofNullable(locale).map(Locale::toLanguageTag).orElse(null), + Optional.ofNullable(timeZone).map(TimeZone::getID).orElse(null)); + } + + /** print some useful debugging information about the environment */ + String getDebuggingInformation() { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + pw.println( + (TEST_ENV_LEAD + " codec=" + codec) + + (", sim=" + similarity) + + (", locale=" + locale) + + (", timezone=" + timeZone)); + pw.println( + "NOTE: " + + (System.getProperty("os.name") + " ") + + (System.getProperty("os.version") + " ") + + (System.getProperty("os.arch") + "/" + System.getProperty("java.vendor")) + + (" " + System.getProperty("java.version")) + + (" " + + (Constants.JRE_IS_64BIT ? "(64-bit)" : "(32-bit)") + + "/" + + "cpus=" + + Runtime.getRuntime().availableProcessors() + + ",") + + ("threads=" + Thread.activeCount() + ",") + + ("free=" + Runtime.getRuntime().freeMemory() + ",") + + ("total=" + Runtime.getRuntime().totalMemory())); + pw.flush(); + return sw.toString(); + } + + String getAdditionalFailureInfo(String seed, Consumer extraArguments) { + if (TEST_LINE_DOCS_FILE.endsWith(JENKINS_LARGE_LINE_DOCS_FILE)) { + System.err.println( + "NOTE: large line-docs file was used in this run. You have to download " + + "it manually ('gradlew getEnWikiRandomLines') and use -P" + + TEST_LINE_DOCS_FILE + + "=... property to point to it."); + } + + final StringBuilder b = new StringBuilder(); + b.append(TEST_REPRO_LEAD + " gradlew test "); + + extraArguments.accept(b); + + // Pass the master seed. + addVmOpt(b, "tests.seed", seed); + addVmOpt(b, "tests.jvmargs", System.getProperty("tests.jvmargs")); + + // Test groups and multipliers. + if (RANDOM_MULTIPLIER != defaultRandomMultiplier()) + addVmOpt(b, "tests.multiplier", RANDOM_MULTIPLIER); + if (TEST_NIGHTLY) addVmOpt(b, SYSPROP_NIGHTLY, TEST_NIGHTLY); + if (TEST_WEEKLY) addVmOpt(b, SYSPROP_WEEKLY, TEST_WEEKLY); + if (TEST_MONSTER) addVmOpt(b, SYSPROP_MONSTER, TEST_MONSTER); + if (TEST_AWAITSFIX) addVmOpt(b, SYSPROP_AWAITSFIX, TEST_AWAITSFIX); + + // Codec, postings, directories. + if (!TEST_CODEC.equals("random")) addVmOpt(b, "tests.codec", TEST_CODEC); + if (!TEST_POSTINGSFORMAT.equals("random")) + addVmOpt(b, "tests.postingsformat", TEST_POSTINGSFORMAT); + if (!TEST_DOCVALUESFORMAT.equals("random")) + addVmOpt(b, "tests.docvaluesformat", TEST_DOCVALUESFORMAT); + if (!TEST_DIRECTORY.equals("random")) addVmOpt(b, "tests.directory", TEST_DIRECTORY); + + // Environment. + if (!TEST_LINE_DOCS_FILE.equals(DEFAULT_LINE_DOCS_FILE)) + addVmOpt(b, "tests.linedocsfile", TEST_LINE_DOCS_FILE); + if (locale() != null) { + addVmOpt(b, "tests.locale", locale()); + } + + if (timeZone() != null) { + addVmOpt(b, "tests.timezone", timeZone()); + } + + if (TEST_ASSERTS_ENABLED) { + addVmOpt(b, "tests.asserts", "true"); + } else { + addVmOpt(b, "tests.asserts", "false"); + } + + addVmOpt(b, "tests.file.encoding", System.getProperty("file.encoding")); + + return b.toString(); + } + + /** + * Append a VM option (-Dkey=value) to a {@link StringBuilder}. Add quotes if spaces or other + * funky characters are detected. + */ + static void addVmOpt(StringBuilder b, String key, Object value) { + if (value == null) return; + + b.append(" -D").append(key).append("="); + String v = value.toString(); + // Add simplistic quoting. This varies a lot from system to system and between + // shells... ANT should have some code for doing it properly. + if (Pattern.compile("[\\s=']").matcher(v).find()) { + v = '"' + v + '"'; + } + b.append(v); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleAssertionsRequired.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleAssertionsRequired.java index 192f2ee8321c..3a8ff2908119 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleAssertionsRequired.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleAssertionsRequired.java @@ -27,30 +27,29 @@ public Statement apply(final Statement base, final Description description) { return new Statement() { @Override public void evaluate() throws Throwable { - try { - // Make sure -ea matches -Dtests.asserts, to catch accidental mis-use: - var assertsEnabled = LuceneTestCase.class.desiredAssertionStatus(); - if (assertsEnabled != LuceneTestCase.TEST_ASSERTS_ENABLED) { - String msg = "Assertions mismatch: "; - if (assertsEnabled) { - msg += "-ea was specified"; - } else { - msg += "-ea was not specified"; - } - if (LuceneTestCase.TEST_ASSERTS_ENABLED) { - msg += " but -Dtests.asserts=true"; - } else { - msg += " but -Dtests.asserts=false"; - } - System.err.println(msg); - throw new Exception(msg); - } - } catch (AssertionError _) { - // Ok, enabled. - } - + checkAssertionStatus(); base.evaluate(); } }; } + + // Make sure -ea matches -Dtests.asserts, to catch accidental mis-use: + static void checkAssertionStatus() throws Exception { + var assertsEnabled = LuceneTestCase.class.desiredAssertionStatus(); + if (assertsEnabled != LuceneTestCase.TEST_ASSERTS_ENABLED) { + String msg = "Assertions mismatch: "; + if (assertsEnabled) { + msg += "-ea was specified"; + } else { + msg += "-ea was not specified"; + } + if (LuceneTestCase.TEST_ASSERTS_ENABLED) { + msg += " but -Dtests.asserts=true"; + } else { + msg += " but -Dtests.asserts=false"; + } + System.err.println(msg); + throw new Exception(msg); + } + } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleLimitSysouts.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleLimitSysouts.java index 865b465a9562..20f68cf26bcf 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleLimitSysouts.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleLimitSysouts.java @@ -138,7 +138,7 @@ public class TestRuleLimitSysouts extends TestRuleAdapter { } /** Test failures from any tests or rules before. */ - private final TestRuleMarkFailure failureMarker; + private final SuiteFailureState failureMarker; interface LimitPredicate { void check(long before, long after) throws IOException; @@ -196,7 +196,7 @@ private void checkLimit(int bytes) throws IOException { } } - public TestRuleLimitSysouts(TestRuleMarkFailure failureMarker) { + public TestRuleLimitSysouts(SuiteFailureState failureMarker) { this.failureMarker = failureMarker; } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleMarkFailure.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleMarkFailure.java index f825802729a0..2d0c6b0e4b09 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleMarkFailure.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleMarkFailure.java @@ -24,7 +24,7 @@ import org.junit.runners.model.Statement; /** A rule for marking failed tests and suites. */ -public final class TestRuleMarkFailure implements TestRule { +public final class TestRuleMarkFailure implements TestRule, BeforeAfterCallback, SuiteFailureState { private final TestRuleMarkFailure[] chained; private volatile boolean failures; @@ -32,14 +32,17 @@ public TestRuleMarkFailure(TestRuleMarkFailure... chained) { this.chained = chained; } + @Override + public void before() throws Exception { + reset(); + } + @Override public Statement apply(final Statement s, Description d) { return new Statement() { @Override public void evaluate() throws Throwable { - // Clear status at start. - failures = false; - + reset(); try { s.evaluate(); } catch (Throwable t) { @@ -96,7 +99,12 @@ public boolean hadFailures() { } /** Check if this object was successful (the opposite of {@link #hadFailures()}). */ + @Override public boolean wasSuccessful() { return !hadFailures(); } + + private void reset() { + failures = false; + } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreInstanceEnv.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreInstanceEnv.java index 4dff8ce4bf80..65f193e75b71 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreInstanceEnv.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreInstanceEnv.java @@ -22,16 +22,16 @@ * Prepares and restores {@link LuceneTestCase} at instance level (fine grained junk that doesn't * fit anywhere else). */ -final class TestRuleSetupAndRestoreInstanceEnv extends AbstractBeforeAfterRule { +final class TestRuleSetupAndRestoreInstanceEnv implements BeforeAfterCallback { private int savedBoolMaxClauseCount; @Override - protected void before() { + public void before() { savedBoolMaxClauseCount = IndexSearcher.getMaxClauseCount(); } @Override - protected void after() { + public void after() { IndexSearcher.setMaxClauseCount(savedBoolMaxClauseCount); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java index 9f900b6d68f8..122817a39223 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java @@ -27,7 +27,6 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; -import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import java.io.BufferedInputStream; @@ -326,7 +325,7 @@ public static CheckIndex.Status checkIndex( checker.setFailFast(failFast); checker.setInfoStream(new PrintStream(output, false, UTF_8), false); if (concurrent) { - checker.setThreadCount(RandomizedTest.randomIntBetween(2, 5)); + checker.setThreadCount(RandomNumbers.randomIntBetween(LuceneTestCase.random(), 2, 5)); } else { checker.setThreadCount(1); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/VerifyTestClassNamingConvention.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/VerifyTestClassNamingConvention.java index 4926f90a8090..905be5f927ff 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/VerifyTestClassNamingConvention.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/VerifyTestClassNamingConvention.java @@ -37,8 +37,11 @@ protected void before() throws Exception { return; } - String suiteName = RandomizedContext.current().getTargetClass().getName(); + check(RandomizedContext.current().getTargetClass()); + } + public void check(Class clazz) { + String suiteName = clazz.getName(); Matcher matcher = namingConvention.matcher(suiteName); if (suiteName.startsWith(packagePrefix) && !matcher.matches()) { throw new AssertionError( diff --git a/lucene/test-framework/src/test/org/apache/lucene/tests/search/TestBaseExplanationTestCase.java b/lucene/test-framework/src/test/org/apache/lucene/tests/search/TestBaseExplanationTestCase.java index 6d184fbbe3fd..c8027055a3a2 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/tests/search/TestBaseExplanationTestCase.java +++ b/lucene/test-framework/src/test/org/apache/lucene/tests/search/TestBaseExplanationTestCase.java @@ -29,7 +29,7 @@ /** * Tests that the {@link BaseExplanationTestCase} helper code, as well as {@link - * CheckHits#checkNoMatchExplanations} are checking what they are suppose to. + * CheckHits#checkNoMatchExplanations} are checking what they are supposed to. */ public class TestBaseExplanationTestCase extends BaseExplanationTestCase { diff --git a/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestFailIfDirectoryNotClosed.java b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestFailIfDirectoryNotClosed.java index f622d326c3f1..c0cddb844258 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestFailIfDirectoryNotClosed.java +++ b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestFailIfDirectoryNotClosed.java @@ -18,7 +18,7 @@ import com.carrotsearch.randomizedtesting.RandomizedTest; import org.apache.lucene.store.Directory; -import org.junit.Assert; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.junit.runner.JUnitCore; import org.junit.runner.Result; @@ -41,7 +41,10 @@ public void testFailIfDirectoryNotClosed() { RandomizedTest.assumeTrue( "Ignoring nested test, very likely zombie threads present.", r.getIgnoreCount() == 0); assertFailureCount(1, r); - Assert.assertTrue( - r.getFailures().get(0).toString().contains("Resource in scope SUITE failed to close")); + Assertions.assertThat(r.getFailures()) + .anyMatch( + failure -> { + return failure.toString().contains(AssertDirectoryClosed.MSG_PREFIX); + }); } } diff --git a/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestLuceneTestCaseJupiter.java b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestLuceneTestCaseJupiter.java new file mode 100644 index 000000000000..a1333834c4a0 --- /dev/null +++ b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestLuceneTestCaseJupiter.java @@ -0,0 +1,432 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.tests.util; + +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; + +import com.carrotsearch.randomizedtesting.jupiter.DetectThreadLeaks; +import com.carrotsearch.randomizedtesting.jupiter.SysProps; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.Supplier; +import java.util.stream.Stream; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.SuppressForbidden; +import org.assertj.core.api.Assertions; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.function.Executable; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.platform.testkit.engine.EngineTestKit; + +/// groups all tests of the junit5/ jupiter parent class and its infrastructure. +@Execution(value = ExecutionMode.SAME_THREAD, reason = "single-threaded.") +public class TestLuceneTestCaseJupiter { + /** + * A test class that calls the provided callable at a specific pointcut (callback method, test). + * This is used to make unit tests more compact. + */ + static class CallAtPointcut extends LuceneTestCaseJupiter { + enum Pointcut { + BEFORE_ALL, + BEFORE_EACH, + TEST_NORMAL, + TEST_DYNAMIC, + TEST_PARAMETERIZED, + AFTER_EACH, + AFTER_ALL, + } + + @SuppressWarnings("NonFinalStaticField") + static Map pointcuts; + + private static void call(Pointcut pointcut) throws Throwable { + if (pointcuts.containsKey(pointcut)) { + pointcuts.get(pointcut).execute(); + } + } + + @BeforeAll + static void beforeAll() throws Throwable { + call(Pointcut.BEFORE_ALL); + } + + @BeforeEach + void beforeEach() throws Throwable { + call(Pointcut.BEFORE_EACH); + } + + @Test + void normalTestMethod() throws Throwable { + call(Pointcut.TEST_NORMAL); + } + + @TestFactory + Stream dynamicTests() throws Throwable { + return Stream.of( + DynamicTest.dynamicTest( + "dynamicTest", + () -> { + call(Pointcut.TEST_DYNAMIC); + })); + } + + @ParameterizedTest + @ValueSource(strings = {"a"}) + void templateTest(String unused) throws Throwable { + call(Pointcut.TEST_PARAMETERIZED); + } + + @AfterEach + void afterEach() throws Throwable { + call(Pointcut.AFTER_EACH); + } + + @AfterAll + static void afterAll() throws Throwable { + call(Pointcut.AFTER_ALL); + } + } + + /// Test cases should use parameter-injected [java.util.Random] or a supplier + /// of [java.util.Random]. Avoid using static methods. + @Nested + class RandomInjection { + @Test + public void testRandomParameterInjected() { + testKitBuilder(T1.class) + .configurationParameter(SysProps.TESTS_SEED.propertyKey, "dead:beef:cafe") + .execute() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } + + static class T1 extends LuceneTestCaseJupiter { + public T1(Random rnd) { + Assert.assertNotNull(rnd); + } + + @BeforeAll + static void beforeAll(Random rnd) { + Assert.assertNotNull(rnd); + } + + @BeforeEach + void beforeEach(Random rnd) { + Assert.assertNotNull(rnd); + } + + @Test + void testMethod(Random rnd) { + Assert.assertNotNull(rnd); + } + + @Test + void testMethodWithSupplier(Supplier supplier) { + Assert.assertNotNull(supplier); + Assert.assertNotNull(supplier.get()); + } + + @AfterEach + void afterEach(Random rnd) { + Assert.assertNotNull(rnd); + } + + @AfterAll + static void afterAll(Random rnd) { + Assert.assertNotNull(rnd); + } + } + } + + /// Verifies that [LuceneTestCaseJupiter.SuiteFailureTracker] correctly tracks test failures + /// across various lifecycle methods of a [LuceneTestCaseJupiter] subclass. + @Nested + class SuiteFailureTracking { + private static final List forkedThreads = new ArrayList<>(); + + @AfterEach + void interruptAndJoinForkedThreads() throws InterruptedException { + for (var t : forkedThreads) t.interrupt(); + for (var t : forkedThreads) t.join(); + forkedThreads.clear(); + } + + @SuppressForbidden(reason = "Thread sleep") + private static void startSleepingThread(Duration duration) { + startThread( + "sleeping-thread", + () -> { + try { + Thread.sleep(duration.toMillis()); + } catch (InterruptedException _) { + } + }); + } + + private static Thread startThread(String name, Runnable r) { + var t = new Thread(r, name); + forkedThreads.add(t); + t.start(); + return t; + } + + @Test + public void testNoFailureLeavesMarkerClear() { + testKitBuilder(SuccessfulSuite.class) + .execute() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + Assert.assertTrue(LuceneTestCaseJupiter.suiteFailureTracker.wasSuccessful()); + } + + static class SuccessfulSuite extends LuceneTestCaseJupiter { + @Test + void testMethod() {} + } + + @TestFactory + Stream suiteFailureIsTracked() { + return Stream.of(CallAtPointcut.Pointcut.values()) + .map( + pointcut -> + DynamicTest.dynamicTest( + pointcut.name(), + () -> { + CallAtPointcut.pointcuts = Map.of(pointcut, Assertions::fail); + testKitBuilder(CallAtPointcut.class) + .execute() + .allEvents() + .assertThatEvents() + .haveAtLeast(1, event(finishedWithFailure())); + Assert.assertFalse( + LuceneTestCaseJupiter.suiteFailureTracker.wasSuccessful()); + })); + } + + @TestFactory + Stream failedAssumptionsAreNotFailures() { + return Stream.of(CallAtPointcut.Pointcut.values()) + .>mapMulti( + (pointcut, downstream) -> { + downstream.accept( + Map.entry( + pointcut, + () -> { + org.junit.jupiter.api.Assumptions.assumeTrue(false); + })); + downstream.accept( + Map.entry( + pointcut, + () -> { + org.assertj.core.api.Assumptions.assumeThat(true).isFalse(); + })); + downstream.accept( + Map.entry( + pointcut, + () -> { + Assume.assumeTrue(false); + })); + }) + .map( + entry -> { + return DynamicTest.dynamicTest( + entry.getKey().name(), + () -> { + CallAtPointcut.pointcuts = Map.ofEntries(entry); + testKitBuilder(CallAtPointcut.class).execute(); + Assert.assertTrue(LuceneTestCaseJupiter.suiteFailureTracker.wasSuccessful()); + }); + }); + } + + @Test + public void testFailBecauseOfLeakedThreads() { + testKitBuilder(FailBecauseOfLeakedThreads.class) + .execute() + .allEvents() + .assertThatEvents() + .haveAtLeast(1, event(finishedWithFailure())); + Assert.assertFalse(LuceneTestCaseJupiter.suiteFailureTracker.wasSuccessful()); + } + + @DetectThreadLeaks.LingerTime(millis = 0) + static class FailBecauseOfLeakedThreads extends LuceneTestCaseJupiter { + @Test + void testMethod() { + startSleepingThread(Duration.ofSeconds(10)); + } + } + } + + /// Verifies that failing to close a [org.apache.lucene.store.Directory] created during + /// a test causes a test failure, mirroring the behavior of [TestFailIfDirectoryNotClosed] + @Nested + class UnclosedDirectoryTracking { + @Test + void testFailIfDirectoryNotClosed() { + testKitBuilder(LeakyDirectorySuite.class) + .execute() + .allEvents() + .assertThatEvents() + .haveAtLeast( + 1, + event( + finishedWithFailure( + instanceOf(AssertionError.class), + message(msg -> msg.contains(AssertDirectoryClosed.MSG_PREFIX))))); + } + + static class LeakyDirectorySuite extends LuceneTestCaseJupiter { + @Test + void testLeaksDirectory() { + LuceneTestCaseParent.newDirectory(); + } + } + } + + @Nested + class ReproduceLinePrinter { + @TestFactory + Stream testReproLineIsPrinted() throws Exception { + return Stream.of(CallAtPointcut.Pointcut.values()) + .map( + pointcut -> { + return DynamicTest.dynamicTest( + pointcut.name(), + () -> { + String testOutput = collectOutputFrom(Map.of(pointcut, Assertions::fail)); + + Assertions.assertThat(testOutput) + .contains( + LuceneTestCaseJupiter.PrintReproduceInfoExtension.TEST_REPRO_LEAD) + .contains( + LuceneTestCaseJupiter.PrintReproduceInfoExtension.TEST_ENV_LEAD); + }); + }); + } + + @TestFactory + Stream testReproLineIsNotPrintedForAssumptions() throws Exception { + return Stream.of(CallAtPointcut.Pointcut.values()) + .map( + pointcut -> { + return DynamicTest.dynamicTest( + pointcut.name(), + () -> { + String testOutput = + collectOutputFrom( + Map.of( + pointcut, + () -> { + Assumptions.assumeTrue(false); + })); + + Assertions.assertThat(testOutput) + .doesNotContain( + LuceneTestCaseJupiter.PrintReproduceInfoExtension.TEST_REPRO_LEAD) + .doesNotContain( + LuceneTestCaseJupiter.PrintReproduceInfoExtension.TEST_ENV_LEAD); + }); + }); + } + + private static String collectOutputFrom(Map callables) + throws IOException { + String testOutput; + try (var baos = new ByteArrayOutputStream(); + var pw = new PrintStream(baos, true, StandardCharsets.UTF_8)) { + CallAtPointcut.pointcuts = callables; + LuceneTestCaseJupiter.PrintReproduceInfoExtension.debugStream = pw; + testKitBuilder(CallAtPointcut.class).execute(); + pw.flush(); + testOutput = baos.toString(StandardCharsets.UTF_8); + } finally { + LuceneTestCaseJupiter.PrintReproduceInfoExtension.debugStream = null; + CallAtPointcut.pointcuts = null; + } + return testOutput; + } + } + + @Nested + class BackCompatBehavior { + @Test + public void testIndexWriterIsRestoredAfterEachTest() throws Exception { + try { + AlterIndexSearcher.expectedValue = IndexSearcher.getMaxClauseCount(); + testKitBuilder(AlterIndexSearcher.class) + .execute() + .allEvents() + .assertThatEvents() + .doNotHave(event(finishedWithFailure())); + } finally { + IndexSearcher.setMaxClauseCount(AlterIndexSearcher.expectedValue); + } + } + + static class AlterIndexSearcher extends LuceneTestCaseJupiter { + @SuppressWarnings("NonFinalStaticField") + static int expectedValue; + + @Test + public void t1() { + Assertions.assertThat(IndexSearcher.getMaxClauseCount()).isEqualTo(expectedValue); + IndexSearcher.setMaxClauseCount(expectedValue + 1); + } + + @Test + public void t2() { + Assertions.assertThat(IndexSearcher.getMaxClauseCount()).isEqualTo(expectedValue); + IndexSearcher.setMaxClauseCount(expectedValue + 1); + } + } + } + + public static EngineTestKit.Builder testKitBuilder(Class testClass) { + return testKitBuilder().selectors(selectClass(testClass)); + } + + public static EngineTestKit.Builder testKitBuilder() { + return EngineTestKit.engine("junit-jupiter"); + } +} diff --git a/versions.lock b/versions.lock index d4c49eb2f707..5afe30f723e0 100644 --- a/versions.lock +++ b/versions.lock @@ -2,6 +2,7 @@ "comment" : "An inventory of resolved dependency versions. Do not edit this file directly.", "configurationGroups" : { "main_dependencies" : { + "com.carrotsearch.randomizedtesting:randomizedtesting-jupiter:0.3.0" : "fa9ef26b,refs=4", "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.4" : "fa9ef26b,refs=4", "com.ibm.icu:icu4j:78.3" : "47ea4550,refs=6", "commons-codec:commons-codec:1.22.0" : "e6288df0,refs=6", @@ -15,17 +16,28 @@ "org.apache.commons:commons-lang3:3.18.0" : "5ce8cdc6,refs=2", "org.apache.commons:commons-math3:3.6.1" : "85a1e4c6,refs=2", "org.apache.opennlp:opennlp-tools:2.5.9" : "2f760bab,refs=4", + "org.apiguardian:apiguardian-api:1.1.2" : "bd5e3ce4,refs=2", "org.carrot2:morfologik-fsa:2.1.9" : "79af844b,refs=4", "org.carrot2:morfologik-polish:2.1.9" : "fe494320,refs=3", "org.carrot2:morfologik-stemming:2.1.9" : "79af844b,refs=4", "org.hamcrest:hamcrest:3.0" : "fa9ef26b,refs=4", + "org.jspecify:jspecify:1.0.0" : "bd5e3ce4,refs=2", + "org.junit.jupiter:junit-jupiter:6.0.3" : "fa9ef26b,refs=4", + "org.junit.jupiter:junit-jupiter-api:6.0.3" : "fa9ef26b,refs=4", + "org.junit.jupiter:junit-jupiter-engine:6.0.3" : "fa9ef26b,refs=4", + "org.junit.jupiter:junit-jupiter-params:6.0.3" : "fa9ef26b,refs=4", + "org.junit.platform:junit-platform-commons:6.0.3" : "fa9ef26b,refs=4", + "org.junit.platform:junit-platform-engine:6.0.3" : "fa9ef26b,refs=4", + "org.junit:junit-bom:6.0.3" : "fa9ef26b,refs=4", "org.locationtech.spatial4j:spatial4j:0.8" : "cbc357ab,refs=4", "org.openjdk.jmh:jmh-core:1.37" : "85a1e4c6,refs=2", + "org.opentest4j:opentest4j:1.3.0" : "fa9ef26b,refs=4", "org.slf4j:slf4j-api:2.0.17" : "2f760bab,refs=4", "ua.net.nlp:morfologik-ukrainian-search:4.9.1" : "fe494320,refs=3", "xerces:xercesImpl:2.12.2" : "5ce8cdc6,refs=2" }, "test_dependencies" : { + "com.carrotsearch.randomizedtesting:randomizedtesting-jupiter:0.3.0" : "543b6dba,refs=74", "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.4" : "129da9bf,refs=76", "com.carrotsearch:procfork:1.0.6" : "b7ba1646,refs=2", "com.github.ben-manes.caffeine:caffeine:3.0.5" : "90685606,refs=39", @@ -51,7 +63,7 @@ "io.sgr:s2-geometry-library-java:1.0.0" : "1d5a4b2b,refs=4", "javax.inject:javax.inject:1" : "90685606,refs=39", "junit:junit:4.13.2" : "129da9bf,refs=76", - "net.bytebuddy:byte-buddy:1.18.3" : "b7ba1646,refs=2", + "net.bytebuddy:byte-buddy:1.18.3" : "39ba18cb,refs=4", "net.sf.jopt-simple:jopt-simple:5.0.4" : "152d9f78,refs=3", "net.sourceforge.nekohtml:nekohtml:1.9.22" : "6f16ff86,refs=2", "org.antlr:antlr4-runtime:4.13.2" : "6fbc4021,refs=5", @@ -59,23 +71,29 @@ "org.apache.commons:commons-lang3:3.18.0" : "6f16ff86,refs=2", "org.apache.commons:commons-math3:3.6.1" : "152d9f78,refs=3", "org.apache.opennlp:opennlp-tools:2.5.9" : "b91715f0,refs=6", - "org.assertj:assertj-core:3.27.7" : "b7ba1646,refs=2", + "org.apiguardian:apiguardian-api:1.1.2" : "cab355bc,refs=37", + "org.assertj:assertj-core:3.27.7" : "39ba18cb,refs=4", "org.carrot2:morfologik-fsa:2.1.9" : "e077a675,refs=8", "org.carrot2:morfologik-polish:2.1.9" : "cb00cecf,refs=5", "org.carrot2:morfologik-stemming:2.1.9" : "e077a675,refs=8", "org.checkerframework:checker-qual:3.19.0" : "90685606,refs=39", "org.hamcrest:hamcrest:3.0" : "129da9bf,refs=76", - "org.jspecify:jspecify:1.0.0" : "90685606,refs=39", - "org.junit.platform:junit-platform-commons:6.0.3" : "1ec8e4d2,refs=39", - "org.junit.platform:junit-platform-engine:6.0.3" : "1ec8e4d2,refs=39", - "org.junit.platform:junit-platform-launcher:6.0.3" : "1ec8e4d2,refs=39", + "org.jspecify:jspecify:1.0.0" : "8b117b59,refs=76", + "org.junit.jupiter:junit-jupiter:6.0.3" : "543b6dba,refs=74", + "org.junit.jupiter:junit-jupiter-api:6.0.3" : "543b6dba,refs=74", + "org.junit.jupiter:junit-jupiter-engine:6.0.3" : "543b6dba,refs=74", + "org.junit.jupiter:junit-jupiter-params:6.0.3" : "543b6dba,refs=74", + "org.junit.platform:junit-platform-commons:6.0.3" : "7b83bd93,refs=76", + "org.junit.platform:junit-platform-engine:6.0.3" : "7b83bd93,refs=76", + "org.junit.platform:junit-platform-launcher:6.0.3" : "09b4d43b,refs=40", + "org.junit.platform:junit-platform-testkit:6.0.3" : "882c7fc6,refs=2", "org.junit.vintage:junit-vintage-engine:6.0.3" : "1ec8e4d2,refs=39", - "org.junit:junit-bom:6.0.3" : "1ec8e4d2,refs=39", + "org.junit:junit-bom:6.0.3" : "7b83bd93,refs=76", "org.locationtech.jts:jts-core:1.20.0" : "180518e6,refs=2", "org.locationtech.spatial4j:spatial4j:0.8" : "1d5a4b2b,refs=4", "org.openjdk.jmh:jmh-core:1.37" : "152d9f78,refs=3", "org.openjdk.jmh:jmh-generator-annprocess:1.37" : "ecaf1d73,refs=1", - "org.opentest4j:opentest4j:1.3.0" : "1ec8e4d2,refs=39", + "org.opentest4j:opentest4j:1.3.0" : "7b83bd93,refs=76", "org.pcollections:pcollections:4.0.1" : "90685606,refs=39", "org.slf4j:slf4j-api:2.0.17" : "b91715f0,refs=6", "ua.net.nlp:morfologik-ukrainian-search:4.9.1" : "cb00cecf,refs=5", @@ -83,6 +101,168 @@ } }, "because" : { + "09b4d43b" : [ + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":build-tools:missing-doclet" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis.tests" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:backward-codecs" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:benchmark-jmh" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:classification" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:codecs" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:core" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:core.tests" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:demo" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:distribution.tests" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:facet" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:grouping" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:highlighter" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:join" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:memory" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:misc" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:monitor" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queries" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queryparser" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:replicator" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:sandbox" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial-extras" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial3d" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:suggest" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:common" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:icu" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:kuromoji" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:morfologik.tests" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:nori" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:phonetic" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:smartcn" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:stempel" + } + ], "129da9bf" : [ { "configuration" : "testCompileClasspath", @@ -607,6 +787,24 @@ "projectPath" : ":lucene:analysis:opennlp" } ], + "39ba18cb" : [ + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:distribution.tests" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:distribution.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:test-framework" + } + ], "47ea4550" : [ { "configuration" : "compileClasspath", @@ -633,87 +831,1007 @@ "projectPath" : ":lucene:analysis:icu" } ], - "5ce8cdc6" : [ + "543b6dba" : [ { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:benchmark" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis.tests" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:benchmark" - } - ], - "6f16ff86" : [ + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis.tests" + }, { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:benchmark" + "projectPath" : ":lucene:backward-codecs" }, { "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:backward-codecs" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:benchmark" - } - ], - "6fbc4021" : [ + }, { "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "testCompileClasspath", "projectPath" : ":lucene:benchmark-jmh" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:demo" + "projectPath" : ":lucene:benchmark-jmh" }, { "configuration" : "testCompileClasspath", - "projectPath" : ":lucene:expressions" + "projectPath" : ":lucene:classification" }, { "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:expressions" + "projectPath" : ":lucene:classification" }, { - "configuration" : "testRuntimeClasspath", - "projectPath" : ":lucene:queries" - } - ], - "79af844b" : [ - { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:luke" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:codecs" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:luke" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:codecs" }, { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:analysis:morfologik" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:analysis:morfologik" - } - ], - "85a1e4c6" : [ - { - "configuration" : "compileClasspath", - "projectPath" : ":lucene:benchmark-jmh" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:core" }, { - "configuration" : "runtimeClasspath", - "projectPath" : ":lucene:benchmark-jmh" - } - ], - "90685606" : [ + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core.tests" + }, { - "configuration" : "annotationProcessor", - "projectPath" : ":build-tools:missing-doclet" + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:core.tests" }, { - "configuration" : "annotationProcessor", - "projectPath" : ":lucene:analysis.tests" + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:demo" }, { - "configuration" : "annotationProcessor", + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:demo" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:facet" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:facet" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:grouping" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:grouping" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:highlighter" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:highlighter" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:join" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:join" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:memory" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:memory" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:misc" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:misc" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:monitor" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:monitor" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:queries" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queries" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:queryparser" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queryparser" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:replicator" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:replicator" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:sandbox" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:sandbox" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial-extras" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial-extras" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial3d" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial3d" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:suggest" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:suggest" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:common" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:common" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:icu" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:icu" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:kuromoji" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:kuromoji" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:morfologik.tests" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:morfologik.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:nori" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:nori" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:phonetic" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:phonetic" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:smartcn" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:smartcn" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:stempel" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:stempel" + } + ], + "5ce8cdc6" : [ + { + "configuration" : "compileClasspath", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:benchmark" + } + ], + "6f16ff86" : [ + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:benchmark" + } + ], + "6fbc4021" : [ + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:benchmark-jmh" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:demo" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queries" + } + ], + "79af844b" : [ + { + "configuration" : "compileClasspath", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "compileClasspath", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:analysis:morfologik" + } + ], + "7b83bd93" : [ + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":build-tools:missing-doclet" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis.tests" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:backward-codecs" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:backward-codecs" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:benchmark-jmh" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:benchmark-jmh" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:classification" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:classification" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:codecs" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:codecs" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:core" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core.tests" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:core.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:demo" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:demo" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:distribution.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:facet" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:facet" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:grouping" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:grouping" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:highlighter" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:highlighter" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:join" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:join" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:memory" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:memory" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:misc" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:misc" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:monitor" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:monitor" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:queries" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queries" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:queryparser" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:queryparser" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:replicator" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:replicator" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:sandbox" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:sandbox" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial-extras" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial-extras" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial3d" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:spatial3d" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:suggest" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:suggest" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:common" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:common" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:icu" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:icu" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:kuromoji" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:kuromoji" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:morfologik.tests" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:morfologik.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:nori" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:nori" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:phonetic" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:phonetic" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:smartcn" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:smartcn" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:stempel" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:analysis:stempel" + } + ], + "85a1e4c6" : [ + { + "configuration" : "compileClasspath", + "projectPath" : ":lucene:benchmark-jmh" + }, + { + "configuration" : "runtimeClasspath", + "projectPath" : ":lucene:benchmark-jmh" + } + ], + "882c7fc6" : [ + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testRuntimeClasspath", + "projectPath" : ":lucene:test-framework" + } + ], + "8b117b59" : [ + { + "configuration" : "annotationProcessor", + "projectPath" : ":build-tools:missing-doclet" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis.tests" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:backward-codecs" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:backward-codecs" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:benchmark-jmh" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:benchmark-jmh" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:classification" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:classification" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:codecs" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:codecs" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:core" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:core.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core.tests" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:demo" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:demo" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:distribution.tests" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:facet" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:facet" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:grouping" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:grouping" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:highlighter" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:highlighter" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:join" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:join" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:memory" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:memory" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:misc" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:misc" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:monitor" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:monitor" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:queries" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:queries" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:queryparser" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:queryparser" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:replicator" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:replicator" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:sandbox" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:sandbox" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:spatial-extras" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial-extras" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:spatial3d" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial3d" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:suggest" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:suggest" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:common" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:common" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:icu" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:icu" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:kuromoji" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:kuromoji" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:morfologik.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:morfologik.tests" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:nori" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:nori" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:phonetic" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:phonetic" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:smartcn" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:smartcn" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis:stempel" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:stempel" + } + ], + "90685606" : [ + { + "configuration" : "annotationProcessor", + "projectPath" : ":build-tools:missing-doclet" + }, + { + "configuration" : "annotationProcessor", + "projectPath" : ":lucene:analysis.tests" + }, + { + "configuration" : "annotationProcessor", "projectPath" : ":lucene:backward-codecs" }, { @@ -931,6 +2049,166 @@ "projectPath" : ":lucene:analysis:opennlp" } ], + "bd5e3ce4" : [ + { + "configuration" : "compileClasspath", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "compileClasspath", + "projectPath" : ":lucene:test-framework" + } + ], + "cab355bc" : [ + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:backward-codecs" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:benchmark" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:benchmark-jmh" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:classification" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:codecs" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:core.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:demo" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:expressions" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:facet" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:grouping" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:highlighter" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:join" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:luke" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:memory" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:misc" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:monitor" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:queries" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:queryparser" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:replicator" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:sandbox" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial-extras" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial-test-fixtures" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:spatial3d" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:suggest" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:test-framework" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:common" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:icu" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:kuromoji" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:morfologik" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:morfologik.tests" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:nori" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:opennlp" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:phonetic" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:smartcn" + }, + { + "configuration" : "testCompileClasspath", + "projectPath" : ":lucene:analysis:stempel" + } + ], "cb00cecf" : [ { "configuration" : "testRuntimeClasspath",