diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index b0d4ef606..2dcc729ae 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,27 @@ jobs: java-version: ${{ matrix.java }} cache: 'maven' - name: Test with Maven - run: ./mvnw install -B -V -D"maven.javadoc.skip"="true" -P"skipBundlePlugin,minimal-fix-latest" -D"java.util.logging.config.file"="${{github.workspace}}/quickfixj-core/src/test/resources/logging.properties" -D"http.keepAlive"="false" -D"maven.wagon.http.pool"="false" -D"maven.wagon.httpconnectionManager.ttlSeconds"="120" + shell: bash + run: | + set +e + log_file="${RUNNER_TEMP}/maven-build.log" + ./mvnw install -B -V -D"maven.javadoc.skip"="true" -P"skipBundlePlugin,minimal-fix-latest" -D"java.util.logging.config.file"="${{github.workspace}}/quickfixj-core/src/test/resources/logging.properties" -D"http.keepAlive"="false" -D"maven.wagon.http.pool"="false" -D"maven.wagon.httpconnectionManager.ttlSeconds"="120" 2>&1 | tee "${log_file}" + status=${PIPESTATUS[0]} + + if [ "${status}" -ne 0 ] && grep -q "COMPILATION ERROR" "${log_file}"; then + grep -Eo '/[^[:space:]]+\.java:\[[0-9]+,[0-9]+\]' "${log_file}" \ + | sed -E 's/:\[[0-9]+,[0-9]+\]$//' \ + | sort -u \ + | while IFS= read -r file; do + if [ -f "${file}" ]; then + echo "::group::Contents of ${file}" + cat "${file}" + echo "::endgroup::" + fi + done + fi + + exit "${status}" test-windows: runs-on: ${{ matrix.os }} @@ -53,4 +73,25 @@ jobs: java-version: ${{ matrix.java }} cache: 'maven' - name: Test with Maven on Windows - run: ./mvnw.cmd install -B -V -D"maven.javadoc.skip"="true" -P"skipBundlePlugin,minimal-fix-latest" -D"java.util.logging.config.file"="${{github.workspace}}/quickfixj-core/src/test/resources/logging.properties" -D"http.keepAlive"="false" -D"maven.wagon.http.pool"="false" -D"maven.wagon.httpconnectionManager.ttlSeconds"="120" + shell: pwsh + run: | + $logFile = Join-Path $env:RUNNER_TEMP "maven-build.log" + & ./mvnw.cmd install -B -V -D"maven.javadoc.skip"="true" -P"skipBundlePlugin,minimal-fix-latest" -D"java.util.logging.config.file"="${{github.workspace}}/quickfixj-core/src/test/resources/logging.properties" -D"http.keepAlive"="false" -D"maven.wagon.http.pool"="false" -D"maven.wagon.httpconnectionManager.ttlSeconds"="120" 2>&1 | Tee-Object -FilePath $logFile + $status = $LASTEXITCODE + + if ($status -ne 0 -and (Select-String -Path $logFile -Pattern 'COMPILATION ERROR' -Quiet)) { + $files = Select-String -Path $logFile -Pattern '(?(?:[A-Za-z]:)?[\\/][^:\s]+\.java):\[\d+,\d+\]' -AllMatches | + ForEach-Object { $_.Matches } | + ForEach-Object { $_.Groups['file'].Value } | + Sort-Object -Unique + + foreach ($file in $files) { + if (Test-Path -Path $file) { + Write-Host "::group::Contents of $file" + Get-Content -Path $file + Write-Host "::endgroup::" + } + } + } + + exit $status diff --git a/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/GenerateMojo.java b/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/GenerateMojo.java index 781c7f057..5e540335e 100644 --- a/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/GenerateMojo.java +++ b/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/GenerateMojo.java @@ -28,6 +28,8 @@ import org.apache.maven.plugins.annotations.LifecyclePhase; import java.io.File; +import java.util.ArrayList; +import java.util.List; /** * A mojo that uses the quickfix code generator to generate @@ -45,6 +47,12 @@ public class GenerateMojo extends AbstractMojo { @Parameter(defaultValue="${basedir}/src/main/quickfixj/dictionary/FIX44.xml") private File dictFile; + /** + * Optional list of dictionaries/tasks to generate in a single execution. + */ + @Parameter + private List tasks; + /** * The source directory containing *.xsd files. */ @@ -72,7 +80,7 @@ public class GenerateMojo extends AbstractMojo { /** * The package for the generated source. */ - @Parameter(required = true) + @Parameter private String packaging; /** @@ -122,29 +130,8 @@ public void execute() throws MojoExecutionException { } generator.setLog(getLog()); - MessageCodeGenerator.Task task = new MessageCodeGenerator.Task(); - if (getLog().isInfoEnabled()) { - getLog().info("Initialising code generator task"); - } - - if (dictFile != null && dictFile.exists()) { - task.setSpecification(dictFile); - } else { - getLog().error("Cannot find file " + dictFile); - throw new MojoExecutionException("File could not be found or was NULL!"); - } - - log("Processing " + dictFile); - task.setName(dictFile.getName()); - task.setTransformDirectory(schemaDirectory); - task.setMessagePackage(packaging); - task.setOutputBaseDirectory(outputDirectory); - task.setFieldPackage(fieldPackage); - task.setUtcTimestampPrecision(utcTimestampPrecision); - task.setOverwrite(overwrite); - task.setOrderedFields(orderedFields); - task.setDecimalGenerated(decimal); - generator.generate(task); + List generationTasks = createGenerationTasks(); + generator.generate(generationTasks); } catch (Throwable t) { throw new MojoExecutionException("QuickFIX/J code generator execution failed", t); } @@ -163,6 +150,135 @@ private void log(final String msg) { getLog().info(msg); } + private List createGenerationTasks() throws MojoExecutionException { + List configuredTasks; + if (tasks == null || tasks.isEmpty()) { + configuredTasks = new ArrayList<>(); + GeneratorTask singleTask = new GeneratorTask(); + singleTask.setDictFile(dictFile); + singleTask.setPackaging(packaging); + singleTask.setFieldPackage(fieldPackage); + singleTask.setUtcTimestampPrecision(utcTimestampPrecision); + singleTask.setOverwrite(overwrite); + singleTask.setOrderedFields(orderedFields); + singleTask.setDecimal(decimal); + configuredTasks.add(singleTask); + } else { + configuredTasks = tasks; + } + + List generationTasks = new ArrayList<>(configuredTasks.size()); + for (GeneratorTask configuredTask : configuredTasks) { + MessageCodeGenerator.Task task = new MessageCodeGenerator.Task(); + if (getLog().isInfoEnabled()) { + getLog().info("Initialising code generator task"); + } + + if (configuredTask.getDictFile() != null && configuredTask.getDictFile().exists()) { + task.setSpecification(configuredTask.getDictFile()); + } else { + getLog().error("Cannot find file " + configuredTask.getDictFile()); + throw new MojoExecutionException("File could not be found or was NULL!"); + } + if (configuredTask.getPackaging() == null || configuredTask.getPackaging().isEmpty()) { + throw new MojoExecutionException("Packaging could not be found or was NULL!"); + } + + log("Processing " + configuredTask.getDictFile()); + task.setName(configuredTask.getDictFile().getName()); + task.setTransformDirectory(schemaDirectory); + task.setMessagePackage(configuredTask.getPackaging()); + task.setOutputBaseDirectory(outputDirectory); + task.setFieldPackage(configuredTask.getFieldPackage() != null ? configuredTask.getFieldPackage() : fieldPackage); + task.setUtcTimestampPrecision(configuredTask.getUtcTimestampPrecision() != null + ? configuredTask.getUtcTimestampPrecision() : utcTimestampPrecision); + task.setOverwrite(configuredTask.getOverwrite() != null ? configuredTask.getOverwrite() : overwrite); + task.setOrderedFields(configuredTask.getOrderedFields() != null ? configuredTask.getOrderedFields() : orderedFields); + task.setDecimalGenerated(configuredTask.getDecimal() != null ? configuredTask.getDecimal() : decimal); + generationTasks.add(task); + } + return generationTasks; + } + + public static class GeneratorTask { + @Parameter(required = true) + private File dictFile; + + @Parameter(required = true) + private String packaging; + + @Parameter + private String fieldPackage; + + @Parameter + private String utcTimestampPrecision; + + @Parameter + private Boolean overwrite; + + @Parameter + private Boolean orderedFields; + + @Parameter + private Boolean decimal; + + public File getDictFile() { + return dictFile; + } + + public void setDictFile(File dictFile) { + this.dictFile = dictFile; + } + + public String getPackaging() { + return packaging; + } + + public void setPackaging(String packaging) { + this.packaging = packaging; + } + + public String getFieldPackage() { + return fieldPackage; + } + + public void setFieldPackage(String fieldPackage) { + this.fieldPackage = fieldPackage; + } + + public String getUtcTimestampPrecision() { + return utcTimestampPrecision; + } + + public void setUtcTimestampPrecision(String utcTimestampPrecision) { + this.utcTimestampPrecision = utcTimestampPrecision; + } + + public Boolean getOverwrite() { + return overwrite; + } + + public void setOverwrite(Boolean overwrite) { + this.overwrite = overwrite; + } + + public Boolean getOrderedFields() { + return orderedFields; + } + + public void setOrderedFields(Boolean orderedFields) { + this.orderedFields = orderedFields; + } + + public Boolean getDecimal() { + return decimal; + } + + public void setDecimal(Boolean decimal) { + this.decimal = decimal; + } + } + /** * Returns the destination directory to used during code generation. * diff --git a/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/MavenMessageCodeGenerator.java b/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/MavenMessageCodeGenerator.java index 8dea32fcb..ec6408c9c 100644 --- a/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/MavenMessageCodeGenerator.java +++ b/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/MavenMessageCodeGenerator.java @@ -34,14 +34,14 @@ public void setLog(Log log) { } protected void logInfo(String msg) { - log.info(msg); + log.info(formatLogMessage(msg)); } protected void logDebug(String msg) { - log.debug(msg); + log.debug(formatLogMessage(msg)); } protected void logError(String msg, Throwable e) { - log.error(msg, e); + log.error(formatLogMessage(msg), e); } } diff --git a/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/MessageCodeGenerator.java b/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/MessageCodeGenerator.java index 4603ef3d1..186fd6f73 100644 --- a/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/MessageCodeGenerator.java +++ b/quickfixj-codegenerator/src/main/java/org/quickfixj/codegenerator/MessageCodeGenerator.java @@ -40,11 +40,17 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintStream; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -62,6 +68,8 @@ public class MessageCodeGenerator { private static final String ORDERED_FIELDS_OPTION = "generator.orderedFields"; private static final String OVERWRITE_OPTION = "generator.overwrite"; private static final String UTC_TIMESTAMP_PRECISION_OPTION = "generator.utcTimestampPrecision"; + private static final String PARALLEL_TASK_EXECUTION_OPTION = "generator.parallelExecution"; + private static final String PARALLEL_THREAD_COUNT_OPTION = "generator.parallelThreads"; // An arbitrary serial UID which will have to be changed when messages and fields won't be compatible with next versions in terms // of java serialization. @@ -76,8 +84,18 @@ public class MessageCodeGenerator { private static final Set UTC_TIMESTAMP_PRECISION_ALLOWED_VALUES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("SECONDS", "MILLIS", "MICROS", "NANOS"))); + private final ThreadLocal logPrefix = new ThreadLocal<>(); + + protected String formatLogMessage(String msg) { + String prefix = logPrefix.get(); + if (prefix == null) { + return msg; + } + return "[" + prefix + "] " + msg; + } + protected void logInfo(String msg) { - System.out.println(msg); + System.out.println(formatLogMessage(msg)); } protected void logDebug(String msg) { @@ -85,14 +103,14 @@ protected void logDebug(String msg) { } protected void logError(String msg, Throwable e) { - System.err.println(msg); + System.err.println(formatLogMessage(msg)); e.printStackTrace(); } private void generateMessageBaseClass(Task task) throws ParserConfigurationException, SAXException, IOException, TransformerFactoryConfigurationError, TransformerException { - logInfo(task.getName() + ": generating message base class"); + logInfo("generating message base class"); Map parameters = new HashMap<>(); parameters.put(XSLPARAM_SERIAL_UID, SERIAL_UID_STR); generateClassCode(task, "Message", parameters); @@ -114,7 +132,7 @@ private void generateClassCode(Task task, String className, Map throws ParserConfigurationException, SAXException, IOException, TransformerFactoryConfigurationError, TransformerException { - logDebug("generating " + className + " for " + task.getName()); + logDebug("generating " + className); if (parameters == null) { parameters = new HashMap<>(); } @@ -130,7 +148,7 @@ private void generateFieldClasses(Task task) throws ParserConfigurationException IOException { String outputDirectory = task.getOutputBaseDirectory() + "/" + task.getFieldDirectory() + "/"; - logInfo(task.getName() + ": generating field classes in " + outputDirectory); + logInfo("generating field classes in " + outputDirectory); writePackageDocumentation(outputDirectory, "FIX field definitions for " + task.getName()); Document document = getSpecification(task); List fieldNames = getNames(document.getDocumentElement(), "fields/field"); @@ -172,7 +190,7 @@ private void generateFieldClasses(Task task) throws ParserConfigurationException private void generateMessageSubclasses(Task task) throws ParserConfigurationException, SAXException, IOException, TransformerFactoryConfigurationError, TransformerException { - logInfo(task.getName() + ": generating message subclasses"); + logInfo("generating message subclasses"); String outputDirectory = task.getOutputBaseDirectory() + "/" + task.getMessageDirectory() + "/"; writePackageDocumentation(outputDirectory, "Message classes"); @@ -195,7 +213,7 @@ private void generateMessageSubclasses(Task task) throws ParserConfigurationExce private void generateComponentClasses(Task task) throws ParserConfigurationException, SAXException, IOException, TransformerFactoryConfigurationError, TransformerException { - logInfo(task.getName() + ": generating component classes"); + logInfo("generating component classes"); String outputDirectory = task.getOutputBaseDirectory() + "/" + task.getMessageDirectory() + "/component/"; Document document = getSpecification(task); @@ -234,7 +252,7 @@ private Transformer createTransformer(Task task, String xsltFile) return transformerFactory.newTransformer(styleSource); } - private final Map specificationCache = new HashMap<>(); + private final Map specificationCache = new ConcurrentHashMap<>(); private Document getSpecification(Task task) throws ParserConfigurationException, SAXException, IOException { @@ -308,24 +326,29 @@ private void generateCodeFile(Task task, Document document, Map } DOMSource source = new DOMSource(document); - FileOutputStream fos = new FileOutputStream(outputFile); - BufferedOutputStream bos = new BufferedOutputStream(fos); + OutputStream outputStream = createOutputStream(outputFile); try { - StreamResult result = new StreamResult(bos); + StreamResult result = new StreamResult(outputStream); transformer.transform(source, result); } finally { try { - bos.close(); + outputStream.close(); } catch (IOException ioe) { logError("error closing " + outputFile, ioe); } } } + protected OutputStream createOutputStream(File outputFile) throws FileNotFoundException { + return new BufferedOutputStream(new FileOutputStream(outputFile)); + } + /* * Generate the Message and Field related source code. */ public void generate(Task task) { + String previousLogPrefix = logPrefix.get(); + logPrefix.set(task.getName()); try { generateFieldClasses(task); generateMessageBaseClass(task); @@ -337,7 +360,89 @@ public void generate(Task task) { throw e; } catch (Exception e) { throw new CodeGenerationException(e); + } finally { + if (previousLogPrefix == null) { + logPrefix.remove(); + } else { + logPrefix.set(previousLogPrefix); + } + } + } + + /* + * Generate the Message and Field related source code for multiple tasks. + */ + public void generate(List tasks) { + if (tasks == null || tasks.isEmpty()) { + return; + } + int totalTasks = tasks.size(); + if (!getOption(PARALLEL_TASK_EXECUTION_OPTION, true) || totalTasks == 1) { + for (int i = 0; i < totalTasks; i++) { + processTaskWithProgress(tasks.get(i), i + 1, totalTasks); + } + return; + } + int parallelism = getParallelism(totalTasks); + logInfo("parallel task execution enabled with " + parallelism + " worker(s) for " + + totalTasks + " task(s)"); + ExecutorService executor = Executors.newFixedThreadPool(parallelism); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < totalTasks; i++) { + Task task = tasks.get(i); + int taskIndex = i + 1; + futures.add(executor.submit(() -> processTaskWithProgress(task, taskIndex, totalTasks))); + } + for (Future future : futures) { + try { + future.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new CodeGenerationException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new CodeGenerationException(cause); + } + } + } finally { + executor.shutdownNow(); + } + } + + private void processTaskWithProgress(Task task, int taskIndex, int totalTasks) { + String taskName = task.getName(); + if (taskName == null || taskName.isEmpty()) { + taskName = "unnamed"; + } + logInfo("Started task for " + taskName + " (" + taskIndex + " / " + totalTasks + ")"); + try { + generate(task); + } finally { + logInfo("Finished task for " + taskName + " (" + taskIndex + " / " + totalTasks + ")"); + } + } + + private int getParallelism(int totalTasks) { + int defaultParallelism = Math.min(totalTasks, Math.max(2, Runtime.getRuntime().availableProcessors())); + String configuredParallelThreads = getOption(PARALLEL_THREAD_COUNT_OPTION, null); + if (configuredParallelThreads == null) { + return defaultParallelism; + } + try { + int configuredParallelism = Integer.parseInt(configuredParallelThreads.trim()); + if (configuredParallelism > 0) { + return Math.min(totalTasks, configuredParallelism); + } + } catch (NumberFormatException ignored) { + // ignored, fallback to default below } + logInfo("ignoring invalid " + PARALLEL_THREAD_COUNT_OPTION + " value '" + configuredParallelThreads + + "', using " + defaultParallelism + " worker(s)"); + return defaultParallelism; } public static class Task { @@ -464,6 +569,7 @@ public static void main(String[] args) { long start = System.currentTimeMillis(); final String[] versions = { "FIXT 1.1", "FIX 5.0", "FIX 4.4", "FIX 4.3", "FIX 4.2", "FIX 4.1", "FIX 4.0" }; + List tasks = new ArrayList<>(); for (String ver : versions) { Task task = new Task(); task.setName(ver); @@ -477,8 +583,9 @@ public static void main(String[] args) { task.setOverwrite(overwrite); task.setOrderedFields(orderedFields); task.setDecimalGenerated(useDecimal); - codeGenerator.generate(task); + tasks.add(task); } + codeGenerator.generate(tasks); double duration = System.currentTimeMillis() - start; DecimalFormat durationFormat = new DecimalFormat("#.###"); codeGenerator.logInfo("Time for generation: " diff --git a/quickfixj-codegenerator/src/test/java/org/quickfixj/codegenerator/ParallelExecutionOptionTest.java b/quickfixj-codegenerator/src/test/java/org/quickfixj/codegenerator/ParallelExecutionOptionTest.java new file mode 100644 index 000000000..84ab8482e --- /dev/null +++ b/quickfixj-codegenerator/src/test/java/org/quickfixj/codegenerator/ParallelExecutionOptionTest.java @@ -0,0 +1,132 @@ +package org.quickfixj.codegenerator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.junit.After; +import org.junit.Test; + +public class ParallelExecutionOptionTest { + + private static final String PARALLEL_OPTION = "generator.parallelExecution"; + private static final String PARALLEL_THREADS_OPTION = "generator.parallelThreads"; + + @After + public void clearParallelOptions() { + System.clearProperty(PARALLEL_OPTION); + System.clearProperty(PARALLEL_THREADS_OPTION); + } + + @Test + public void testSequentialExecutionWhenParallelOptionIsDisabled() { + System.setProperty(PARALLEL_OPTION, "false"); + + TrackingMessageCodeGenerator generator = new TrackingMessageCodeGenerator(); + generator.generate(createTasks(4)); + + assertEquals(1, generator.getMaxConcurrentTasks()); + assertTrue(generator.containsInfoLog("Started task for task-0 (1 / 4)")); + assertTrue(generator.containsInfoLog("Finished task for task-3 (4 / 4)")); + } + + @Test + public void testParallelExecutionWhenParallelOptionIsEnabled() { + System.setProperty(PARALLEL_OPTION, "true"); + + TrackingMessageCodeGenerator generator = new TrackingMessageCodeGenerator(); + generator.generate(createTasks(4)); + + assertTrue(generator.getMaxConcurrentTasks() > 1); + assertTrue(generator.containsInfoLog("parallel task execution enabled with")); + assertTrue(generator.containsInfoLog("for 4 task(s)")); + assertTrue(generator.containsInfoLog("Started task for task-0 (1 / 4)")); + assertTrue(generator.containsInfoLog("Finished task for task-3 (4 / 4)")); + } + + @Test + public void testParallelExecutionWhenParallelThreadsAreConfigured() { + System.setProperty(PARALLEL_OPTION, "true"); + System.setProperty(PARALLEL_THREADS_OPTION, "2"); + + TrackingMessageCodeGenerator generator = new TrackingMessageCodeGenerator(); + generator.generate(createTasks(4)); + + assertEquals(2, generator.getMaxConcurrentTasks()); + assertTrue(generator.containsInfoLog("parallel task execution enabled with 2 worker(s)")); + assertTrue(generator.containsInfoLog("for 4 task(s)")); + } + + @Test + public void testParallelExecutionIsEnabledByDefault() { + System.clearProperty(PARALLEL_OPTION); + + TrackingMessageCodeGenerator generator = new TrackingMessageCodeGenerator(); + generator.generate(createTasks(4)); + + assertTrue(generator.getMaxConcurrentTasks() > 1); + assertTrue(generator.containsInfoLog("parallel task execution enabled with")); + assertTrue(generator.containsInfoLog("for 4 task(s)")); + assertTrue(generator.containsFinishLogForAllTasks(4)); + } + + private static List createTasks(int count) { + List tasks = new ArrayList<>(); + for (int i = 0; i < count; i++) { + MessageCodeGenerator.Task task = new MessageCodeGenerator.Task(); + task.setName("task-" + i); + tasks.add(task); + } + return tasks; + } + + private static class TrackingMessageCodeGenerator extends MessageCodeGenerator { + private final AtomicInteger currentConcurrentTasks = new AtomicInteger(); + private final AtomicInteger maxConcurrentTasks = new AtomicInteger(); + private final List infoMessages = new CopyOnWriteArrayList<>(); + + @Override + public void generate(Task task) { + int concurrentTaskCount = currentConcurrentTasks.incrementAndGet(); + maxConcurrentTasks.accumulateAndGet(concurrentTaskCount, Math::max); + try { + Thread.sleep(100L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + currentConcurrentTasks.decrementAndGet(); + } + } + + int getMaxConcurrentTasks() { + return maxConcurrentTasks.get(); + } + + @Override + protected void logInfo(String msg) { + infoMessages.add(msg); + } + + boolean containsInfoLog(String token) { + for (String infoMessage : infoMessages) { + if (infoMessage.contains(token)) { + return true; + } + } + return false; + } + + boolean containsFinishLogForAllTasks(int taskCount) { + for (int i = 0; i < taskCount; i++) { + if (!containsInfoLog("Finished task for task-" + i)) { + return false; + } + } + return true; + } + } +} diff --git a/quickfixj-codegenerator/src/test/java/org/quickfixj/codegenerator/ParallelFieldGenerationRaceTest.java b/quickfixj-codegenerator/src/test/java/org/quickfixj/codegenerator/ParallelFieldGenerationRaceTest.java new file mode 100644 index 000000000..efa074dba --- /dev/null +++ b/quickfixj-codegenerator/src/test/java/org/quickfixj/codegenerator/ParallelFieldGenerationRaceTest.java @@ -0,0 +1,218 @@ +package org.quickfixj.codegenerator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Stream; + +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class ParallelFieldGenerationRaceTest { + + private static final String PARALLEL_OPTION = "generator.parallelExecution"; + private static final String PARALLEL_THREADS_OPTION = "generator.parallelThreads"; + private static final int TOTAL_FIELDS = 1000; + private static final int PARALLEL_TASKS = 16; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @After + public void clearParallelOptions() { + System.clearProperty(PARALLEL_OPTION); + System.clearProperty(PARALLEL_THREADS_OPTION); + } + + @Test + public void testParallelSharedOutputAgainstGoldenSource() throws Exception { + File transformDirectory = new File("./src/main/resources/org/quickfixj/codegenerator"); + File dictionaryTwoEnums = createDictionary("two-enums", false); + File dictionaryThreeEnums = createDictionary("three-enums", true); + + // Use a plain generator for the single-task golden baseline so that the + // barrier (which requires two parties) does not deadlock. + File goldenOutput = tempFolder.newFolder("golden"); + new MessageCodeGenerator().generate( + createTask("golden", dictionaryThreeEnums, transformDirectory, goldenOutput)); + Map goldenFieldSources = collectFieldSources(goldenOutput); + assertEquals(TOTAL_FIELDS, goldenFieldSources.size()); + + // For every output file path, keep a CyclicBarrier(2) so that the first + // two threads that open the same file must both reach their first write() + // call before either is allowed to proceed. This forces two writers to + // be simultaneously mid-write on every field file, turning the + // probabilistic race into a near-deterministic one. + ConcurrentHashMap barriers = new ConcurrentHashMap<>(); + MessageCodeGenerator generator = new MessageCodeGenerator() { + @Override + protected OutputStream createOutputStream(File outputFile) throws FileNotFoundException { + CyclicBarrier barrier = barriers.computeIfAbsent( + outputFile.getAbsolutePath(), k -> new CyclicBarrier(2)); + return new FilterOutputStream(super.createOutputStream(outputFile)) { + private boolean awaited = false; + + @Override + public void write(byte[] b, int off, int len) throws IOException { + awaitBarrierOnce(); + out.write(b, off, len); + } + + @Override + public void write(int b) throws IOException { + awaitBarrierOnce(); + out.write(b); + } + + private void awaitBarrierOnce() throws IOException { + if (!awaited) { + awaited = true; + try { + barrier.await(10, TimeUnit.SECONDS); + } catch (BrokenBarrierException | TimeoutException e) { + // Proceed: either the barrier timed out because an odd + // number of threads wrote this file, or it was broken + // by a previous exception. + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + }; + } + }; + + File sharedOutput = tempFolder.newFolder("shared-output"); + List tasks = new ArrayList<>(); + for (int i = 0; i < PARALLEL_TASKS; i++) { + File dictionary = i % 2 == 0 ? dictionaryTwoEnums : dictionaryThreeEnums; + tasks.add(createTask("race-" + i, dictionary, transformDirectory, sharedOutput)); + } + + System.setProperty(PARALLEL_OPTION, "true"); + System.setProperty(PARALLEL_THREADS_OPTION, Integer.toString(PARALLEL_TASKS)); + generator.generate(tasks); + + assertFirstDifferenceOnly(goldenFieldSources, collectFieldSources(sharedOutput)); + } + + private MessageCodeGenerator.Task createTask(String name, File dictionary, File transformDirectory, + File outputDirectory) { + MessageCodeGenerator.Task task = new MessageCodeGenerator.Task(); + task.setName(name); + task.setSpecification(dictionary); + task.setTransformDirectory(transformDirectory); + task.setMessagePackage("quickfix.race"); + task.setOutputBaseDirectory(outputDirectory); + task.setFieldPackage("quickfix.field"); + task.setOverwrite(true); + task.setOrderedFields(true); + task.setDecimalGenerated(true); + return task; + } + + private Map collectFieldSources(File outputDirectory) throws Exception { + Map sources = new TreeMap<>(); + Path fieldDir = outputDirectory.toPath().resolve("quickfix/field"); + try (Stream stream = Files.walk(fieldDir)) { + stream.filter(path -> path.toString().endsWith(".java")).forEach(path -> { + try { + String relative = fieldDir.relativize(path).toString(); + String content = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + sources.put(relative, content); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + return sources; + } + + private void assertFirstDifferenceOnly(Map expected, Map actual) { + for (Map.Entry entry : expected.entrySet()) { + String className = entry.getKey(); + if (!actual.containsKey(className)) { + fail("First failing class: " + className + "\nDifference: class missing in generated output"); + } + + String expectedSource = entry.getValue(); + String actualSource = actual.get(className); + if (!expectedSource.equals(actualSource)) { + fail(buildFirstDifferenceMessage(className, expectedSource, actualSource)); + } + } + + for (String className : actual.keySet()) { + if (!expected.containsKey(className)) { + fail("First failing class: " + className + + "\nDifference: unexpected class present in generated output"); + } + } + } + + private String buildFirstDifferenceMessage(String className, String expected, String actual) { + String[] expectedLines = expected.split("\\R", -1); + String[] actualLines = actual.split("\\R", -1); + int lineCount = Math.min(expectedLines.length, actualLines.length); + for (int i = 0; i < lineCount; i++) { + if (!expectedLines[i].equals(actualLines[i])) { + return "First failing class: " + className + + "\nDifference at line " + (i + 1) + + "\nExpected: " + expectedLines[i] + + "\nActual: " + actualLines[i]; + } + } + + return "First failing class: " + className + + "\nDifference: source length mismatch" + + "\nExpected lines: " + expectedLines.length + + "\nActual lines: " + actualLines.length; + } + + private File createDictionary(String name, boolean withExtraEnum) throws Exception { + File dictionary = tempFolder.newFile("RaceCondition-" + name + ".xml"); + StringBuilder xml = new StringBuilder(); + int minorVersion = withExtraEnum ? 4 : 2; + xml.append("\n"); + xml.append("\n"); + xml.append("
\n"); + xml.append(" \n"); + xml.append(" \n"); + xml.append(" \n"); + for (int i = 1; i <= TOTAL_FIELDS; i++) { + int tag = 10000 + i; + xml.append(" \n"); + xml.append(" \n"); + xml.append(" \n"); + if (withExtraEnum) { + xml.append(" \n"); + } + xml.append(" \n"); + } + xml.append(" \n"); + xml.append("\n"); + Files.write(dictionary.toPath(), xml.toString().getBytes(StandardCharsets.UTF_8)); + return dictionary; + } +} diff --git a/quickfixj-messages/quickfixj-messages-all/pom.xml b/quickfixj-messages/quickfixj-messages-all/pom.xml index 571d10e57..300cc6540 100644 --- a/quickfixj-messages/quickfixj-messages-all/pom.xml +++ b/quickfixj-messages/quickfixj-messages-all/pom.xml @@ -249,99 +249,61 @@ ${project.version} - fix40 + all-fix-versions generate - ../quickfixj-messages-fix40/src/main/resources/FIX40.xml - quickfix.fix40 - quickfix.field - ${generator.decimal} - - - - fix41 - - generate - - - ../quickfixj-messages-fix41/src/main/resources/FIX41.xml - quickfix.fix41 - quickfix.field - ${generator.decimal} - - - - fix42 - - generate - - - ../quickfixj-messages-fix42/src/main/resources/FIX42.xml - quickfix.fix42 - quickfix.field - ${generator.decimal} - - - - fix43 - - generate - - - ../quickfixj-messages-fix43/src/main/resources/FIX43.xml - quickfix.fix43 - quickfix.field - ${generator.decimal} - - - - fix44 - - generate - - - ../quickfixj-messages-fix44/src/main/resources/FIX44.modified.xml - quickfix.fix44 - quickfix.field - ${generator.decimal} - - - - fix50 - - generate - - - ../quickfixj-messages-fix50/src/main/resources/FIX50.xml - quickfix.fix50 - quickfix.field - ${generator.decimal} - - - - fix50sp1 - - generate - - - ../quickfixj-messages-fix50sp1/src/main/resources/FIX50SP1.modified.xml - quickfix.fix50sp1 - quickfix.field - ${generator.decimal} - - - - fix50sp2 - - generate - - - ../quickfixj-messages-fix50sp2/src/main/resources/FIX50SP2.modified.xml - quickfix.fix50sp2 - quickfix.field - ${generator.decimal} + + + ../quickfixj-messages-fix40/src/main/resources/FIX40.xml + quickfix.fix40 + quickfix.field + ${generator.decimal} + + + ../quickfixj-messages-fix41/src/main/resources/FIX41.xml + quickfix.fix41 + quickfix.field + ${generator.decimal} + + + ../quickfixj-messages-fix42/src/main/resources/FIX42.xml + quickfix.fix42 + quickfix.field + ${generator.decimal} + + + ../quickfixj-messages-fix43/src/main/resources/FIX43.xml + quickfix.fix43 + quickfix.field + ${generator.decimal} + + + ../quickfixj-messages-fix44/src/main/resources/FIX44.modified.xml + quickfix.fix44 + quickfix.field + ${generator.decimal} + + + ../quickfixj-messages-fix50/src/main/resources/FIX50.xml + quickfix.fix50 + quickfix.field + ${generator.decimal} + + + ../quickfixj-messages-fix50sp1/src/main/resources/FIX50SP1.modified.xml + quickfix.fix50sp1 + quickfix.field + ${generator.decimal} + + + ../quickfixj-messages-fix50sp2/src/main/resources/FIX50SP2.modified.xml + quickfix.fix50sp2 + quickfix.field + ${generator.decimal} + +