DevUtil.java

/**
 * (C) Copyright IBM Corporation 2019, 2026
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.openliberty.tools.common.plugins.util;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.ServerSocket;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardWatchEventKinds;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.nio.file.Watchable;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Scanner;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.jar.Attributes;
import java.util.jar.JarFile;
import java.util.jar.Manifest;

import javax.tools.JavaCompiler;
import javax.tools.JavaFileObject;
import javax.tools.StandardJavaFileManager;
import javax.tools.StandardLocation;
import javax.tools.ToolProvider;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import com.sun.nio.file.SensitivityWatchEventModifier;

import io.openliberty.tools.ant.ServerTask;
import io.openliberty.tools.common.plugins.util.ServerFeatureUtil.FeaturesPlatforms;

import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.NameFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.commons.io.monitor.FileAlterationListener;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationObserver;
import org.json.JSONException;
import org.json.JSONObject;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

/**
 * Utility class for dev mode.
 */
public abstract class DevUtil extends AbstractContainerSupportUtil {

    private static final String START_SERVER_MESSAGE_PREFIX = "CWWKF0011I:";
    private static final String START_APP_MESSAGE_REGEXP = "CWWKZ0001I:";
    private static final String UPDATED_APP_MESSAGE_REGEXP = "CWWKZ0003I:";
    private static final String STOPPED_APP_MESSAGE_REGEXP = "CWWKZ0009I:";
    private static final String PORT_IN_USE_MESSAGE_PREFIX = "CWWKO0221E:";
    private static final String WEB_APP_AVAILABLE_MESSAGE_PREFIX = "CWWKT0016I:";
    private static final String LISTENING_ON_PORT_MESSAGE_PREFIX = "CWWKO0219I:";
    private static final String HTTP_PREFIX = "http://";
    private static final String HTTP_PREFIX_ESCAPED = "http:\\/\\/";
    private static final String HTTPS_PREFIX = "https://";
    private static final String HTTPS_PREFIX_ESCAPED = "https:\\/\\/";
    private static final String DEVMODE_DIR_NAME = "/devmode";
    public static final String DEVMODE_PROJECT_ROOT = "io.openliberty.tools.projectRoot";
    private static final String GENERATED_HEADER_REGEX = "# Generated by liberty-.*-plugin";
    private static final String DEVMODE_CONTAINER_BASE_NAME = "liberty-dev";
    private static final String DEVC_CONTAINER_DOCKER = "docker";
    private static final String DEVC_CONTAINER_PODMAN = "podman";
    private static final String DEVMODE_IMAGE_SUFFIX = "-dev-mode";
    public static final String SKIP_BETA_INSTALL_WARNING = "skipBetaInstallFeatureWarning";
    public static final String DEVC_HIDDEN_FOLDER = ".libertyDevc";

    private static final String[] IGNORE_DIRECTORY_PREFIXES = new String[] { "." };
    private static final String[] IGNORE_FILE_PREFIXES = new String[] { "." };
    private static final String[] IGNORE_FILE_POSTFIXES = new String[] {
            // core dumps
            ".dmp",
            // vim
            "~",
            // intellij
            "___jb_tmp___", "___jb_old___" };

    private static final String[] DEFAULT_COMPILER_OPTIONS = new String[] { "-g", "-parameters" };
    private static final int LIBERTY_DEFAULT_HTTP_PORT = 9080;
    private static final int LIBERTY_DEFAULT_HTTPS_PORT = 9443;

    /**
     * Log debug
     * 
     * @param msg
     */
    public abstract void debug(String msg);

    /**
     * Log debug
     * 
     * @param msg
     * @param e
     */
    public abstract void debug(String msg, Throwable e);

    /**
     * Log debug
     * 
     * @param e
     */
    public abstract void debug(Throwable e);

    /**
     * Log warning
     * 
     * @param msg
     */
    public abstract void warn(String msg);

    /**
     * Log info
     * 
     * @param msg
     */
    public abstract void info(String msg);

    /**
     * Log error
     * 
     * @param msg
     */
    public abstract void error(String msg);

    /**
     * Log error
     * 
     * @param msg
     * @param e
     */
    public abstract void error(String msg, Throwable e);

    /**
     * Returns whether debug is enabled by the current logger
     * 
     * @return whether debug is enabled
     */
    public abstract boolean isDebugEnabled();

    /**
     * Recompile the build file
     * 
     * @param buildFile
     * @param compileArtifactPaths
     * @param testArtifactPaths
     * @param generateFeatures     boolean, true if auto generation of features is
     *                             on
     * @param executor             The thread pool executor
     * @throws PluginExecutionException if there was an error when restarting the
     *                                  server
     * @return true if the build file was recompiled with changes
     */
    public abstract boolean recompileBuildFile(File buildFile, Set<String> compileArtifactPaths,
            Set<String> testArtifactPaths, boolean generateFeatures, ThreadPoolExecutor executor)
            throws PluginExecutionException;

    /**
     * Updates the compile artifact paths of the given project module. Only used in
     * multi module scenario.
     * 
     * @param projectModule    The corresponding project module to update artifact
     *                         paths for
     * @param redeployCheck    Whether to redeploy the application if changes in the
     *                         dependencies are detected
     * @param generateFeatures boolean, true if auto generation of features is on
     * @param executor         The thread pool executor
     * @return true if the compile artifact paths are updated
     * @throws PluginExecutionException if there was an error when restarting the
     *                                  server
     */
    public abstract boolean updateArtifactPaths(ProjectModule projectModule, boolean redeployCheck,
            boolean generateFeatures, ThreadPoolExecutor executor) throws PluginExecutionException;

    /**
     * Update the compile artifact paths of any child modules of the given build
     * file.
     * 
     * @param parentBuildFile The parent build file
     * @return true if the compile artifact paths are updated
     */
    public abstract boolean updateArtifactPaths(File parentBuildFile);

    /**
     * Run the unit tests
     * 
     * @param buildFile corresponding build file to run tests on
     * @throws PluginScenarioException  if unit tests failed
     * @throws PluginExecutionException if unit tests could not be run
     */
    public abstract void runUnitTests(File buildFile) throws PluginScenarioException, PluginExecutionException;

    /**
     * Run the integration tests
     * 
     * @param buildFile corresponding buildFile to run tests on
     * @throws PluginScenarioException  if integration tests failed
     * @throws PluginExecutionException if integration tests could not be run
     */
    public abstract void runIntegrationTests(File buildFile) throws PluginScenarioException, PluginExecutionException;

    /**
     * Check the configuration file for new features and install features if needed
     * 
     * @param configFile
     * @param serverDir
     * @param generateFeatures
     */
    public abstract void installFeatures(File configFile, File serverDir, boolean generateFeatures);

    /**
     * Get the ServerFeatureUtil object
     * 
     * @return ServerFeatureUtil object
     */
    public abstract ServerFeatureUtil getServerFeatureUtilObj();

    /**
     * Get the set of existing features
     * 
     * @return existing features set
     */
    public abstract Set<String> getExistingFeatures();

    /**
     * Update the existing features list using the files in the server directory.
     * Called on configuration file change to ensure we have the most current
     * feature list.
     */
    public abstract void updateExistingFeatures();

    /**
     * Compile the specified directory
     * 
     * @param dir
     * @return
     */
    public abstract boolean compile(File dir);

    /**
     * Compile the specified directory for the project module
     * 
     * @param dir
     * @param project project module (used in multi-module scenario)
     * @return
     */
    public abstract boolean compile(File dir, ProjectModule project);

    /**
     * Stop the server
     */
    public abstract void stopServer();

    /**
     * Get the ServerTask to start the server, which can be in either "run" or
     * "debug" mode
     * 
     * @return ServerTask the task to start the server
     * @throws Exception if there was an error copying/creating config files
     */
    public abstract ServerTask getServerTask() throws Exception;

    /**
     * Redeploy the application
     */
    public abstract void redeployApp() throws PluginExecutionException;

    /**
     * Get an example command using the server start timeout parameter. The example
     * command is unique to each plugin.
     *
     * @return String containing the example command
     */
    public abstract String getServerStartTimeoutExample();

    /**
     * Get the name of the current project running dev mode.
     *
     * @return String of the project name
     */
    public abstract String getProjectName();

    /**
     * Is the application deployed as a loose application.
     */
    public abstract boolean isLooseApplication();

    /**
     * Get the loose application configuration file.
     * @return File loose application configuration file
     */
    public abstract File getLooseApplicationFile();

    private enum FileTrackMode {
        NOT_SET, FILE_WATCHER, POLLING
    }

    private File serverDirectory;
    private File sourceDirectory;
    private File testSourceDirectory;
    private File configDirectory;
    private File projectDirectory;
    private File multiModuleProjectDirectory;
    protected List<File> resourceDirs;
    // Not all webResource dirs need to be monitored, but those for which a Maven filtering will be applied do, since they can't be added to the loose app as source
    protected List<Path> monitoredWebResourceDirs;
    private boolean hotTests;
    private Path tempConfigPath;
    private boolean changeOnDemandTestsAction;
    private boolean skipTests;
    private boolean skipUTs;
    private boolean skipITs;
    private String applicationId;
    private int appStartupTimeout;
    private int appUpdateTimeout;
    private Thread serverThread;
    private PluginExecutionException serverThreadException;

    /** If user stopped dev mode manually, this is true. If an external process caused dev mode to stop, this is false */
    private AtomicBoolean devStop;
    
    private String hostName;
    private String httpPort;
    private String httpsPort;
    private String containerHttpPort;
    private String containerHttpsPort;
    private final long compileWaitMillis;
    private AtomicBoolean inputUnavailable;
    private int alternativeDebugPort = -1;
    private boolean libertyDebug;
    private int libertyDebugPort;
    private AtomicBoolean detectedAppStarted;
    private long serverStartTimeout;
    private boolean useBuildRecompile;
    private Map<File, Properties> propertyFilesMap;
    final private Set<FileAlterationObserver> fileObservers;
    final private Set<FileAlterationObserver> newFileObservers;
    final private Set<FileAlterationObserver> cancelledFileObservers;
    private AtomicBoolean calledShutdownHook;
    private boolean gradle;
    private long pollingInterval;
    private FileTrackMode trackingMode;
    private final boolean container;
    private String imageName;
    private String containerName;
    private File containerfile;
    private File containerBuildContext;
    private Path tempContainerfilePath = null;
    private String containerRunOpts;
    private volatile Process containerRunProcess;
    private File defaultContainerfile;
    private int containerBuildTimeout;
    private boolean skipDefaultPorts;
    private boolean keepTempContainerfile;
    protected List<String> srcMount = new ArrayList<String>();
    protected List<String> destMount = new ArrayList<String>();
    private boolean firstStartup = true;
    private Set<Path> containerfileDirectoriesToWatch = new HashSet<Path>();
    private Set<Path> containerfileDirectoriesTracked = new HashSet<Path>();
    private Set<WatchKey> containerfileDirectoriesWatchKeys = new HashSet<WatchKey>();
    private Set<FileAlterationObserver> containerfileDirectoriesFileObservers = new HashSet<FileAlterationObserver>();
    private JavaCompilerOptions compilerOptions;
    private final String mavenCacheLocation;
    private AtomicBoolean externalContainerShutdown;
    private AtomicBoolean shownFeaturesShWarning;
    protected AtomicBoolean hasFeaturesSh;
    protected AtomicBoolean serverFullyStarted;
    private final File buildDirectory;
    private List<ProjectModule> upstreamProjects; // supports multi module scenario, null for single module projects
    private boolean recompileDependencies;
    private String packagingType;
    protected File buildFile;
    /** Map of parent build files (parent build file, list of children build files) */
    protected Map<String, List<String>> parentBuildFiles;
    private boolean generateFeatures;
    private boolean generateToSrc;
    private Set<String> generatedFeaturesSet; // set of features in generated-features.xml file
    private boolean generatedFeaturesModified;
    private boolean generatedFeaturesCopied;
    private Set<String> compileArtifactPaths;
    private Set<String> testArtifactPaths;
    protected File generateFeaturesFile; // the file that is created from the generate-features goal/task
    protected File generateFeaturesOutputDir; // output directory for the generate-features goal/task (i.e. where the file is generated)
    protected File generateFeaturesTmpDir; // the location where the generated features file is written during dev mode loop when generateToSrc is false
    private File modifiedSrcBuildFile;

    protected boolean skipInstallFeature;
    // for gradle, this map will be kept as null
    protected Map<String, Boolean> projectRecompileMap;

    // constructor for maven
    public DevUtil(File buildDirectory, File serverDirectory, File sourceDirectory, File testSourceDirectory,
                   File configDirectory, File projectDirectory, File multiModuleProjectDirectory, List<File> resourceDirs, boolean changeOnDemandTestsAction,
                   boolean hotTests, boolean skipTests, boolean skipUTs, boolean skipITs, boolean skipInstallFeature, String applicationId,
                   long serverStartTimeout, int appStartupTimeout, int appUpdateTimeout, long compileWaitMillis,
                   boolean libertyDebug, boolean useBuildRecompile, boolean gradle, boolean pollingTest, boolean container,
                   File containerfile, File containerBuildContext, String containerRunOpts, int containerBuildTimeout,
                   boolean skipDefaultPorts, JavaCompilerOptions compilerOptions, boolean keepTempContainerfile,
                   String mavenCacheLocation, List<ProjectModule> upstreamProjects, boolean recompileDependencies,
                   String packagingType, File buildFile, Map<String, List<String>> parentBuildFiles, boolean generateFeatures, boolean generateToSrc,
                   Set<String> compileArtifactPaths, Set<String> testArtifactPaths, List<Path> monitoredWebResourceDirs, Map<String, Boolean> projectRecompileMap) {
        this(buildDirectory, serverDirectory, sourceDirectory, testSourceDirectory,
                configDirectory, projectDirectory, multiModuleProjectDirectory, resourceDirs, changeOnDemandTestsAction,
                hotTests, skipTests, skipUTs, skipITs, skipInstallFeature, applicationId,
                serverStartTimeout, appStartupTimeout, appUpdateTimeout, compileWaitMillis,
                libertyDebug, useBuildRecompile, gradle, pollingTest, container,
                containerfile, containerBuildContext, containerRunOpts, containerBuildTimeout,
                skipDefaultPorts, compilerOptions, keepTempContainerfile,
                mavenCacheLocation, upstreamProjects, recompileDependencies,
                packagingType, buildFile, parentBuildFiles, generateFeatures, generateToSrc,
                compileArtifactPaths, testArtifactPaths, monitoredWebResourceDirs);
        // setting projectRecompileMap as empty if input is null from ci.maven
        this.projectRecompileMap = projectRecompileMap != null ? projectRecompileMap : new HashMap<>();
    }

    // constructor for gradle
    public DevUtil(File buildDirectory, File serverDirectory, File sourceDirectory, File testSourceDirectory,
            File configDirectory, File projectDirectory, File multiModuleProjectDirectory, List<File> resourceDirs, boolean changeOnDemandTestsAction,
            boolean hotTests, boolean skipTests, boolean skipUTs, boolean skipITs, boolean skipInstallFeature, String applicationId,
            long serverStartTimeout, int appStartupTimeout, int appUpdateTimeout, long compileWaitMillis,
            boolean libertyDebug, boolean useBuildRecompile, boolean gradle, boolean pollingTest, boolean container,
            File containerfile, File containerBuildContext, String containerRunOpts, int containerBuildTimeout,
            boolean skipDefaultPorts, JavaCompilerOptions compilerOptions, boolean keepTempContainerfile,
            String mavenCacheLocation, List<ProjectModule> upstreamProjects, boolean recompileDependencies,
            String packagingType, File buildFile, Map<String, List<String>> parentBuildFiles, boolean generateFeatures, boolean generateToSrc,
            Set<String> compileArtifactPaths, Set<String> testArtifactPaths, List<Path> monitoredWebResourceDirs) {
        this.buildDirectory = buildDirectory;
        this.serverDirectory = serverDirectory;
        this.sourceDirectory = sourceDirectory;
        this.testSourceDirectory = testSourceDirectory;
        this.configDirectory = configDirectory;
        this.projectDirectory = projectDirectory;
        this.multiModuleProjectDirectory = multiModuleProjectDirectory;
        this.resourceDirs = resourceDirs;
        this.changeOnDemandTestsAction = changeOnDemandTestsAction;
        this.hotTests = hotTests;
        this.skipTests = skipTests;
        this.skipUTs = skipUTs;
        this.skipITs = skipITs;
        this.skipInstallFeature = skipInstallFeature;
        this.applicationId = applicationId;
        this.serverStartTimeout = serverStartTimeout;
        this.appStartupTimeout = appStartupTimeout;
        this.appUpdateTimeout = appUpdateTimeout;
        this.devStop = new AtomicBoolean(false);
        this.compileWaitMillis = compileWaitMillis;
        this.inputUnavailable = new AtomicBoolean(false);
        this.libertyDebug = libertyDebug;
        this.detectedAppStarted = new AtomicBoolean(false);
        this.useBuildRecompile = useBuildRecompile;
        this.calledShutdownHook = new AtomicBoolean(false);
        this.gradle = gradle;
        this.fileObservers = new HashSet<FileAlterationObserver>();
        this.newFileObservers = new HashSet<FileAlterationObserver>();
        this.cancelledFileObservers = new HashSet<FileAlterationObserver>();
        this.pollingInterval = 100;
        if (pollingTest) {
            this.trackingMode = FileTrackMode.POLLING;
        } else {
            this.trackingMode = FileTrackMode.NOT_SET;
        }
        this.container = container;
        this.containerfile = containerfile;
        this.containerBuildContext = containerBuildContext;
        this.containerRunOpts = containerRunOpts;

        if (projectDirectory != null) {
            //Use Containerfile if it exists, but default to Dockerfile if both present or neither exist
            File defaultDockerFile = new File(projectDirectory, "Dockerfile");
            File userDefaultContainerFile = new File(projectDirectory, "Containerfile");
            if (!defaultDockerFile.exists() && userDefaultContainerFile.exists()) {
                this.defaultContainerfile = userDefaultContainerFile;
            } else {
                this.defaultContainerfile = defaultDockerFile;
            }
        }
        if (containerBuildTimeout < 1) {
            this.containerBuildTimeout = 600;
        } else {
            this.containerBuildTimeout = containerBuildTimeout;
        }
        this.skipDefaultPorts = skipDefaultPorts;
        this.compilerOptions = compilerOptions;
        this.keepTempContainerfile = keepTempContainerfile;
        this.mavenCacheLocation = mavenCacheLocation;
        this.upstreamProjects = upstreamProjects;
        this.recompileDependencies = recompileDependencies;
        this.externalContainerShutdown = new AtomicBoolean(false);
        this.shownFeaturesShWarning = new AtomicBoolean(false);
        this.hasFeaturesSh = new AtomicBoolean(false);
        this.serverFullyStarted = new AtomicBoolean(false);
        this.packagingType = packagingType;
        this.buildFile = buildFile;
        if (parentBuildFiles == null) {
            this.parentBuildFiles = new HashMap<String, List<String>>();
        } else {
            this.parentBuildFiles = parentBuildFiles;
        }
        this.generateFeatures = generateFeatures;
        this.generateToSrc = generateToSrc;
        this.generateFeaturesTmpDir = new File(buildDirectory, FeatureGeneratorUtil.GENERATED_FEATURES_TEMP_DIR);
        initGenerationContext();
        this.compileArtifactPaths = compileArtifactPaths;
        this.testArtifactPaths = testArtifactPaths;
        this.monitoredWebResourceDirs = monitoredWebResourceDirs;
        this.generatedFeaturesModified = false;
        this.generatedFeaturesCopied = false;
        if (this.generateFeatures) {
            this.generatedFeaturesSet = getGeneratedFeatures();
        } else {
            this.generatedFeaturesSet = new HashSet<String>();
        }
        this.modifiedSrcBuildFile = null;
    }

    private void initGenerationContext() {
        this.generateFeaturesOutputDir = generateToSrc ? configDirectory : generateFeaturesTmpDir;
        this.generateFeaturesFile = new File(generateFeaturesOutputDir, FeatureGeneratorUtil.GENERATED_FEATURES_FILE_PATH);
    }

    public void copyGeneratedFeaturesFile(File destinationDir) throws IOException {
        copyFile(generateFeaturesFile, generateFeaturesOutputDir, destinationDir, null);
        if (destinationDir.equals(serverDirectory)) {
            generatedFeaturesCopied = true; // features copied into server dir and not some temp dir
        }
    }
    /**
     * Run unit and/or integration tests
     * 
     * @param waitForApplicationUpdate Whether to wait for the application to update
     *                                 before running integration tests
     * @param messageOccurrences       The previous number of times the application
     *                                 updated message has appeared.
     * @param executor                 The thread pool executor
     * @param forceSkipTests           Whether to force skip all tests
     * @param forceSkipUTs             Whether to force skip the unit tests
     * @param forceSkipITs             Whether to force skip the integration tests
     * @param currentBuildFile         The build file to run tests against
     * @param projectName              The name of the current project, null if only
     *                                 one project exists
     */
    public void runTests(boolean waitForApplicationUpdate, int messageOccurrences, ThreadPoolExecutor executor,
            boolean forceSkipTests, boolean forceSkipUTs, boolean forceSkipITs, File currentBuildFile,
            String projectName) {
        debug("Running tests for: " + currentBuildFile + "; skipTests: " + forceSkipTests + "; skipITs: " + forceSkipITs
                + "; skipUTs: " + forceSkipUTs);
        if (!forceSkipTests) {
            ServerTask serverTask = null;
            try {
                serverTask = getServerTask();
            } catch (Exception e) {
                // not expected since server should already have been started
                error("Could not get the server task for running tests.", e);
            }

            File logFile = getMessagesLogFile(serverTask);

            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
                debug("Thread interrupted while waiting to start tests.", e);
            }

            // if queue size >= 1, it means a newer test has been queued so we
            // should skip this and let that run instead
            if (executor.getQueue().size() >= 1) {
                Runnable head = executor.getQueue().peek();
                boolean manualInvocation = ((TestJob) head).isManualInvocation();

                if (manualInvocation) {
                    debug("Tests were re-invoked before previous tests began. Cancelling previous tests and resubmitting them.");
                } else {
                    debug("Changes were detected before tests began. Cancelling tests and resubmitting them.");
                }
                return;
            }

            // skip unit tests if invoked by Gradle
            if (!gradle && !(forceSkipUTs)) {
                if (projectName != null) {
                    info("Running unit tests for " + projectName + " ...");
                } else {
                    info("Running unit tests...");
                }
                try {
                    runUnitTests(currentBuildFile);
                    if (projectName != null) {
                        info("Unit tests for " + projectName + " finished.");
                        info("");
                    } else {
                        info("Unit tests finished.");
                        info("");
                    }
                } catch (PluginScenarioException e) {
                    debug(e);
                    error(e.getMessage());
                    // if unit tests failed, don't run integration tests
                    return;
                } catch (PluginExecutionException e) {
                    error(e.getMessage(), e);
                }
            }

            // if queue size >= 1, it means a newer test has been queued so we
            // should skip this and let that run instead
            if (executor.getQueue().size() >= 1) {
                Runnable head = executor.getQueue().peek();
                boolean manualInvocation = ((TestJob) head).isManualInvocation();

                if (manualInvocation) {
                    info("Tests were invoked while previous tests were running. Restarting tests.");
                } else {
                    info("Changes were detected while tests were running. Restarting tests.");
                }
                return;
            }

            if (!forceSkipITs) {
                if (!detectedAppStarted.get()) {
                    // very first time app is started wait for START_APP or UPDATED_APP message
                    if (appStartupTimeout < 0) {
                        warn("The verifyTimeout (verifyAppStartTimeout) value needs to be an integer greater than or equal to 0.  The default value of 30 seconds will be used.");
                        appStartupTimeout = 30;
                    }
                    long timeout = appStartupTimeout * 1000;

                    // Wait for the app started message in messages.log
                    info("Waiting up to " + appStartupTimeout
                            + " seconds to find the application start up or update message...");
                    String startMessage = serverTask.waitForStringInLog(
                            "(" + START_APP_MESSAGE_REGEXP + "|" + UPDATED_APP_MESSAGE_REGEXP + ")",
                            timeout, logFile);
                    if (startMessage == null) {
                        error("Unable to verify if the application was started after " + appStartupTimeout
                                + " seconds.  Consider increasing the verifyTimeout value if this continues to occur.");
                    } else {
                        detectedAppStarted.set(true);
                    }
                } else if (waitForApplicationUpdate) {
                    // wait until application has been updated
                    int timesStopped = serverTask.countStringOccurrencesInFile(STOPPED_APP_MESSAGE_REGEXP, logFile);
                    int timesUpdated = serverTask.countStringOccurrencesInFile(UPDATED_APP_MESSAGE_REGEXP, logFile);
                    debug("timesStopped=" + timesStopped + " timesUpdated=" + timesUpdated);
                    if (timesStopped > timesUpdated) {
                        // timesStopped == timesUpdated indicates the app is already updated and no wait for update required
                        if (appUpdateTimeout < 0) {
                            appUpdateTimeout = 5;
                        }
                        long timeout = appUpdateTimeout * 1000;
                        serverTask.waitForUpdatedStringInLog(UPDATED_APP_MESSAGE_REGEXP, timeout, logFile, messageOccurrences);
                    }
                }

                if (gradle) {
                    info("Running tests...");
                } else {
                    if (projectName != null) {
                        info("Running integration tests for " + projectName + "...");
                    } else {
                        info("Running integration tests...");
                    }
                }
                try {
                    runIntegrationTests(currentBuildFile);
                    if (gradle) {
                        info("Tests finished.");
                    } else {
                        if (projectName != null) {
                            info("Integration tests for " + projectName + " finished.");
                            info("");
                        } else {
                            info("Integration tests finished.");
                            info("");
                        }
                    }
                } catch (PluginScenarioException e) {
                    debug(e);
                    error(e.getMessage());
                    // if unit tests failed, don't run integration tests
                    return;
                } catch (PluginExecutionException e) {
                    error(e.getMessage(), e);
                }
            }
        } else {
            if (projectName != null) {
                info("Tests will not run on demand for " + projectName + " because skipTests is set to true.");
            } else {
                info("Tests will not run on demand because skipTests is set to true.");
            }
        }
    }

    /**
     * Get the number of times the application updated message has appeared in the
     * application log
     * 
     * @return the number of times the application has updated
     */
    public int countApplicationUpdatedMessages() {
        int messageOccurrences = -1;
        if (!(skipTests || skipITs)) {
            try {
                ServerTask serverTask = getServerTask();
                File logFile = getMessagesLogFile(serverTask);
                String regexp = UPDATED_APP_MESSAGE_REGEXP;
                messageOccurrences = serverTask.countStringOccurrencesInFile(regexp, logFile);
                debug("Message occurrences before compile: " + messageOccurrences);
            } catch (Exception e) {
                debug("Failed to get message occurrences before compile", e);
            }
        }
        return messageOccurrences;
    }

    /**
     * Get the log file from server directory if using container, or from server task otherwise.
     * 
     * @param serverTask the server task
     * @return the messages log file for the server
     */
    private File getMessagesLogFile(ServerTask serverTask) {
        File logFile;
        if (container) {
            logFile = new File(serverDirectory, "logs/messages.log");
        } else {
            logFile = serverTask.getLogFile();           
        }
        return logFile;
    }

    public void startServer() throws PluginExecutionException {
        startServer(true, true);
    }

    /**
     * Start the server and keep it running in a background thread.
     * 
     * @param buildContainer  Force a Docker build when in container mode. Ignored
     *                        otherwise.
     * @param pullParentImage If buildContainer is true, this determines whether the
     *                        Docker build should also pull the latest parent image.
     *                        Ignored otherwise.
     * 
     * @throws PluginExecutionException If the server startup could not be verified
     *                                  within the timeout, or server startup
     *                                  failed.
     */
    public void startServer(boolean buildContainer, boolean pullParentImage) throws PluginExecutionException {
        try {
            final ServerTask serverTask;
            try {
                serverTask = getServerTask();
            } catch (Exception e) {
                throw new PluginExecutionException("An error occurred while starting the server: " + e.getMessage(), e);
            }

            // Set debug variables in server.env if debug enabled
            enableServerDebug();

            if (container) {
                if (!checkDockerVersion()) {
                    // Did not find valid docker or podman installation
                    throw new PluginExecutionException("Could not find a valid Docker or Podman installation.");
                }
            }

            // build container image if in container mode
            if (container && buildContainer) {
                File containerfileToUse = getContainerfile();
                debug("Containerfile to use: " + containerfileToUse);
                if (containerfileToUse.exists()) {
                    // The build context comes from the specified containerBuildContext (or the user's Containerfile location by default)
                    File buildContext = containerBuildContext == null ? containerfileToUse.getParentFile() : containerBuildContext;
                    String buildContextString = buildContext.getAbsolutePath();
                    debug("Container build context: " + buildContextString);

                    File tempContainerfile = prepareTempContainerfile(containerfileToUse, buildContextString);
                    buildContainerImage(tempContainerfile, containerfileToUse, pullParentImage, buildContext);
                } else {
                    // this message is mainly for the default containerfile scenario, since the containerfile parameter was already validated in Maven/Gradle plugin.
                    throw new PluginExecutionException("No Containerfile or Dockerfile was found at " + containerfileToUse.getAbsolutePath() + ". Create a Containerfile/Dockerfile at the specified location to use dev mode with container support. For an example of how to configure a Dockerfile, see https://github.com/OpenLiberty/ci.docker");
                }
            }

            String logsDirectory = serverDirectory.getCanonicalPath() + "/logs";
            final File messagesLogFile = new File(logsDirectory + "/messages.log");

            // Watch logs directory if it already exists
            boolean logsExist = new File(logsDirectory).isDirectory();

            // Start server
            serverThread = new Thread(new Runnable() {

                @Override
                public void run() {
                    try {
                        if (container) {
                            startContainer();
                        } else {
                            serverTask.execute();
                        }
                    } catch (RuntimeException e) {
                        // If devStop is true server was stopped with Ctl-c, do not throw exception
                        if (devStop.get() == false) {
                            // If a runtime exception occurred in the server task, log and set the exception field
                            PluginExecutionException e2;
                            if (container) {
                                e2 = new PluginExecutionException("An error occurred while running the container: " + e.getMessage(), e);
                            } else {
                                e2 = new PluginExecutionException("An error occurred while starting the server: " + e.getMessage(), e);
                            }
                            error(e2.getMessage());
                            serverThreadException = e2;
                        }
                    } catch (PluginExecutionException pe) {
                        error(pe.getMessage());
                        serverThreadException = pe;
                    }
                }
            });
            serverThread.start();

            // If the server thread dies at any point after this, allow the error to say
            // that the server stopped
            setDevStop(false);

            // If there were already logs from a previous server run, wait for it to be updated.
            if (logsExist) {
                final AtomicBoolean messagesModified = new AtomicBoolean(false);

                // If logs already exist, then watch the directory to ensure
                // messages.log is modified before continuing.
                FileFilter singleFileFilter = new FileFilter() {
                    @Override
                    public boolean accept(File file) {
                        try {
                            if (file.getCanonicalFile().equals(messagesLogFile.getCanonicalFile())) {
                                return true;
                            }
                        } catch (IOException e) {
                            if (file.equals(messagesLogFile)) {
                                return true;
                            }
                        }
                        return false;
                    }
                };

                FileAlterationObserver observer = new FileAlterationObserver(logsDirectory, singleFileFilter);
                observer.addListener(new FileAlterationListenerAdaptor() {
                    @Override
                    public void onFileCreate(File file) {
                        messagesModified.set(true);
                    }

                    @Override
                    public void onFileChange(File file) {
                        messagesModified.set(true);
                    }
                });

                try {
                    observer.initialize();
                    while (!messagesModified.get()) {
                        checkStopDevMode(false); // stop dev mode if the server thread was terminated
                        observer.checkAndNotify();
                        // wait for the log file to update during server startup
                        Thread.sleep(500);
                    }
                    debug("messages.log has been changed");
                } catch (PluginScenarioException e) {
                    if (serverThreadException != null) {
                        throw serverThreadException;
                    } else {
                        // the server/container failed to start, so wrap this as an execution exception
                        throw new PluginExecutionException(e);
                    }
                } catch (Exception e) {
                    error("An error occurred while waiting for the server to update messages.log: " + e.getMessage(), e);
                } finally {
                    try {
                        observer.destroy();
                    } catch (Exception e) {
                        debug("Could not destroy FileAlterationObserver for logs directory " + logsDirectory, e);
                    }
                }
            } else {
                // Wait until log exists
                try {
                    while (!messagesLogFile.exists()) {
                        checkStopDevMode(false); // stop dev mode if the server thread was terminated
                        // wait for the log file to appear during server startup
                        Thread.sleep(500);
                    }
                    debug("messages.log has been created");
                } catch (PluginScenarioException e) {
                    if (serverThreadException != null) {
                        throw serverThreadException;
                    } else {
                        // the server/container failed to start, so wrap this as an execution exception
                        throw new PluginExecutionException(e);
                    }
                } catch (Exception e) {
                    error("An error occurred while waiting for the server to create messages.log: " + e.getMessage(), e);
                }
            }
            // Set server start timeout
            if (serverStartTimeout < 0) {
                warn("The serverStartTimeout value needs to be an integer greater than or equal to 0.  The default value of 90 seconds will be used.");
                serverStartTimeout = 90;
            }
            long serverStartTimeoutMillis = serverStartTimeout * 1000;
            // Wait for the server started message in messages.log
            String startMessage = serverTask.waitForStringInLog(START_SERVER_MESSAGE_PREFIX, serverStartTimeoutMillis,
                    messagesLogFile);
            if (startMessage == null) {
                setDevStop(true);
                if (container) {
                    stopContainer();
                } else {
                    stopServer();
                }
                throw new PluginExecutionException("The server has not started within " + serverStartTimeout + " seconds. " +
                        "Consider increasing the server start timeout if this continues to occur. " +
                        "For example, " + getServerStartTimeoutExample());
            } else {
                serverFullyStarted.set(true);
            }
            // Check for port already in use error
            String portError = serverTask.findStringInFile(PORT_IN_USE_MESSAGE_PREFIX, messagesLogFile);
            if (portError != null) {
                error(portError.split(PORT_IN_USE_MESSAGE_PREFIX)[1]);
            }
            // Parse hostname, http, https ports for integration tests to use
            parseHostNameAndPorts(serverTask, messagesLogFile);
        } catch (IOException e) {
            throw new PluginExecutionException("An error occurred while starting the server: " + e.getMessage(), e);
        }
    }


    private File getContainerfile() {
        return containerfile != null ? containerfile : defaultContainerfile;
    }

    protected List<String> readContainerfile(File containerfile) throws PluginExecutionException {
        // Convert Containerfile to List of strings for each line
        List<String> containerfileLines = null;
        try {
            containerfileLines = Files.readAllLines(containerfile.toPath());
        } catch (IOException e) {
            error("Failed to read Containerfile located at " + containerfile);
            throw new PluginExecutionException("Could not read Containerfile " + containerfile + ": " + e.getMessage(), e);
        }
        return containerfileLines;
    }

    /**
     * Get escape character from the escape directive at the top of the Containerfile.
     * Docker documents a couple of directives, but it seems escape must always be the first line to work.
     */
    protected static char getEscapeCharacter(List<String> containerfileLines) throws PluginExecutionException {
        if (containerfileLines.size() > 0) {
            // Remove white space from the beginning and end of the line
            String pendingLine = containerfileLines.get(0).trim();
            int directiveSymbolIndex = pendingLine.indexOf("#");
            if (directiveSymbolIndex >= 0) {
                String contentAfterSymbol = pendingLine.substring(directiveSymbolIndex + 1, pendingLine.length());
                // trim again after removing preceding symbol
                String directive = contentAfterSymbol.trim();
                String[] split = directive.split("=");
                if (split.length == 2 && split[0].trim().equalsIgnoreCase("escape")) {
                    String escapeChar = split[1].trim();
                    if (escapeChar.length() > 0) {
                        // Get the first char, and don't validate here whether it's a valid escape char
                        // but just let Docker/Podman fail the build if it determines that it is invalid.
                        return escapeChar.charAt(0);
                    }
                }
            }
        }
        return '\\';
    }

    /**
     * Trim all lines and get them without comments or empty lines after the first FROM command
     * (so that directives at the beginning of the file are preserved)
     */
    protected static List<String> getCleanedLines(List<String> containerfileLines) throws PluginExecutionException {
        List<String> result = new ArrayList<String>();
        boolean fromFound = false;
        for (String line : containerfileLines) {
            // Remove white space from the beginning and end of the line
            String pendingLine = line.trim();
            if (!fromFound) {
                if (pendingLine.startsWith("FROM")) {
                    fromFound = true;
                } else {
                    // until we find a FROM line, just keep all the lines
                    result.add(pendingLine);
                    continue;
                }
            }
            int commentIndex = pendingLine.indexOf("#");
            if (commentIndex >= 0) {
                String contentBeforeSymbol = pendingLine.substring(0, commentIndex);
                // trim again after removing trailing comment
                pendingLine = contentBeforeSymbol.trim();
            }
            if (!pendingLine.isEmpty()) {
                result.add(pendingLine);
            }
        }
        return result;
    }

    /**
     * Combine multi-line commands into single lines. Requires that getCleanedLines() be called first.
     */
    protected static List<String> getCombinedLines(List<String> containerfileLines, char escape) throws PluginExecutionException {
        List<String> result = new ArrayList<String>();
        int i = 0;
        while (i < containerfileLines.size()) {
            String pendingLine = containerfileLines.get(i).trim();
            int multilineIndex;
            int j = i+1;
            while (pendingLine.length() > 0 && !pendingLine.startsWith("#") && (pendingLine.charAt(pendingLine.length() - 1) == escape) && j < containerfileLines.size()) {
                multilineIndex = pendingLine.length() - 1;
                String contentBeforeSymbol = pendingLine.substring(0, multilineIndex);
                String nextLine = containerfileLines.get(j);
                String combined = contentBeforeSymbol + nextLine;
                pendingLine = combined.trim(); // trim the combined string to remove whitespace around any further line escapes
                j++;
            }
            result.add(pendingLine);
            i = j;
        }
        return result;
    }

    protected void removeWarFileLines(List<String> containerfileLines) throws PluginExecutionException {
        removeFileExtensionLines(containerfileLines, ".war");
    }

    protected void removeEarFileLines(List<String> containerfileLines) throws PluginExecutionException {
        removeFileExtensionLines(containerfileLines, ".ear");
    }

    private void removeFileExtensionLines(List<String> containerfileLines, String extension) throws PluginExecutionException {
        List<String> fileExtensionLines = new ArrayList<String>();
        for (String line : containerfileLines) {
            // Remove white space from the beginning and end of the line
            String trimLine = line.trim();
            if (!trimLine.startsWith("#") && trimLine.toLowerCase().contains(extension)) {
                // Break the Containerfile line down into segments based on any amount of whitespace.
                // The command must be to the left of any comments.
                String[] cmdSegments = trimLine.split("#")[0].split("\\s+");
                // if the line starts with COPY and the second to last segment ends with extension, it is a COPY line of that file type
                if (cmdSegments[0].equalsIgnoreCase("COPY") || cmdSegments[0].equalsIgnoreCase("ADD")) {
                    if (cmdSegments.length < 3) {
                        throw new PluginExecutionException("Incorrect syntax on this line in the Containerfile: '" + line + 
                        "'. There must be at least two arguments for the COPY or ADD command, a source path and a destination path.");
                    }
                    if (cmdSegments[cmdSegments.length - 2].toLowerCase().endsWith(extension)) {
                        fileExtensionLines.add(line);
                    }
                }
            }
        }
        debug(extension + " file lines: " + fileExtensionLines.toString());
        containerfileLines.removeAll(fileExtensionLines);
    }

    /**
     * Disables OpenJ9 SCC to speed up build times by injecting environment variable before "RUN configure.sh" is called.
     * Reference: https://github.com/OpenLiberty/ci.docker#openj9-shared-class-cache-scc
     * <br>
     * Note: lines must have been trimmed and cleaned of comments using getCleanedLines() before calling this.
     */
    protected void disableOpenJ9SCC(List<String> containerfileLines) {
        final String RUN_CONFIGURE_COMMAND_LOWERCASE = "run configure.sh";
        for (int i=0; i<containerfileLines.size(); i++) {
            String line = containerfileLines.get(i);
            // RUN command is case insensitive, so use lowercase matching.
            if (line.toLowerCase().equals(RUN_CONFIGURE_COMMAND_LOWERCASE)) {
                debug("Detected RUN configure.sh command.  Skipping OpenJ9 Shared Class Cache.");
                containerfileLines.add(i, "ENV OPENJ9_SCC=false");
                return;
            }
        }
    }

    protected void detectFeaturesSh(List<String> containerfileLines) {
        // Reset features.sh warning flag
        shownFeaturesShWarning.set(false);

        final String FEATURES_SH_COMMAND_LOWERCASE = "run features.sh";
        for (int i=0; i<containerfileLines.size(); i++) {
            String line = containerfileLines.get(i);
            // RUN command is case insensitive, so use lowercase matching.
            if (line.toLowerCase().equals(FEATURES_SH_COMMAND_LOWERCASE)) {
                debug("Detected RUN features.sh command.");
                hasFeaturesSh.set(true);
                return;
            }
        }
        // if not detected, reset to false in case the Containerfile is being rebuilt
        debug("Did not find RUN features.sh command.");
        hasFeaturesSh.set(false);
    }

    protected void processCopyLines(List<String> containerfileLines, String buildContext) throws PluginExecutionException {
        srcMount.clear();
        destMount.clear();
        for (String line : containerfileLines) {
            // Remove white space from the beginning and end of the line
            String trimLine = line.trim();
            if (!trimLine.startsWith("#")) {
                // Break the Containerfile line down into segments based on any amount of whitespace.
                // The command must be to the left of any comments.
                String[] cmdSegments = trimLine.split("#")[0].split("\\s+");
                // If the line starts with COPY or ADD
                if (cmdSegments[0].equalsIgnoreCase("COPY") || cmdSegments[0].equalsIgnoreCase("ADD")) {
                    if (cmdSegments.length < 3) { // preliminary check but some of these segments could be options
                        throw new PluginExecutionException("Incorrect syntax on this line in the Containerfile: '" + line + 
                        "'. There must be at least two arguments for the COPY or ADD command, a source path and a destination path.");
                    }
                    if (line.contains("$")) {
                        warn("The Containerfile line '" + line + "' will not be able to be hot deployed to the dev mode container. Dev mode does not currently support environment variables in COPY or ADD commands. If you make changes to files specified by this line, type 'r' and press Enter to rebuild the container image and restart the container.");
                        continue;
                    }
                    List<String> srcOrDestArguments = new ArrayList<String>();
                    boolean skipLine = false;
                    for (int i = 1; i < cmdSegments.length; i++) { // start after the word COPY (or ADD)
                        String segment = cmdSegments[i];
                        if (segment.startsWith("--from")) {
                            // multi-stage build, COPY only (not ADD), give a warning
                            warn("The Containerfile line '" + line + "' will not be able to be hot deployed to the dev mode container. Dev mode does not currently support hot deployment with multi-stage COPY commands.");
                            skipLine = true; // don't mount the dirs in this COPY command
                            break;
                        } else if (segment.startsWith("--")) {
                            continue; // ignore options
                        } else {
                            srcOrDestArguments.add(segment);
                        }
                    }
                    if (skipLine) {
                        continue;
                    }
                    if (srcOrDestArguments.size() < 2) { // proper check for number of src and dest args
                        throw new PluginExecutionException("Incorrect syntax on this line in the Containerfile: '" + line + 
                        "'. There must be at least two arguments for the COPY or ADD command, a source path and a destination path.");
                    }
                    // dest is the last argument
                    String dest = srcOrDestArguments.get(srcOrDestArguments.size() - 1);
                    List<String> srcArguments = srcOrDestArguments.subList(0, srcOrDestArguments.size() - 1);
                    for (String src : srcArguments) {
                        if (isURL(src)) {
                            debug("COPY/ADD do not watch/mount URL:" + src);
                            continue;
                        }
                        String sourcePath = buildContext + "/" + src;
                        File sourceFile = new File(sourcePath);
                        if (src.contains("*") || src.contains("?")) {
                            warn("The COPY or ADD source " + src + " in the Containerfile line '" + line + "' will not be able to be hot deployed to the dev mode container. Dev mode does not currently support wildcards in the COPY or ADD commands. If you make changes to files specified by this line, type 'r' and press Enter to rebuild the container image and restart the container.");
                        } else if (sourceFile.isDirectory() || cmdSegments[0].equalsIgnoreCase("ADD")) {
                            synchronized(containerfileDirectoriesToWatch) {
                                try {
                                    containerfileDirectoriesToWatch.add(sourceFile.getCanonicalFile().toPath());
                                    debug("COPY/ADD line=" + line + ", src=" + sourcePath + ", added to containerfileDirectoriesToWatch: " + sourceFile);
                                } catch (IOException e) {
                                    // Do not fail here.  Let the Docker/Podman build fail instead.
                                    error("Could not resolve the canonical path of the directory specified in the Containerfile: " + sourcePath, e);
                                }
                            }
                        } else {
                            // No need to validate existence of the file, just let the Docker/Podman build fail
                            String destMountString = formatDestMount(dest, sourceFile);
                            srcMount.add(sourcePath);
                            destMount.add(destMountString);
                            debug("COPY line=" + line + ", src=" + sourcePath + ", dest=" + destMountString);
                        }
                    }
                }
            }
        }
    }
    
    /**
     * 
     * The intention here is to give a chance to run logic for an app update
     * but not necessarily a full re-deploy
     * 
     * @throws PluginExecutionException
     */
    protected abstract void updateLooseApp() throws PluginExecutionException;

    private String formatDestMount(String destMountString, File srcMountFile) {
        // Cannot mount a file onto a directory, so must add a filename to the end of the destination argument for mounting
        if (destMountString.endsWith("/") || destMountString.endsWith("\\")) {
            destMountString += srcMountFile.getName();
        }
        return destMountString;
    }

    /**
     * Check the name used in Containerfile for URL e.g. ADD https://repo.maven.apache.org/maven2/postgres9.jar /lib/
     * @param name the source name in a copy or add command
     * @return true if the name is a URL
     */
    private boolean isURL(String name) {
        try {
            URL url = new URL(name);
        } catch (MalformedURLException m) {
            return false;
        }
        return true;
    }

    protected File prepareTempContainerfile(File containerfile, String buildContextString) throws PluginExecutionException {
        // Create a temp Containerfile to build image from

        List<String> containerfileLines = readContainerfile(containerfile);
        char escape = getEscapeCharacter(containerfileLines);
        containerfileLines = getCleanedLines(containerfileLines);
        containerfileLines = getCombinedLines(containerfileLines, escape);
        removeWarFileLines(containerfileLines);
        removeEarFileLines(containerfileLines);
        processCopyLines(containerfileLines, buildContextString);
        detectFeaturesSh(containerfileLines);
        disableOpenJ9SCC(containerfileLines);
        for (String line : containerfileLines) {
            debug(line);
        }

        File tempContainerfile = null;
        try {
            debug("Creating temp Containerfile...");
            File devcHiddenFolder = new File(buildDirectory, DEVC_HIDDEN_FOLDER);
            devcHiddenFolder.mkdirs();
            tempContainerfile = File.createTempFile("tempContainerfile", "", devcHiddenFolder);
            debug("temp Containerfile: " + tempContainerfile);
            tempContainerfilePath = tempContainerfile.toPath(); // save name to clean up later
            if (keepTempContainerfile) {
                info("Keeping temporary Containerfile: "+tempContainerfilePath);
            } else {
                // set the tempContainerfile to be deleted when the JVM exits
                tempContainerfile.deleteOnExit();
            }
            Files.write(tempContainerfile.toPath(), containerfileLines, StandardCharsets.UTF_8);
        } catch (IOException e) {
            error("Failed to create temp Containerfile");
            throw new PluginExecutionException("Could not create temp Containerfile: " + e.getMessage(), e);
        }
        return tempContainerfile;
    }

    private void buildContainerImage(File tempContainerfile, File userContainerfile, boolean pullParentImage, File buildContext) throws PluginExecutionException {
        info("Building container image...");

        try {
            imageName = getProjectName() + DEVMODE_IMAGE_SUFFIX;
            // Name rules: may contain lowercase letters, digits and a period, one or two underscores, or one or more dashes. Cannot start with dash.
            imageName = imageName.replaceAll("[^a-zA-Z0-9]", "-").replaceAll("^[\\-]+", "").toLowerCase();

            // Use List<String> for building the command. This will be converted to a String[] when passed to ProcessBuilder. 
            // The ProcessBuilder will handle quoting of paths (for blanks) as needed for various OS.
            List<String> commandElements = new ArrayList<String>();
            commandElements.add(getContainerCommandPrefix().trim());
            commandElements.add("build");

            if (pullParentImage) {
                commandElements.add("--pull");
            }
            commandElements.add("-f");
            commandElements.add(tempContainerfile.getAbsolutePath());
            commandElements.add("-t");
            commandElements.add(imageName);
            commandElements.add(buildContext.getAbsolutePath());

            String[] newCommand = commandElements.toArray(new String[commandElements.size()]);
            info("Container command: "+String.join(" ", newCommand));
    
            if (hasFeaturesSh.get()) {
                info("The RUN features.sh command is detected in the Containerfile and extra time may be necessary when installing features.");
            }
            long startTime = System.currentTimeMillis();

            execContainerCmdAndLog(getRunProcess(newCommand), containerBuildTimeout, false);

            checkContainerBuildTime(startTime, buildContext);
            info("Completed building container image.");
        } catch (IllegalThreadStateException  e) {
            // the timeout was too short and the container command has not yet completed.
            debug("IllegalThreadStateException, message="+e.getMessage());
            throw new PluginExecutionException("The container build command did not complete within the timeout period: " + containerBuildTimeout + " seconds. " +
                "Use the containerBuildTimeout option to specify a longer period or " +
                "add files not needed in the container to the .containerignore file", e);
        } catch (IOException e) {
            error("Input or output error building container image: " + e.getMessage());
            throw new RuntimeException(e);
        } catch (InterruptedException e) {
            debug("Thread InterruptedException while building the container image: " + e.getMessage());
            throw new PluginExecutionException("Could not build container image using Containerfile: " +
                userContainerfile.getAbsolutePath() + ". Address the following container build error and then start dev mode again: " + e.getMessage(), e);
        } catch (RuntimeException r) {
            debug("RuntimeException building container image: " + r.getMessage());
            throw new PluginExecutionException("Could not build container image using Containerfile: " + 
                userContainerfile.getAbsolutePath() + ". Address the following container build error and then start dev mode again: " + r.getMessage(), r);
        }
    }

    // Suggest a performance improvement if container build takes too long.
    private static final long DOCKER_BUILD_SOFT_TIMEOUT = 30000; // millis
    private void checkContainerBuildTime(long startTime, File containerBuildContext) {
        if (System.currentTimeMillis() - startTime < DOCKER_BUILD_SOFT_TIMEOUT) {
            return;
        }
        debug("checkContainerBuildTime, containerBuildContext=" + containerBuildContext.getAbsolutePath());
        //TODO: ADD DOCKERIGNORE VS CONTAINERIGNORE CHECK
        File dockerIgnore = new File(containerBuildContext, ".dockerignore");
        if (!dockerIgnore.exists()) { // provide some advice
            String buildContextPath;
            try {
                buildContextPath = containerBuildContext.getCanonicalPath();
            } catch (IOException e) {
                buildContextPath = containerBuildContext.getAbsolutePath();
            }
            warn("The container build command took longer than " + DOCKER_BUILD_SOFT_TIMEOUT / 1000 + " seconds. " +
                "You may increase performance by adding unneeded files and directories " +
                "such as any Liberty runtime directories to a .dockerignore file in " +
                buildContextPath + ".");
        }
    }

    private void startContainer() throws PluginExecutionException {
        try {
            // Allow the server to write to the log files. If we don't create it here docker daemon will create it as root.
            new File(serverDirectory, "logs").mkdirs();
            // Added two hidden folders since started getting permissions errors on the dropins folder.
            new File(buildDirectory, DEVC_HIDDEN_FOLDER + "/apps").mkdirs();
            new File(buildDirectory, DEVC_HIDDEN_FOLDER + "/dropins").mkdirs();

            info("Starting container...");
            String[] startContainerCommand = getContainerCommand();
            // Use String[] instead for command parameters. It will handle quoting of paths for various OS.
            containerRunProcess = getRunProcess(startContainerCommand);
            execContainerCmdAndLog(containerRunProcess, 0, true);
        } catch (IOException e) {
            error("Error starting container: " + e.getMessage());
            throw new RuntimeException(e);
        } catch (InterruptedException e) {
            error("Thread was interrupted while starting the container: " + e.getMessage());
        } catch (RuntimeException r) {
            try {
                // remove container in case of an error trying to run the container because the container run --rm will not rm the container
                String containerRmCmd = "container rm " + containerName;
                execContainerCmdWithPrefix(containerRmCmd, CONTAINER_TIMEOUT);
            } catch (Exception e) {
                // do not report the "container rm" error so that we can instead report the startContainer() error
                debug("Exception running container command rm:", e);
            }
            throw r;
        }
    }

    // Pass String[] containing command and all of its individual arguments. 
    // This ensures proper escaping/quoting for arguments that might contain spaces or special characters.
    private Process getRunProcess(String[] command) throws IOException {
        ProcessBuilder processBuilder = new ProcessBuilder();
        processBuilder.command(command);
        Map<String, String> env = processBuilder.environment();
        if (!OSUtil.isLinux()){
            if (!env.keySet().contains("DOCKER_BUILDKIT")) { // don't touch if already set
                env.put("DOCKER_BUILDKIT", "0"); // must set 0 on Windows VMs
                debug("Generating environment for docker build & run: DOCKER_BUILDKIT=0");
            }
        }
        if (!env.keySet().contains("DOCKER_SCAN_SUGGEST")) { // don't touch if already set
            env.put("DOCKER_SCAN_SUGGEST", "false"); // Disable stderr contents from showing as error message
            debug("Generating environment for docker build & run: DOCKER_SCAN_SUGGEST=false");
        }
        return processBuilder.start();
    }

    private void execContainerCmdAndLog(final Process startingProcess, int timeout, boolean isStartCommand) throws InterruptedException {
        Thread logCopyInputThread = new Thread(new Runnable() {
            @Override
            public void run() {
                copyStreamToBuildLog(startingProcess.getInputStream(), true);
            }
        });
        logCopyInputThread.start();

        final StringBuilder firstErrorLine = new StringBuilder();
        Thread logCopyErrorThread = new Thread(new Runnable() {
            @Override
            public void run() {
                if (OSUtil.isLinux() && !isDocker) {
                    firstErrorLine.append(copyStreamToBuildLog(startingProcess.getErrorStream(), true));
                } else {
                    firstErrorLine.append(copyStreamToBuildLog(startingProcess.getErrorStream(), false));
                }
            }
        });
        logCopyErrorThread.start();

        if (isStartCommand) {
            writeDevcMetadata(true);
        }

        if (timeout == 0) {
            startingProcess.waitFor();
        } else {
            startingProcess.waitFor(timeout, TimeUnit.SECONDS);
        }
        if (startingProcess.exitValue() != 0 && !devStop.get()) { // if there was an error and the user didn't choose to stop dev mode
            // return code 143 corresponds to 'docker stop xxx' so assume user requested to stop
            if (startingProcess.exitValue() == 143) {
                setDevStop(true); // indicate intentional shutdown
                externalContainerShutdown.set(true); // container shut down by external command
            }
            debug("Unexpected exit running container command, return value=" + startingProcess.exitValue());
            // show first message from standard err
            String errorMessage = new String(firstErrorLine).trim() + " RC=" + startingProcess.exitValue();
            if (isStartCommand) {
                writeDevcMetadata(false);
            }
            throw new RuntimeException(errorMessage);
        }
    }

    private String runCmd(String cmd) throws IOException, InterruptedException {
        String result = null;
        Process p = Runtime.getRuntime().exec(cmd);
        p.waitFor(5, TimeUnit.SECONDS);
        if (p.exitValue() != 0) {
            error("Error running command:" + cmd + ", return value=" + p.exitValue());
        } else {
            result = readStdOut(p);
        }
        return result;
    }

    /**
     * Copies the process output to the Maven/Gradle logs
     * 
     * @param stream The stream to copy
     * @param info If true, log as info. Else log as error.
     * @throws RuntimeException if there was an error reading the process output
     * @return The first line from the stream
     */
    private String copyStreamToBuildLog(InputStream stream, boolean info) {
        String firstLine = null;
        BufferedReader inputReader = new BufferedReader(new InputStreamReader(stream));
        try {
            for (String line; (line = inputReader.readLine()) != null;) {
                if (firstLine == null) {
                    firstLine = line;
                }
                if (info) {
                    info(line);
                } else {
                    error(line);

                    // Look for JVM version error in the line
                    alertOnServerError(line, "JVMCFRE003",
                            "Java classes were compiled with a higher version of Java than the JVM in the container. To resolve this issue, set the source and target Java versions in your Gradle build to correspond to the Java version used in your Containerfile/Dockerfile or its parent image, then restart dev mode.",
                            // Maven project should be cleaned before restarting dev mode, otherwise compile does not realize Java version settings have changed
                            "Java classes were compiled with a higher version of Java than the JVM in the container. To resolve this issue, set the source and target Java versions in your Maven build to correspond to the Java version used in your Containerfile/Dockerfile or its parent image, then clean the project output and restart dev mode.",
                            false);

                    // Look for features not available message during server startup if features.sh was not defined in Containerfile
                    if (!serverFullyStarted.get() && !hasFeaturesSh.get() && !shownFeaturesShWarning.get()) {
                        String errMsg = "Feature definitions were not found in the container. To install features to the container, specify 'RUN features.sh' in your Containerfile/Dockerfile. For an example of how to configure a Containerfile/Dockerfile, see https://github.com/OpenLiberty/ci.docker";
                        shownFeaturesShWarning.set(alertOnServerError(line, "CWWKF0001E", errMsg, errMsg, true));
                    }
                }
            }
        } catch (IOException e) {
            throw new RuntimeException("Error reading container output: " + e.getMessage());
        } finally {
            try {
                inputReader.close();
            } catch (IOException e) {
                // nothing to do
            }
        }
        return firstLine;
    }

    private boolean alertOnServerError(String line, String errorCode, String gradleMessage, String mavenMessage, boolean warning) {
        if (container && line.contains(errorCode)) {
            if (gradle) {
                // Gradle doesn't show errors in an obvious way, so use some formatting to make it stand out more
                if (warning) {
                    warn("***** [ WARNING ] ***** " + gradleMessage);
                } else {
                    error("***** [ ERROR ] ***** " + gradleMessage);
                }
            } else {
                if (warning) {
                    warn(mavenMessage);
                } else {
                    error(mavenMessage);
                }
            }
            return true;
        }
        return false;
    }

    // Add the passed runOpts to the commandElements as separate options with the option and value specified individually.
    //
    // Note: If an option's value in runOpts contains spaces and needs quoting/escaping, the getCommandTokens method won't handle it correctly.
    // Two ways to approach this:
    //  1) parse the opts into separate options ('-xyz abc' or '--xyz abc') and then break the option apart from its value at the first space
    //  2) use the returned String[] from getCommandTokens and detect incorrectly broken up option values and splice them back together 
    //
    // Going with option 2.
    protected void addContainerRunOpts(String runOpts, List<String> commandElements) {
        String[] opts = getCommandTokens(runOpts);
        int index = 0;
        while (index < opts.length) {
            if ((index == 0) || opts[index].startsWith("-")) {
                commandElements.add(opts[index]);
            } else {
                String lastListElement = commandElements.get(commandElements.size()-1);
                if (!lastListElement.startsWith("-")) {
                    // add this to previous element
                    commandElements.set(commandElements.size()-1, lastListElement+" "+opts[index]);
                } else {
                    commandElements.add(opts[index]);
                }
            }
            index++;
        }
    }

    protected String[] getCommandTokens(String command) {
        StringTokenizer stringTokenizer = new StringTokenizer(command);
        String[] commandTokens = new String[stringTokenizer.countTokens()];
        for (int i = 0; stringTokenizer.hasMoreTokens(); i++) {
            commandTokens[i] = stringTokenizer.nextToken();
        }
        return commandTokens;
    }

    private void stopContainer()  throws PluginExecutionException {
        try {
            serverFullyStarted.set(false);

            // see if container run command (container) is still running before trying to stop it.
            if (containerRunProcess != null && containerRunProcess.isAlive()) {
                info("Stopping container...");
                String containerStopCmd = "stop " + containerName;
                debug("Stopping container " + containerName);
                execContainerCmdWithPrefix(containerStopCmd, CONTAINER_TIMEOUT + 20); // allow extra time for server shutdown
                writeDevcMetadata(false);
            }
        } catch (RuntimeException r) {
            error("Error stopping container: " + r.getMessage());
            throw r;
        } finally {
            containerRunProcess = null;
        }
    }

    /**
     * Get the root directory for mounting loose app in container.  This is the longest common directory between the projectDirectory and multiModuleProjectDirectory.
     * 
     * @param projectDirectory The current project directory. Must not be null.
     * @param multiModuleProjectDirectory The multi module project directory. Can be null.
     * @return The longest common directory, or projectDirectory if multiModuleProjectDirectory is null
     */
    public static File getLooseAppProjectRoot(File projectDirectory, File multiModuleProjectDirectory) {
        if (multiModuleProjectDirectory == null) {
            return projectDirectory;
        }
        try {
            return getLongestCommonDir(projectDirectory.getCanonicalFile(), multiModuleProjectDirectory.getCanonicalFile());
        } catch (IOException e) {
            return getLongestCommonDir(projectDirectory.getAbsoluteFile(), multiModuleProjectDirectory.getAbsoluteFile());
        }
    }

    protected static File getLongestCommonDir(File dir1, File dir2) {
        // based on https://stackoverflow.com/a/54596165
        Path relativePath = dir1.toPath().relativize(dir2.toPath()).normalize();
        while (relativePath != null && !relativePath.endsWith("..")) {
            relativePath = relativePath.getParent();
        }
        if (relativePath == null || relativePath.toString().isEmpty()) {
            return dir1;
        } else {
            return dir1.toPath().resolve(relativePath).normalize().toFile();
        }
    }

    /**
     * Build a container run command with all the ports and directories required to run Open Liberty 
     * inside a container. Also included is the image name and the server run command to override
     * the CMD attribute of the Open Liberty docker image. 
     * @return the command string to use to start the container
     */
    private String[] getContainerCommand() throws IOException, PluginExecutionException {
        // Use List<String> for building the command. This will be converted to a String[] when passed to ProcessBuilder. 
        // The ProcessBuilder will handle quoting of paths (for blanks) as needed for various OS.
        List<String> commandElements = new ArrayList<String>();
        commandElements.add(getContainerCommandPrefix().trim());
        commandElements.add("run");
        commandElements.add("--rm");

        if (!skipDefaultPorts) {
            int httpPortToUse, httpsPortToUse;
            try {
                httpPortToUse = findAvailablePort(LIBERTY_DEFAULT_HTTP_PORT, false);
                httpsPortToUse = findAvailablePort(LIBERTY_DEFAULT_HTTPS_PORT, false);
            } catch (IOException x) {
                error("An error occurred while trying to find an available network port. Using default port numbers.", x);
                httpPortToUse = LIBERTY_DEFAULT_HTTP_PORT;
                httpsPortToUse = LIBERTY_DEFAULT_HTTPS_PORT;
            }
            commandElements.add("-p");
            commandElements.add(httpPortToUse+":"+LIBERTY_DEFAULT_HTTP_PORT);

            commandElements.add("-p");
            commandElements.add(httpsPortToUse+":"+LIBERTY_DEFAULT_HTTPS_PORT);
        }
        
        if (libertyDebug) {
            // map debug port
            int containerDebugPort, hostDebugPort;
            try {
                if (alternativeDebugPort == -1) {
                    // it is possible another JVM has grabbed our port since dev mode last checked
                    hostDebugPort = findAvailablePort(libertyDebugPort, true);
                    containerDebugPort = libertyDebugPort;
                } else {
                    // dev mode has already selected an ephemeral port
                    containerDebugPort = hostDebugPort = alternativeDebugPort;
                }
            } catch (IOException x) {
                containerDebugPort = hostDebugPort = libertyDebugPort;
            }
            commandElements.add("-p");
            commandElements.add(hostDebugPort+":"+containerDebugPort);
            // set environment variables in the container to ensure debug mode does not suspend the server, and to enable a custom debug port to be used
            // and to allow remote debugging into the container
            commandElements.add("-e");
            commandElements.add("WLP_DEBUG_SUSPEND=n");
            commandElements.add("-e");
            commandElements.add("WLP_DEBUG_ADDRESS=" + containerDebugPort);
            commandElements.add("-e");
            commandElements.add("WLP_DEBUG_REMOTE=y");
        }

        // mount potential directories containing .war.xml from devc specific folder - override /config/apps and /config/dropins
        File tempApps = new File(buildDirectory, DEVC_HIDDEN_FOLDER + "/apps");
        File tempDropins = new File(buildDirectory, DEVC_HIDDEN_FOLDER + "/dropins");
        commandElements.add("-v");
        commandElements.add(tempApps + ":/config/apps");

        commandElements.add("-v");
        commandElements.add(tempDropins + ":/config/dropins");

        // mount the loose application resources in the container using the appropriate project root
        File looseApplicationProjectRoot = getLooseAppProjectRoot(projectDirectory, multiModuleProjectDirectory);
        commandElements.add("-v");
        commandElements.add(looseApplicationProjectRoot.getAbsolutePath() + ":" + DEVMODE_DIR_NAME);

        // mount the server logs directory over the /logs used by the open liberty container as defined by the LOG_DIR env. var.
        File logsDir = new File(serverDirectory.getAbsolutePath(), "logs");
        commandElements.add("-v");
        commandElements.add(logsDir + ":/logs");

        // mount the Maven .m2 cache directory for featureUtility to use. For now, featureUtility does not support Gradle cache.
        commandElements.add("-v");
        commandElements.add(mavenCacheLocation + ":/devmode-maven-cache");

        // mount all files from COPY commands in the Containerfile to allow for hot deployment
        addCopiedFiles(commandElements);

        // Add a --user option when running Linux
        addUserId(commandElements);

        // Do not generate a name if the user has specified a name
        String name = getContainerOption("--name");
        if (name == null || name.isEmpty()) {
            if (name != null && name.isEmpty()) {
                error("The container option --name is specified with an unsupported value: empty string.");
                // now generate a name so that the container errors make some sense to the user.
            }
            containerName = generateNewContainerName();
            commandElements.add("--name");
            commandElements.add(containerName);
        } else {
            containerName = name;
        }
        debug("containerName: " + containerName + ".");

        // Allow the user to add their own options to this command via a system property.
        if (containerRunOpts != null) {
            addContainerRunOpts(containerRunOpts, commandElements);
        }

        // Options must precede this in any order. Image name and command code follows.
        commandElements.add(imageName);

        // Command to start the server
        commandElements.add("server");
        if (libertyDebug) {
            commandElements.add("debug");
        } else {
            commandElements.add("run");
        }
        commandElements.add("defaultServer");

        // All the Liberty variable definitions must appear after the -- option.
        // Important: other Liberty options must appear before --
        commandElements.add("--");
        commandElements.add("--"+DEVMODE_PROJECT_ROOT+"="+DEVMODE_DIR_NAME);

        //return command.toString();
        String[] newCommand = commandElements.toArray(new String[commandElements.size()]);
        info("Container command: "+String.join(" ", newCommand));
        return newCommand;
    }

    /**
     * Obtain a given container run option from the containerRunOpts parameter
     * @param optionName the name of the option to extract from the containerRunOpts
     * @return a string representation of the value of the option or null
     *
     * The option of interest must not use a quoted string.
     */
    private String getContainerOption(String optionName) {
        if (containerRunOpts == null || containerRunOpts.isEmpty()) {
            return null;
        }
        String[] options = containerRunOpts.split("\\s+"); // split on whitespace
        for (int i = 0; i < options.length; i++) {
            if (options[i].equals(optionName)) { // --name ABC format
                return (i < options.length - 1) ? options[i+1] : null;
            } else if (options[i].startsWith(optionName + "=")) { // --name=ABC format
                return options[i].substring(optionName.length()+1); // could be empty string
            }
        }
        return null;
    }

    private String generateNewContainerName() throws PluginExecutionException {
        String containerContNamesCmd = "ps -a --format \"{{.Names}}\"";
        debug("container names list command: " + containerContNamesCmd);
        String result = execContainerCmdWithPrefix(containerContNamesCmd, CONTAINER_TIMEOUT);
        if (result == null) {
            return DEVMODE_CONTAINER_BASE_NAME;
        }
        String[] containerNames = result.split(" ");
        int highestNum = -1;
        for(int i = 0; i < containerNames.length; i++) {
            String name = removeSurroundingQuotes(containerNames[i]);
            int num = -1;
            if (name.equals(DEVMODE_CONTAINER_BASE_NAME)) {
                num = 0;
            } else if (name.startsWith(DEVMODE_CONTAINER_BASE_NAME + "-")) {
                String[] nameSegments = name.split("-");
                // if DEVMODE_CONTAINER_BASE_NAME changes, the logic below may need to change
                if (nameSegments.length == 3) {
                    String lastSegment = nameSegments[nameSegments.length - 1];
                    try {
                        num = Integer.parseInt(lastSegment);
                    } catch (NumberFormatException e) {
                        debug("Last segment of container name is not a number.");
                    }
                }
            }
            if (num > highestNum) {
                highestNum = num;
            }
        }
        
        return DEVMODE_CONTAINER_BASE_NAME + ((highestNum != -1) ? "-" + ++highestNum : "");
    }

    /**
     * Retrieves all the networks a container is connected to
     * @param contName name of the container to check for networks
     * @return a String array containing the names of the networks the specified container is connected to
     */
    private String[] getContainerNetworks(String contName) throws PluginExecutionException {
        String containerNetworkCmd = "inspect -f '{{.NetworkSettings.Networks}}' " + contName;
        String cmdResult = execContainerCmdWithPrefix(containerNetworkCmd, CONTAINER_TIMEOUT, false);
        if (cmdResult == null || cmdResult.contains(" RC=")) { // RC is added in execContainerCmd if there is an error
            warn("Unable to retrieve container networks.");
            return null;
        }
        else {
            return parseNetworks(removeSurroundingQuotes(cmdResult.trim()));
        }
    }

     /**
     * Parses container network names from a "docker inspect" command result on a container.
     * @param containerResult the result from the command "docker inspect -f '{{.NetworkSettings.Networks}}' containerName", it must not contain surrounding quotes or leading/trailing whitespace
     * @return a String array containing the names of the networks contained in the containerResult parameter
     */
    protected static String[] parseNetworks(String containerResult) {
        // Example containerResult value: map[bridge:0xc000622000 myNet:0xc0006220c0 otherNet:0xc000622180]
        if (!containerResult.matches("map\\[(.*?)\\]")) {
            return null;
        }
        String networkMap = containerResult.substring(containerResult.indexOf("[")+1, containerResult.indexOf("]"));
        String[] networkHex = networkMap.split(" ");
        String[] networks = new String[networkHex.length];
        for (int i=0; i < networkHex.length; i++) {
            networks[i] = networkHex[i].split(":")[0];
        }
        return networks;
    }

    private String getContainerIPAddress(String contName, String network) throws PluginExecutionException {
        String containerIPAddressCmd = "inspect -f '{{.NetworkSettings.Networks." + network + ".IPAddress}}' " + contName;
        String result = execContainerCmdWithPrefix(containerIPAddressCmd, CONTAINER_TIMEOUT, false);
        if (result == null || result.contains(" RC=")) { // RC is added in execContainerCmd if there is an error
            warn("Unable to retrieve container IP address for network '" + network + "'.");
            return "<no value>"; // this is what Docker/Podman displays when an IP address it not found for a network
        }
        return removeSurroundingQuotes(result.trim());
    }

    protected static String removeSurroundingQuotes(String str) {
        if (str != null && str.length() >= 2 && ((str.startsWith("\"") && str.endsWith("\"")) || (str.startsWith("\'") && str.endsWith("\'")))) {
            return str.substring(1, str.length()-1);
        }
        return str;
    }

    // Read all the files from the array list.
    private void addCopiedFiles(List<String> commandElements) {
        for (int i=0; i < srcMount.size(); i++) {
            if (new File(srcMount.get(i)).exists()) { // only Files are in this list
                commandElements.add("-v");
                commandElements.add(srcMount.get(i)+":"+destMount.get(i));
            } else {
                error("A file referenced by the Containerfile is not found: " + srcMount.get(i) +
                    ". Update the Containerfile or ensure the file is in the correct location.");
            }
        }
    }

    private void addUserId(List<String> commandElements) {
        if (OSUtil.isLinux() || !isDocker) {
            try {
                String id = runCmd("id -u");
                if (id != null) {
                    commandElements.add("--user");
                    commandElements.add(id.trim());
                }
            } catch (IOException e) {
                // can't get user id. runCmd has printed an error message.
            } catch (InterruptedException e) {
                // can't get user id. runCmd has printed an error message.
            }
        }
    }

    /**
     * Create a Liberty server using the default name "defaultServer" or the server
     * specified by the common parameter "serverName" in pom.xml.
     * 
     * @throws PluginExecutionException
     */
    public abstract void libertyCreate() throws PluginExecutionException;

    /**
     * Deploy the default apps on the Liberty server or deploy the apps
     * specified in the configuration of the deploy goal in pom.xml.
     * 
     * @throws PluginExecutionException
     */
    public abstract void libertyDeploy() throws PluginExecutionException;

    /**
     * Generate features for the application
     * 
     * @param classes class file paths features should be generated for (can be null if no modified classes)
     * @param optimize if true, generate optimized feature list
     * @param generateToSrc if true, generate feature list into file in src/main/liberty
     * @param useTmpDirOut if true, generate feature file in a hidden directory named in FeatureGeneratorUtil
     * @param useTmpDirIn if true, the hidden directory named in FeatureGeneratorUtil will be used as the
     *                    context or input values to generate features
     * @return true if feature generation was successful
     */
    public abstract boolean libertyGenerateFeatures(Collection<String> classes, boolean optimize, boolean generateToSrc, boolean useTmpDirOut, boolean useTmpDirIn);

    /**
     * Install features in regular dev mode. This method should not be used in container mode.
     * @throws PluginExecutionException
     */
    public abstract void libertyInstallFeature() throws PluginExecutionException;

    public void restartServer() throws PluginExecutionException {
        restartServer(false);
    }

    /**
     * Stop the server, set up Liberty and restart it.
     * @param buildContainer  Force a container build when in container mode. Ignored otherwise.
     */
    public void restartServer(boolean buildContainer) throws PluginExecutionException {
        info("Restarting server...");
        setDevStop(true);
        if (container) {
            stopContainer(); // this command is synchronous
        } else {
            stopServer();
            if (serverThread != null) {
                final long threadShutdownTimeoutSeconds = 30;
                try {
                    serverThread.join(threadShutdownTimeoutSeconds * 1000);
                    if (serverThread.isAlive()) {
                        throw new PluginExecutionException("Could not stop the server after " + threadShutdownTimeoutSeconds
                                + " seconds.  Ensure that the server has been stopped, then start dev mode again.");
                    }
                } catch (InterruptedException e) {
                    if (serverThread.isAlive()) {
                        throw new PluginExecutionException(
                                "Could not stop the server.  Ensure that the server has been stopped, then start dev mode again.",
                                e);
                    } else {
                        // the thread was interrupted, but the server thread is already stopped
                        debug(e);
                    }
                }
            }
        }
        // suppress install feature warning
        System.setProperty(SKIP_BETA_INSTALL_WARNING, Boolean.TRUE.toString());
        libertyCreate();
        // Skip installing features on container during restart, since the Containerfile/Dockerfile
        // should have 'RUN features.sh'
        // Also skip install feature on restart if config parameter specified.
        if (!container && !skipInstallFeature) {
            libertyInstallFeature();
        } else if (skipInstallFeature) {
            info("Skipping installation of features due to skipInstallFeature configuration.");
        }
        libertyDeploy();
        startServer(buildContainer, false);
        setDevStop(false);
        info("The server has been restarted.");
        printDevModeMessages(inputUnavailable.get(), true);
    }

    private void parseHostNameAndPorts(final ServerTask serverTask, File messagesLogFile)
            throws PluginExecutionException {
        String webAppMessage = serverTask.findStringInFile(WEB_APP_AVAILABLE_MESSAGE_PREFIX, messagesLogFile);
        debug("Web app available message: " + webAppMessage);
        if (webAppMessage != null) {
            int portPrefixIndex = parseHostName(webAppMessage);
            parseHttpPort(webAppMessage, portPrefixIndex);
        }
        List<String> listeningOnPortMessages = serverTask.findStringsInFile(LISTENING_ON_PORT_MESSAGE_PREFIX,
                messagesLogFile);
        if (listeningOnPortMessages != null) {
            parseHttpsPort(listeningOnPortMessages);
        }
    }

    protected int parseHostName(String webAppMessage) throws PluginExecutionException {
        // Get unescaped HTTP protocol and hostname index
        int protocolIndex = webAppMessage.indexOf(HTTP_PREFIX);
        int hostNameIndex = protocolIndex + HTTP_PREFIX.length();

        // if not found, get escaped HTTP protocol and hostname index
        if (protocolIndex < 0) {
            protocolIndex = webAppMessage.indexOf(HTTP_PREFIX_ESCAPED);
            hostNameIndex = protocolIndex + HTTP_PREFIX_ESCAPED.length();
        }

        // if not found, get unescaped HTTPS protocol and hostname index
        if (protocolIndex < 0) {
            protocolIndex = webAppMessage.indexOf(HTTPS_PREFIX);
            hostNameIndex = protocolIndex + HTTPS_PREFIX.length();
        }

        // if not found, get escaped HTTPS protocol and hostname index
        if (protocolIndex < 0) {
            protocolIndex = webAppMessage.indexOf(HTTPS_PREFIX_ESCAPED);
            hostNameIndex = protocolIndex + HTTPS_PREFIX_ESCAPED.length();
        }

        if (protocolIndex < 0) {
            throw new PluginExecutionException(
                    "Could not parse the host name from the log message: " + webAppMessage);
        }

        int portPrefixIndex = webAppMessage.indexOf(":", hostNameIndex);
        if (portPrefixIndex < 0) {
            throw new PluginExecutionException(
                    "Could not parse the port number from the log message: " + webAppMessage);
        }
        if (container) {
            hostName = "localhost";
        } else {
            hostName = webAppMessage.substring(hostNameIndex, portPrefixIndex);
        }
        debug("Parsed host name: " + hostName);
        return portPrefixIndex;
    }

    protected void parseHttpPort(String webAppMessage, int portPrefixIndex) throws PluginExecutionException {
        if (!webAppMessage.contains(HTTP_PREFIX) && !webAppMessage.contains(HTTP_PREFIX_ESCAPED)) {
            return;
        }
        int portIndex = portPrefixIndex + 1;
        int portEndIndex = webAppMessage.indexOf("/", portIndex);
        if (portEndIndex < 0) {
            // if no ending slash, the port ends at the end of the message
            portEndIndex = webAppMessage.length();
        }
        String parsedHttpPort = webAppMessage.substring(portIndex, portEndIndex).replace("/","").replace("\\","");
        debug("Parsed http port: " + parsedHttpPort);
        if (container) {
            httpPort = findLocalPort(parsedHttpPort);
            containerHttpPort = parsedHttpPort;
        }
        else {
            httpPort = parsedHttpPort;
        }
    }

    /**
     * Extracts the "message" field value from a JSON string.
     *
     * @param jsonString The JSON string to parse
     * @return The value of the "message" field if present, null otherwise
     */
    private static String getMessageKeyFromJson(String jsonString) {
        if (jsonString == null || jsonString.isEmpty()) {
            return null;
        }
        try {
            JSONObject jsonObject = new JSONObject(jsonString);
            return jsonObject.optString("message", null);
        } catch (JSONException e) {
            // If parsing fails, it's not a valid JSON object
            return null;
        }
    }

    protected void parseHttpsPort(List<String> messages) throws PluginExecutionException {
        for (String message : messages) {
            debug("Looking for https port in message: " + message);
            String httpsMessageContents;
            // check message format. if message is a json, only extract "message" node
            String messageFromJson = getMessageKeyFromJson(message);
            if (messageFromJson != null) {
                debug("Message is in json format");
                httpsMessageContents = messageFromJson.split(LISTENING_ON_PORT_MESSAGE_PREFIX)[1];
            } else {
                httpsMessageContents = message.split(LISTENING_ON_PORT_MESSAGE_PREFIX)[1];
            }
            // Look for endpoint with name containing "-ssl"
            if (httpsMessageContents.contains("-ssl")) {
                String[] messageTokens = httpsMessageContents.split(" ");
                String parsedHttpsPort = getPortFromMessageTokens(messageTokens);
                if (parsedHttpsPort != null) {
                    debug("Parsed https port: " + parsedHttpsPort);
                    if (container) {
                        httpsPort = findLocalPort(parsedHttpsPort);
                        containerHttpsPort = parsedHttpsPort;
                    } else {
                        httpsPort = parsedHttpsPort;
                    }
                    return;
                } else {
                    throw new PluginExecutionException(
                            "Could not parse the https port number from the log message: " + message);
                }
            }
        }
        debug("Could not find https port. The server might not be configured for https.");
    }

    private String getPortFromMessageTokens(String[] messageTokens) throws PluginExecutionException {
        // For each space-separated token, keep only the numeric parts.
        // The port is the last numeric token which is a number <= 65535.
        for (int i = messageTokens.length - 1; i >= 0; i--) {
            String numericToken = messageTokens[i].replaceAll("[^\\d]", "");
            if (numericToken.length() > 0) {
                try {
                    int parsedPort = Integer.parseInt(numericToken);
                    if (parsedPort <= 65535) {
                        return numericToken;
                    }
                } catch (NumberFormatException e) {
                    // If the token is not parseable for some reason, then it's probably not a port
                    // number
                    debug("Could not parse integer from numeric token " + numericToken + " from message token "
                            + messageTokens[i], e);
                }
            }
        }
        return null;
    }

    private String findLocalPort(String internalContainerPort) throws PluginExecutionException {
        String containerPortCmd = "port " + containerName + " " + internalContainerPort;
        String cmdResult = execContainerCmdWithPrefix(containerPortCmd, CONTAINER_TIMEOUT, false);
        if (cmdResult == null) {
            warn("Unable to retrieve locally mapped port.");
            return null;
        }
        if (cmdResult.contains(" RC=")) { // This piece of the string is added in execContainerCmd if there is an error
            warn("Unable to retrieve locally mapped port. Container result: \"" + cmdResult.split(" RC=")[0] + "\". Ensure the container ports are mapped correctly.");
            return null;
        }
        String[] cmdResultSplit = cmdResult.split(":");
        String localPort = cmdResultSplit[cmdResultSplit.length - 1].trim();
        debug("Local port: " + localPort);
        return localPort;
    }

    public void cleanUpServerEnv() {
        // clean up server.env file
        File serverEnvFile;
        File serverEnvBackup;
        try {
            serverEnvBackup = new File(serverDirectory.getCanonicalPath() + "/server.env.bak");
            serverEnvFile = new File(serverDirectory.getCanonicalPath() + "/server.env");
            if (serverEnvBackup.exists()) {
                // Restore original server.env file
                try {
                    Files.copy(serverEnvBackup.toPath(), serverEnvFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
                } catch (IOException e) {
                    error("Could not restore server.env: " + e.getMessage());
                }
                serverEnvBackup.delete();
            } else {
                // Delete server.env file
                serverEnvFile.delete();
            }
        } catch (IOException e) {
            error("Could not retrieve server.env: " + e.getMessage());
        }
    }

    public void cleanUpTempConfig(Path myTempConfigPath) {
        if (myTempConfigPath != null) {
            File tempConfig = myTempConfigPath.toFile();
            if (tempConfig.exists()) {
                try {
                    FileUtils.deleteDirectory(tempConfig);
                    debug("Successfully deleted liberty:dev temporary configuration folder: " + myTempConfigPath);
                } catch (IOException e) {
                    warn("Could not delete liberty:dev temporary configuration folder: " + e.getMessage());
                }
            }
        }
    }

    public void cleanUpTempContainerfile() {
        if (!keepTempContainerfile && tempContainerfilePath != null) {
            File tempContainerfile = tempContainerfilePath.toFile();
            if (tempContainerfile.exists()) {
                try {
                    Files.delete(tempContainerfilePath);
                    debug("Successfully deleted dev mode temporary Containernerfile");
                } catch (IOException e) {
                    warn("Could not delete dev mode temporary Containerfile: " + e.getMessage());
                }
            }
        }
    }

    /**
     * Whether dev mode intentionally caused the server to stop.
     * 
     * @param devStop If true, stopping the server will not cause dev mode to print
     *                an error message.
     */
    public void setDevStop(boolean devStop) {
        this.devStop.set(devStop);
    }

    public void addShutdownHook(final ThreadPoolExecutor executor) {
        // shutdown hook to stop server when dev mode is terminated
        Runtime.getRuntime().addShutdownHook(new Thread() {
            @Override
            public void run() {
                runShutdownHook(executor);
            }
        });
    }

    private void runShutdownHook(final ThreadPoolExecutor executor) {
        try {
            if (!calledShutdownHook.getAndSet(true)) {

                if (trackingMode == FileTrackMode.POLLING || trackingMode == FileTrackMode.NOT_SET) {
                    disablePolling();
                }

                setDevStop(true);
                cleanUpTempConfig(this.tempConfigPath);
                cleanUpServerEnv();

                if (hotkeyReader != null) {
                    hotkeyReader.shutdown();
                }

                // shutdown tests
                executor.shutdown();


                // stopping server
                if (container) {
                    cleanUpTempContainerfile();
                    stopContainer();
                } else {
                    stopServer();
                }
            }
        } catch (PluginExecutionException pe) {
            error("Exception during container shutdown. Container may have shutdown incorrectly. " + pe.getMessage());
        }
    }

    private void disablePolling() {
        synchronized (newFileObservers) {
            consolidateFileObservers();
            for (FileAlterationObserver observer : fileObservers) {
                try {
                    observer.destroy();
                } catch (Exception e) {
                    debug("Could not destroy file observer", e);
                }
            }
        }
    }

    /**
     * Gets a map of the environment variables to set for debug mode.
     * 
     * @return a Map of debug environment variables with name as key
     */
    public Map<String, String> getDebugEnvironmentVariables() throws IOException {
        Map<String, String> map = new HashMap<String, String>();
        map.put("WLP_DEBUG_SUSPEND", "n");
        map.put("WLP_DEBUG_ADDRESS", String.valueOf(findAvailablePort(libertyDebugPort, true)));
        return map;
    }

    /**
     * Enable server debug variables in server.env, using the user specified debug
     * port if it's available, otherwise uses a random available port.
     * 
     * @throws IOException if there was an IO exception when reading or writing the
     *                     server.env
     */
    public void enableServerDebug() throws IOException {
        enableServerDebug(true);
    }

    private void enableServerDebug(boolean doFindPort) throws IOException {
        if (!libertyDebug) {
            return;
        }

        String serverEnvPath = serverDirectory.getCanonicalPath() + "/server.env";
        File serverEnvFile = new File(serverEnvPath);
        StringBuilder sb = new StringBuilder();
        File serverEnvBackup = new File(serverEnvPath + ".bak");
        if (serverEnvFile.exists()) {
            debug("server.env already exists");

            Files.copy(serverEnvFile.toPath(), serverEnvBackup.toPath(), StandardCopyOption.REPLACE_EXISTING);
            boolean deleted = serverEnvFile.delete();
            if (!deleted) {
                error("Could not move existing server.env file");
            }

            BufferedReader reader = new BufferedReader(new FileReader(serverEnvBackup));
            try {
                String line;
                while ((line = reader.readLine()) != null) {
                    sb.append(line);
                    sb.append("\n");
                }
            } finally {
                reader.close();
            }
        } else {
            // if server.env does not exist, clean up any backup file
            serverEnvBackup.delete();
        }

        debug("Creating server.env file: " + serverEnvFile.getCanonicalPath());
        sb.append("WLP_DEBUG_SUSPEND=n\n");
        sb.append("WLP_DEBUG_ADDRESS=");
        if (doFindPort) {
            sb.append(findAvailablePort(libertyDebugPort, true));
        } else {
            sb.append(alternativeDebugPort == -1 ? libertyDebugPort : alternativeDebugPort);
        }
        sb.append("\n");

        BufferedWriter writer = new BufferedWriter(new FileWriter(serverEnvFile));
        try {
            writer.write(sb.toString());
        } finally {
            writer.close();
        }

        if (serverEnvFile.exists()) {
            debug("Successfully created liberty:dev server.env file");
        }
    }

    /**
     * Finds an available port to use. There are two semantics. If looking for a port
     * for the server debug connection and the port is in use then return an
     * ephemeral port. If looking for a port for the server http connection then
     * try sequential port numbers.
     * 
     * In the case of the server debug connection, if the preferred port is not
     * available, return a random available port and cache the result which will
     * override the preferredPort if this method is called again.
     * 
     * @param  preferredPort The number of the port to start the search for an available port.
     * @param  isDebugPort Whether to choose an ephemeral port. True to choose an ephemeral port,
     *         false to search sequentially.
     * @return An available port.
     * @throws IOException if it could not find any available port, or there was an
     *                     error when opening a server socket regardless of port.
     */
    public int findAvailablePort(int preferredPort, boolean isDebugPort) throws IOException {
        int portToTry = preferredPort;
        if (isDebugPort && alternativeDebugPort != -1) {
            portToTry = alternativeDebugPort;
        }

        ServerSocket serverSocket = null;
        while (portToTry < 65535) {
            if (OSUtil.isWindows()) {
                try {
                    // try binding to the portToTry
                    serverSocket = new ServerSocket(portToTry);
                    return serverSocket.getLocalPort();
                } catch (IOException e) {
                    if (serverSocket != null) {
                        serverSocket.close();
                    }
                    if (isDebugPort) {
                        // if binding failed, try binding to a random port
                        serverSocket = new ServerSocket(0);
                        int availablePort = serverSocket.getLocalPort();
                        processAvailableDebugPort(preferredPort, portToTry, availablePort);
                        return availablePort;
                    } else {
                        debug("findAvailablePort found port is in use: " + portToTry);
                        ++portToTry;
                    }
                } finally {
                    if (serverSocket != null) {
                        serverSocket.close();
                    }
                }
            } else {
                try {
                    serverSocket = new ServerSocket();
                    serverSocket.setReuseAddress(false);
                    // try binding to the loopback address at the port to try
                    serverSocket.bind(new InetSocketAddress(InetAddress.getByName(null), portToTry), 1);
                    return serverSocket.getLocalPort();
                } catch (IOException e) {
                    if (serverSocket != null) {
                        if (isDebugPort) {
                            // if binding failed, try binding to a random port
                            serverSocket.bind(null, 1);
                            int availablePort = serverSocket.getLocalPort();
                            processAvailableDebugPort(preferredPort, portToTry, availablePort);
                            return availablePort;
                        } else {
                            debug("findAvailablePort found port is in use: " + portToTry);
                            ++portToTry;
                        }
                    } else {
                        throw new IOException("Could not create a server socket.", e);
                    }
                } finally {
                    if (serverSocket != null) {
                        serverSocket.close();
                    }
                }
            }
        }
        return preferredPort; // usual return is from the try or the catch
    }

    private void processAvailableDebugPort(int preferredPort, int portToTry, int availablePort) {
        if (portToTry == preferredPort) {
            warn("The debug port " + preferredPort + " is not available.  Using " + availablePort
                    + " as the debug port instead.");
        } else {
            debug("The previous debug port " + alternativeDebugPort + " is no longer available.  Using "
                    + availablePort + " as the debug port instead.");
        }
        alternativeDebugPort = availablePort;
    }

    private HotkeyReader hotkeyReader = null;

    /**
     * Run a hotkey reader thread. If the thread is already running, re-prints the
     * message about pressing enter to run tests.
     * 
     * @param executor the test thread executor
     */
    public void runHotkeyReaderThread(ThreadPoolExecutor executor) {
        if (inputUnavailable.get()) {
            return;
        }
        boolean startedNewHotkeyReader = false;
        if (hotkeyReader == null) {
            hotkeyReader = new HotkeyReader(executor);
            new Thread(hotkeyReader).start();
            debug("Started hotkey reader.");
            startedNewHotkeyReader = true;
        }

        synchronized (inputUnavailable) {
            try {
                if (startedNewHotkeyReader) {
                    // if new hotkey reader started, wait for it to try getting the input to see if
                    // it's available
                    inputUnavailable.wait(500);
                }
                printDevModeMessages(inputUnavailable.get(), firstStartup);
                firstStartup = false;
            } catch (InterruptedException e) {
                debug("Interrupted while waiting to determine whether input can be read", e);
            } catch (PluginExecutionException pe) {
                error(pe.getMessage());
            }
        }

    }

    /**
     * Print the dev mode startup and/or run tests messages.
     * 
     * @param inputUnavailable If true, indicates that the console is non-interactive so hotkey messages should not be printed.
     * @param startup If true, include attention barriers (asterisks lines) and overall dev mode startup messages such as list of hotkeys and ports.
     */
    private void printDevModeMessages(boolean inputUnavailable, boolean startup) throws PluginExecutionException {
        // the following will be printed only on startup or restart
        if (startup) {
            // print barrier header
            info(formatAttentionBarrier());

            info(formatAttentionTitle("Liberty is running in dev mode."));

            printFeatureGenerationStatus();
        }

        if (!inputUnavailable) {
            if (startup) {
                // the following will be printed only on startup or restart
                info(formatAttentionMessage("h - see the help menu for available actions, type 'h' and press Enter."));
                info(formatAttentionMessage("q - stop the server and quit dev mode, press Ctrl-C or type 'q' and press Enter."));
            } else {
                // the following will be printed every time after the tests run
                printTestsMessage(false);
            }
        } else {
            debug("Cannot read user input, setting hotTests to true.");
            String message = "Tests will run automatically when changes are detected.";
            info(startup ? formatAttentionMessage("Enter - " + message) : message);
            hotTests = true;
        }
        if (startup) {
            printPortInfo(true);
            // print barrier footer
            info(formatAttentionBarrier());
        }
    }

    private void printPortInfo(boolean pKeyPressed) throws PluginExecutionException {
        if (container) {
            boolean nonDefaultHttpPortUsed = !skipDefaultPorts && !String.valueOf(LIBERTY_DEFAULT_HTTP_PORT).equals(httpPort);
            boolean nonDefaultHttpsPortUsed = !skipDefaultPorts && !String.valueOf(LIBERTY_DEFAULT_HTTPS_PORT).equals(httpsPort);
            boolean nonDefaultDebugPortUsed = alternativeDebugPort != -1; // this is set when a random ephemeral port is selected
            if (containerHttpPort != null || containerHttpsPort != null || libertyDebug) {
                if (!pKeyPressed) {
                    //this line not needed when showing port info on keypress of "p"
                    info(formatAttentionMessage(""));
                }
                info(formatAttentionTitle("Liberty container port information:"));
            }
            if ((containerHttpPort != null && httpPort != null && nonDefaultHttpPortUsed)
                    || (containerHttpsPort != null && httpsPort != null && nonDefaultHttpsPortUsed)
                    || (libertyDebug && nonDefaultDebugPortUsed)) {
                warn(formatAttentionMessage("The Liberty container is using non-default host ports to avoid port conflict errors."));
            }
            if (containerHttpPort != null) {
                if (httpPort != null) {
                    if (!nonDefaultHttpPortUsed) {
                        info(formatAttentionMessage("Internal container HTTP port [ " + containerHttpPort + " ] is mapped to container host port [ " + httpPort + " ]"));
                    } else {
                        info(formatAttentionMessage("Internal container HTTP port [ " + containerHttpPort + " ] is mapped to container host port [ " + httpPort + " ] <"));
                    }
                } else {
                    info(formatAttentionMessage("Internal container HTTP port: [ " + containerHttpPort + " ]"));
                }
            }
            if (containerHttpsPort != null) {
                if (httpsPort != null) {
                    if (!nonDefaultHttpsPortUsed) {
                        info(formatAttentionMessage("Internal container HTTPS port [ " + containerHttpsPort + " ] is mapped to container host port [ " + httpsPort + " ]"));
                    } else {
                        info(formatAttentionMessage("Internal container HTTPS port [ " + containerHttpsPort + " ] is mapped to container host port [ " + httpsPort + " ] <"));
                    }
                } else {
                    info(formatAttentionMessage("Internal container HTTPS port: [ " + containerHttpsPort + " ]"));
                }
            }
            if (libertyDebug) {
                int debugPort = (alternativeDebugPort == -1 ? libertyDebugPort : alternativeDebugPort);
                if (!nonDefaultDebugPortUsed) {
                    info(formatAttentionMessage("Liberty debug port mapped to container host port: [ " + debugPort + " ]"));
                } else {
                    info(formatAttentionMessage("Liberty debug port mapped to container host port: [ " + debugPort + " ] <"));
                }
            }
            info(formatAttentionMessage(""));
            info(formatAttentionTitle("Container network information:"));
            info(formatAttentionMessage("Container name: [ " + containerName + " ]"));

            String[] networks = getContainerNetworks(containerName);
            if (networks != null) {
                for (String network : networks) {
                    info(formatAttentionMessage("IP address [ " + getContainerIPAddress(containerName, network) + " ] on container network [ " + network + " ]"));
                }
            }
        }
        else {
            if (httpPort != null || httpsPort != null || libertyDebug) {
                if (!pKeyPressed) {
                    //this line not needed when showing port info on keypress of "p"
                    info(formatAttentionMessage(""));
                }
                info(formatAttentionTitle("Liberty server port information:"));
            }
            if (httpPort != null) {
                info(formatAttentionMessage("Liberty server HTTP port: [ " + httpPort + " ]"));
            }
            if (httpsPort != null) {
                info(formatAttentionMessage("Liberty server HTTPS port: [ " + httpsPort + " ]"));
            }
            if (libertyDebug) {
                int debugPort = (alternativeDebugPort == -1 ? libertyDebugPort : alternativeDebugPort);
                info(formatAttentionMessage("Liberty debug port: [ " + debugPort + " ]"));
            }
        }
    }

    private void printTestsMessage(boolean formatForAttention) {
        // setting skipTests to true overrides the setting of hotTests, and prevents any tests from running in dev mode (automatically or on demand)
        if (skipTests) {
            return;
        }
        String actionKey = changeOnDemandTestsAction ? "t" : "Enter";
        String actionKeyInstruction = changeOnDemandTestsAction ? "type 't' and press Enter" : "press the Enter key";
        String message = hotTests ? "Tests will run automatically when changes are detected. You can also "+actionKeyInstruction+" to run tests on demand." : "run tests on demand, "+actionKeyInstruction+".";

        if (hotTests) {
            info(formatForAttention ? formatAttentionMessage(actionKey + " - " + message) : message);
        } else {
            info(formatForAttention ? formatAttentionMessage(actionKey + " - " + message) : "To " + message);
        }
    }

    private void printHelpMessages() {
        printFeatureGenerationStatus();
        printFeatureGenerationHotkeys();
        printTestsMessage(true);
        if (container) {
            info(formatAttentionMessage("r - rebuild the container image and restart the container, type 'r' and press Enter."));
        } else {
            info(formatAttentionMessage("r - restart the server, type 'r' and press Enter."));
        }
        info(formatAttentionMessage("h - see the help menu for available actions, type 'h' and press Enter."));
        info(formatAttentionMessage("p - see the port information, type 'p' and press Enter."));
        info(formatAttentionMessage("q - stop the server and quit dev mode, press Ctrl-C or type 'q' and press Enter."));
    }

    private void printFeatureGenerationStatus() {
        info(formatAttentionMessage("Automatic generation of features: " + getFormattedBooleanString(generateFeatures)));
        info(formatAttentionMessage("Generation of features to src directory: " + getFormattedBooleanString(generateToSrc)));
    }

    private void printFeatureGenerationHotkeys() {
        info(formatAttentionMessage(
                "g - toggle the automatic generation of features, type 'g' and press Enter."));
        info(formatAttentionMessage("s - toggle the option to generate features to the src directory, type 's' and press Enter."));
        if (generateFeatures) {
            // If generateFeatures is enabled, then also describe the optimize hotkey
            info(formatAttentionMessage("o - optimize the list of generated features, type 'o' and press Enter."));
        }
    }

    private String formatAttentionBarrier() {
        return "************************************************************************";
    }

    private String formatAttentionTitle(String message) {
        return "*    " + message;
    }

    private String formatAttentionMessage(String message) {
        return "*        " + message;
    }

    private String getFormattedBooleanString(boolean bool) {
        return "[ " + (bool ? "On" : "Off") + " ]";
    }

    private void toggleFeatureGeneration() {
        generateFeatures = !generateFeatures;
        logFeatureGenerationStatus();
        if (generateFeatures) {
            if (generateToSrc) {
                infoSrcDirModified();
            }
            // If hotkey is toggled to “true”, generate features right away.
            optimizeGenerateFeatures(!generateToSrc);
        }
    }

    private void logFeatureGenerationStatus() {
        info("Setting automatic generation of features to: " + getFormattedBooleanString(generateFeatures));
    }

    private void toggleGenerateToSrc() {
        generateToSrc = !generateToSrc;
        logGenerateToSrcStatus();
        initGenerationContext();
        // When you toggle generateToSrc delete the old file you no longer need
        if (generateToSrc) {
            deleteGenFeaturesFile(generateFeaturesTmpDir); // delete the old gen file in tmpdir
        } else {
            deleteGenFeaturesFile(configDirectory); // delete the old gen file in src/main/liberty/config
        }
        if (generateFeatures) {
            if (generateToSrc) {
                infoSrcDirModified();
            }
            // If hotkey is toggled, generate features right away.
            optimizeGenerateFeatures(!generateToSrc);
        }
    }

    private void deleteGenFeaturesFile(File dir) {
        // N.B. processConfigFileChange() will be called upon deletion of generated features file, it should be ignored
        File oldGenFeaturesFile = new File(dir, FeatureGeneratorUtil.GENERATED_FEATURES_FILE_PATH);
        if (oldGenFeaturesFile.exists()) {
            if (!oldGenFeaturesFile.delete()) {
                debug("Error trying to delete the generated features file:" + oldGenFeaturesFile.getAbsolutePath());
            }
        }
    }

    private void logGenerateToSrcStatus() {
        info("Setting generation of features in src directory to: " + getFormattedBooleanString(generateToSrc));
    }

    private void infoSrcDirModified() {
        String generatedFileCanonicalPath;
        try {
            generatedFileCanonicalPath = new File(configDirectory, FeatureGeneratorUtil.GENERATED_FEATURES_FILE_PATH).getCanonicalPath();
        } catch (IOException e) {
            generatedFileCanonicalPath = new File(configDirectory, FeatureGeneratorUtil.GENERATED_FEATURES_FILE_PATH).toString();
        }
        info("The source configuration directory will be modified. Features will automatically be generated in a new file: " + generatedFileCanonicalPath);
    }

    // called by Liberty plugins
    protected void setFeatureGeneration(boolean generateFeatures) {
        this.generateFeatures = generateFeatures;
        logFeatureGenerationStatus();
    }

    protected boolean optimizeGenerateFeatures(boolean useTmpDir) {
        debug("Entering optimizeGenerateFeatures(boolean)");
        return optimizeGenerateFeatures(useTmpDir, false);
    }

    /**
     * Generate features using all classes and only user specified features.
     */
    protected boolean optimizeGenerateFeatures(boolean useTmpDirOut, boolean useTmpDirIn) {
        debug("Generating optimized features list...use temp directory for output=" + useTmpDirOut + " use temp directory for input=" + useTmpDirIn);
        // scan all class files and provide only user specified features
        boolean generatedFeatures = libertyGenerateFeatures(null, true, generateToSrc, useTmpDirOut, useTmpDirIn);
        if (generatedFeatures) {
            modifiedClasses.clear();
            failedToGenerateClasses.clear();
            generatedFeaturesModified = generatedFeaturesModified();
        } // do not need to log an error if generatedFeatures is false because that would
          // have already been logged by libertyGenerateFeatures
        return generatedFeatures;
    }

    /**
     * Generate features using updated classes and all existing features.
     * Returns true if successful
     */
    private boolean incrementGenerateFeatures(boolean useTmpDir) {
        debug("Generating feature list from incremental changes...use temp directory=" + useTmpDir);
        boolean generatedFeatures = false;
        try {
            Collection<String> classPaths = getClassPaths(modifiedClasses);
            generatedFeatures = libertyGenerateFeatures(classPaths, false, generateToSrc, useTmpDir, false);
            if (generatedFeatures) {
                modifiedClasses.clear();
                failedToGenerateClasses.clear();
                generatedFeaturesModified = generatedFeaturesModified();
            } else {
                // do not need to log an error if generatedFeatures is false because that would
                // have already been logged by libertyGenerateFeatures
                failedToGenerateClasses.addAll(modifiedClasses);
                modifiedClasses.clear();
            }
        } catch (IOException e) {
            error("An error occurred while trying to generate features: " + e.getMessage(), e);
        }

        return generatedFeatures;
    }

    private class HotkeyReader implements Runnable {
        private Scanner scanner;
        private ThreadPoolExecutor executor;
        private boolean shutdown = false;

        public HotkeyReader(ThreadPoolExecutor executor) {
            this.executor = executor;
        }

        @Override
        public void run() {
            debug("Running hotkey reader thread");
            scanner = new Scanner(new CloseShieldInputStream(System.in)); // shield allows us to close the scanner without closing System.in.
            try {
                readInput();
            } finally {
                scanner.close();
            }
        }

        public void shutdown() {
            shutdown = true;
        }

        private void readInput() {
            HotKey q = new HotKey("q", "quit", "exit");
            HotKey h = new HotKey("h", "help");
            HotKey r = new HotKey("r");
            HotKey g = new HotKey("g");
            HotKey s = new HotKey("s");
            HotKey o = new HotKey("o");
            HotKey t = new HotKey("t");
            HotKey p = new HotKey("p");
            HotKey enter = new HotKey("");
            if (scanner.hasNextLine()) {
                synchronized (inputUnavailable) {
                    inputUnavailable.notify();
                }
                while (!shutdown) {
                    debug("Waiting for action key");
                    
                    if (!scanner.hasNextLine()) {
                        break;
                    }
                    String line = scanner.nextLine();
                    if (q.isPressed(line)) {
                        debug("Detected exit command");
                        runShutdownHook(executor);
                    } else if (r.isPressed(line)) {
                        debug("Detected restart command");
                        try {
                            restartServer(true);
                        } catch (PluginExecutionException e) {
                            debug("Exiting dev mode due to server restart failure");
                            error("Could not restart the server.", e);
                            runShutdownHook(executor);
                        }
                    } else if (h.isPressed(line)) {
                        info(formatAttentionBarrier());
                        printHelpMessages();
                        info(formatAttentionBarrier());
                    } else if (g.isPressed(line)) {
                        toggleFeatureGeneration();
                    } else if (s.isPressed(line)) {
                        toggleGenerateToSrc();
                    } else if (o.isPressed(line)) {
                        if (generateFeatures) {
                            optimizeGenerateFeatures(!generateToSrc);
                        } else {
                            warn("Cannot optimize features because automatic generation of features is off.");
                            warn("To toggle the automatic generation of features, type 'g' and press Enter.");
                        }
                    } else if (p.isPressed(line)) {
                        try {
                            info(formatAttentionBarrier());
                            printPortInfo(true);
                            info(formatAttentionBarrier());
                        } catch (PluginExecutionException e) {
                            error("Could not get port information.", e);
                        }
                    } else if ((t.isPressed(line) && isChangeOnDemandTestsAction()) || (enter.isPressed(line) && !isChangeOnDemandTestsAction())) {
                        debug("Detected test command. Running tests... ");
                        if (isMultiModuleProject()) {
                            // force run tests across all modules in multi module scenario
                            runTestThread(false, executor, -1, true, getAllBuildFiles());
                        } else {
                            runTestThread(false, executor, -1, true, buildFile);
                        }
                    } else if (enter.isPressed(line) && isChangeOnDemandTestsAction()) {
                        warn("Unrecognized command: Enter. To see the help menu, type 'h' and press Enter.");
                    } else {
                        warn("Unrecognized command: " + line + ". To see the help menu, type 'h' and press Enter.");
                    }
                }
            } else {
                synchronized (inputUnavailable) {
                    inputUnavailable.set(true);
                    inputUnavailable.notify();
                }
            }
        }
    }

    Collection<File> recompileJavaSources;
    Collection<File> recompileJavaTests;
    Collection<File> deleteJavaSources;
    Collection<File> deleteJavaTests;
    Collection<File> failedCompilationJavaSources;
    Collection<File> failedCompilationJavaTests;
    Collection<File> modifiedClasses; // can contain class files or output class dirs
    Collection<File> failedToGenerateClasses; // classes that we failed to generate features for
    Collection<File> omitWatchingFiles;
    long lastJavaSourceChange;
    long lastJavaTestChange;
    Map<File, Long> lastBuildFileChange;
    boolean triggerJavaSourceRecompile;
    boolean triggerJavaTestRecompile;
    File outputDirectory;
    File testOutputDirectory;
    File serverXmlFile;
    File serverXmlFileParent;
    File bootstrapPropertiesFile;
    File bootstrapPropertiesFileParent;
    File jvmOptionsFile;
    File jvmOptionsFileParent;
    File containerfileUsed;
    File looseAppFile;
    WatchService watcher;
    boolean lastChangeCompiled; // tracks when the module containing the latest src file change is compiled successfully

    // used for multi module projects
    boolean triggerUpstreamJavaSourceRecompile;
    boolean initialCompile;
    boolean disableDependencyCompile;

    /**
     * Watch files for changes.
     * 
     * @param outputDirectory
     * @param testOutputDirectory
     * @param executor
     * @param serverXmlFile           Can be null when using the server.xml from the configDirectory, which has a default value.
     * @param bootstrapPropertiesFile
     * @param jvmOptionsFile
     * @throws Exception
     */
    public void watchFiles(File outputDirectory, File testOutputDirectory, final ThreadPoolExecutor executor,
            File serverXmlFile, File bootstrapPropertiesFile, File jvmOptionsFile) throws Exception {
        this.outputDirectory = outputDirectory;
        this.testOutputDirectory = testOutputDirectory;
        this.serverXmlFile = serverXmlFile;
        this.bootstrapPropertiesFile = bootstrapPropertiesFile;
        this.jvmOptionsFile = jvmOptionsFile;
        this.containerfileUsed = null;
        this.initialCompile = true;
        this.lastChangeCompiled = false;
        this.disableDependencyCompile = false;
        this.omitWatchingFiles = new ArrayList<File>();

        try {
            if (isLooseApplication()) {
                this.looseAppFile = getLooseApplicationFile();
                debug("Loose application configuration file set to: " + looseAppFile);
            }

            watcher = FileSystems.getDefault().newWatchService();
            serverXmlFileParent = null;
            if (serverXmlFile != null && serverXmlFile.exists()) {
                serverXmlFileParent = serverXmlFile.getParentFile();
            }

            bootstrapPropertiesFileParent = null;
            if (bootstrapPropertiesFile != null && bootstrapPropertiesFile.exists()) {
                bootstrapPropertiesFileParent = bootstrapPropertiesFile.getParentFile();
            }

            jvmOptionsFileParent = null;
            if (jvmOptionsFile != null && jvmOptionsFile.exists()) {
                jvmOptionsFileParent = jvmOptionsFile.getParentFile();
            }

            Path srcPath = this.sourceDirectory.getCanonicalFile().toPath();
            Path testSrcPath = this.testSourceDirectory.getCanonicalFile().toPath();
            Path configPath = this.configDirectory.getCanonicalFile().toPath();
            Path outputPath = this.outputDirectory.getCanonicalFile().toPath();

            boolean sourceDirRegistered = false;
            boolean testSourceDirRegistered = false;
            boolean configDirRegistered = false;
            boolean serverXmlFileRegistered = false;
            boolean bootstrapPropertiesFileRegistered = false;
            boolean jvmOptionsFileRegistered = false;
            boolean sourceOutputDirRegistered = false;
            boolean mmOutputDirRegAttempted = false; //multi module output directories registration attempted

            // register parent poms
            if (!parentBuildFiles.isEmpty()) {
                for (String key : parentBuildFiles.keySet()) {
                    File parentBuildFile = new File(key);
                    if (parentBuildFile.exists()) {
                        registerSingleFile(parentBuildFile, executor);
                    }
                }
            }

            // check for upstream projects
            if (isMultiModuleProject()) {
                for (ProjectModule p : upstreamProjects) {
                    // TODO: Consider whether or not we should generate features at this point
                    updateArtifactPaths(p, false, generateFeatures, executor);

                    if (shouldIncludeSources(p.getPackagingType())) {
                        // watch src/main/java dir
                        if (p.getSourceDirectory().exists()) {
                            omitWatchingFiles.addAll(getOmitFilesList(looseAppFile, p.getSourceDirectory().getCanonicalPath()));
                            registerAll(p.getSourceDirectory().getCanonicalFile().toPath(), executor);
                            p.sourceDirRegistered = true;
                        }
                    }

                    // watch src/test/java dir
                    if (p.getTestSourceDirectory().exists()) {
                        registerAll(p.getTestSourceDirectory().getCanonicalFile().toPath(), executor);
                        p.testSourceDirRegistered = true;
                    }

                    // watch resource directories
                    HashMap<File, Boolean> upstreamResourceMap = new HashMap<File, Boolean>();
                    for (File upstreamResourceDir : p.getResourceDirs()) {
                        upstreamResourceMap.put(upstreamResourceDir, false);
                        if (upstreamResourceDir.exists()) {
                            registerAll(upstreamResourceDir.getCanonicalFile().toPath(), executor);
                            upstreamResourceMap.put(upstreamResourceDir, true);
                        }
                    }
                    p.setResourceMap(upstreamResourceMap);

                    // watch pom.xml
                    if (p.getBuildFile().exists()) {
                        registerSingleFile(p.getBuildFile(), executor);
                    }
                }
            }

            if (shouldIncludeSources(packagingType)) {
                if (this.sourceDirectory.exists()) {
                    omitWatchingFiles.addAll(getOmitFilesList(looseAppFile, this.sourceDirectory.getCanonicalPath()));
                    registerAll(srcPath, executor);
                    sourceDirRegistered = true;
                }
            }

            if (this.testSourceDirectory.exists()) {
                registerAll(testSrcPath, executor);
                testSourceDirRegistered = true;
            }

            if (this.configDirectory.exists()) {
                registerAll(configPath, executor);
                configDirRegistered = true;
            }

            if (serverXmlFile != null && serverXmlFile.exists() && serverXmlFileParent.exists()) {
                Path serverXmlFilePath = serverXmlFileParent.getCanonicalFile().toPath();
                registerAll(serverXmlFilePath, executor);
                serverXmlFileRegistered = true;
            }

            if (bootstrapPropertiesFile != null && bootstrapPropertiesFile.exists() && bootstrapPropertiesFileParent.exists()) {
                Path bootstrapPropertiesFilePath = bootstrapPropertiesFileParent.getCanonicalFile().toPath();
                registerAll(bootstrapPropertiesFilePath, executor);
                bootstrapPropertiesFileRegistered = true;
            }

            if (jvmOptionsFile != null && jvmOptionsFile.exists() && jvmOptionsFileParent.exists()) {
                Path jvmOptionsFilePath = jvmOptionsFileParent.getCanonicalFile().toPath();
                registerAll(jvmOptionsFilePath, executor);
                jvmOptionsFileRegistered = true;
            }

            if (container) {
                containerfileUsed = getContainerfile();
                registerSingleFile(containerfileUsed, executor);
            }

            // Always register the generated file in the temp dir. because generateToSrc can be toggled on and off in dev mode
            File hiddenTempGenerateFeaturesFile = new File(generateFeaturesTmpDir, FeatureGeneratorUtil.GENERATED_FEATURES_FILE_PATH);
            hiddenTempGenerateFeaturesFile.getParentFile().mkdirs(); // must only mkdir on the directories
            registerSingleFile(hiddenTempGenerateFeaturesFile, executor);

            HashMap<File, Boolean> resourceMap = new HashMap<File, Boolean>();
            for (File resourceDir : resourceDirs) {
                resourceMap.put(resourceDir, false);
                if (resourceDir.exists()) {
                    registerAll(resourceDir.getCanonicalFile().toPath(), executor);
                    resourceMap.put(resourceDir, true);
                }
            }
            
            HashMap<Path, Boolean> webResourceMap = new HashMap<Path, Boolean>();
            for (Path webResourceDir : monitoredWebResourceDirs) {
                webResourceMap.put(webResourceDir, false);
                if (Files.exists(webResourceDir)) {
                    registerAll(webResourceDir, executor);
                    webResourceMap.put(webResourceDir, true);
                }
            }

            registerSingleFile(buildFile, executor);

            if (propertyFilesMap != null) {
                for (File f : propertyFilesMap.keySet()) {
                    registerSingleFile(f, executor);
                }
            }

            initWatchLoop();

            while (true) {
                // Check the server and stop dev mode by throwing an exception if the server stopped.
                checkStopDevMode(true);

                if (container) {
                    synchronized(containerfileDirectoriesToWatch) {
                        if (!containerfileDirectoriesToWatch.isEmpty()) {
                            for (Path path : containerfileDirectoriesToWatch) {
                                File f = path.toFile();
                                if (f.isDirectory()) {
                                    debug("Registering path from containerfileDirectoriesToWatch: " + path);
                                    registerAll(path, executor, true);
                                } else {
                                    debug("Registering file path from containerfileDirectoriesToWatch: " + path);
                                    registerSingleFile(f, executor, true);
                                }
                                containerfileDirectoriesTracked.add(path);
                            }
                            containerfileDirectoriesToWatch.clear();
                        }
                    }
                }
                if (isMultiModuleProject()) { // process java compilation for upstream projects
                    boolean change = processUpstreamJavaCompilation(upstreamProjects, executor);

                    // process java compilation for main project
                    processJavaCompilation(outputDirectory, testOutputDirectory, executor, compileArtifactPaths,
                            testArtifactPaths, applicationId, change);
                } else {
                    // process java compilation for main project
                    processJavaCompilation(outputDirectory, testOutputDirectory, executor, compileArtifactPaths,
                            testArtifactPaths, null, false);
                }

                // Assumption is that initialCompile will only be set to false after ALL modules are complete
                // Will need to update logic once multiple attempts are enabled for upstream projects
                if (!recompileDependencies && !initialCompile && !sourceOutputDirRegistered) {
                    // Allow more than one register attempt for adding a new output directory once adding a src/main/java dir
                    // to an upstream project is supported - https://github.com/OpenLiberty/ci.maven/issues/1202
                    if (isMultiModuleProject() && !mmOutputDirRegAttempted) {
                        debug("Attempt to register upstream class file directories");
                        mmOutputDirRegAttempted = true;
                        for (ProjectModule p : upstreamProjects) {
                            if (shouldIncludeSources(p.getPackagingType())) {
                                // register source classes directory for upstream modules
                                if (p.getOutputDirectory().exists()) {
                                    registerAll(p.getOutputDirectory().getCanonicalFile().toPath(), executor);
                                    p.sourceOutputDirRegistered = true;
                                }
                            }
                        }
                    }
                    if (shouldIncludeSources(packagingType)) {
                        // register class output directory for main module
                        if (this.outputDirectory.exists() && this.outputDirectory.list().length > 0) {
                            debug("Registering class output directory: " + this.outputDirectory);
                            registerAll(outputPath, executor);
                            sourceOutputDirRegistered = true;
                        }
                    }
                }

                // Generate features from source or class file changes
                // do not run generate features if there are classes failing to compile
                if (generateFeatures && !classesFailingToCompile() && !modifiedClasses.isEmpty()) {
                    // recompileDependencies = no class file tracking, so we wait for compilation to be complete
                    // !recompileDepenencies = class file tracking, so waiting on class file changes
                    if ((recompileDependencies && lastChangeCompiled) || !recompileDependencies) {
                        if (!failedToGenerateClasses.isEmpty()) {
                            modifiedClasses.addAll(failedToGenerateClasses);
                        }
                        debug("Detected a change in the following classes/directories: " + modifiedClasses);
                        // reset lastChangeCompiled and modifiedSrcBuildFile
                        lastChangeCompiled = false; // only needed when recompileDependencies is true
                        modifiedSrcBuildFile = null; // only needed when recompileDependencies is true
                        long generatedTime = generateFeaturesFile.lastModified();
                        int numApplicationUpdatedMessages = countApplicationUpdatedMessages();
                        incrementGenerateFeatures(!generateToSrc);
                        if (!generateFeaturesFile.exists()) {
                            // run tests if generated-features.xml does not exist as there are no new features to install
                            // (typically tests run after generate features & install when hotTests=true)
                            if (isMultiModuleProject()) {
                                runTestThread(false, executor, -1, false, getAllBuildFiles());
                            } else {
                                runTestThread(false, executor, -1, false, false, buildFile);
                            }
                        } else if (generateFeaturesFile.lastModified() == generatedTime) {
                            // The generated-features.xml file was not modified by adding or removing features as a
                            // result of the compilation so call tests now. If it had been changed tests would be called
                            // after processing the config file change.
                            if (isMultiModuleProject()) {
                                runTestThread(false, executor, numApplicationUpdatedMessages, false, getAllBuildFiles());
                            } else {
                                runTestThread(false, executor, numApplicationUpdatedMessages, false, false, buildFile);
                            }
                        }
                    }
                }

                if (shouldIncludeSources(packagingType)) {
                    // check if javaSourceDirectory has been added
                    if (!sourceDirRegistered && this.sourceDirectory.exists()
                            && this.sourceDirectory.listFiles().length > 0) {
                        // Count the messages before the compile.
                        int numApplicationUpdatedMessages = countApplicationUpdatedMessages();
                        compile(this.sourceDirectory);
                        registerAll(srcPath, executor);
                        debug("Registering Java source directory: " + this.sourceDirectory);
                        // run tests after waiting for app update since app changed
                        runTestThread(true, executor, numApplicationUpdatedMessages, false, buildFile);
                        sourceDirRegistered = true;
                    } else if (sourceDirRegistered && !this.sourceDirectory.exists()) {
                        cleanTargetDir(outputDirectory);
                        sourceDirRegistered = false;
                    }
                }

                // check if testSourceDirectory has been added
                if (!testSourceDirRegistered && this.testSourceDirectory.exists()
                        && this.testSourceDirectory.listFiles().length > 0) {
                    compile(this.testSourceDirectory);
                    registerAll(testSrcPath, executor);
                    debug("Registering Java test directory: " + this.testSourceDirectory);
                    runTestThread(false, executor, -1, false, buildFile);
                    testSourceDirRegistered = true;

                } else if (testSourceDirRegistered && !this.testSourceDirectory.exists()) {
                    cleanTargetDir(testOutputDirectory);
                    testSourceDirRegistered = false;
                }

                // check if configDirectory has been added
                if (!configDirRegistered && this.configDirectory.exists()) {
                    configDirRegistered = true;
                    if (serverXmlFile != null && !serverXmlFile.exists()) {
                        registerAll(configPath, executor);
                        debug("Registering configuration directory: " + this.configDirectory);
                    } else {
                        warn("The server configuration directory " + configDirectory
                                + " has been added. Restart dev mode for it to take effect.");
                    }
                }

                // check if serverXmlFile has been added
                if (!serverXmlFileRegistered && serverXmlFile != null && serverXmlFile.exists()) {
                    serverXmlFileRegistered = true;
                    debug("Server configuration file has been added: " + serverXmlFile);
                    warn("The server configuration file " + serverXmlFile
                            + " has been added. Restart dev mode for it to take effect.");
                }

                if (!bootstrapPropertiesFileRegistered && bootstrapPropertiesFile != null && bootstrapPropertiesFile.exists()) {
                    bootstrapPropertiesFileRegistered = true;
                    debug("Bootstrap properties file has been added: " + bootstrapPropertiesFile);
                    warn("The bootstrap properties file " + bootstrapPropertiesFile
                            + " has been added. Restart dev mode for it to take effect.");
                }

                if (!jvmOptionsFileRegistered && jvmOptionsFile != null && jvmOptionsFile.exists()) {
                    jvmOptionsFileRegistered = true;
                    debug("JVM Options file has been added: " + jvmOptionsFile);
                    warn("The JVM Options file " + jvmOptionsFile
                            + " has been added. Restart dev mode for it to take effect.");
                }

                // check if resourceDirectory has been added
                for (File resourceDir : resourceDirs) {
                    if (!resourceMap.get(resourceDir) && resourceDir.exists()) {
                    	resourceDirectoryCreated();
                        registerAll(resourceDir.getCanonicalFile().toPath(), executor);
                        resourceMap.put(resourceDir, true);
                    } else if (resourceMap.get(resourceDir) && !resourceDir.exists()) {
                        // deleted resource directory
                        warn("The resource directory " + resourceDir
                                + " was deleted.  Restart dev mode for it to take effect.");
                        resourceMap.put(resourceDir, false);
                    }
                }
                
                // Check if webResourceDirectory has been added or deleted
                for (Path webResourceDir : monitoredWebResourceDirs) {
                    if (!webResourceMap.get(webResourceDir) && Files.exists(webResourceDir)) {
                    	updateLooseApp();
                        registerAll(webResourceDir, executor);
                        webResourceMap.put(webResourceDir, true);
                    	runTestThread(false, executor, -1, false, false);
                    } else if (webResourceMap.get(webResourceDir) && !Files.exists(webResourceDir)) {
                        // deleted webResource directory
                    	updateLooseApp();
                        warn("The webResource directory " + webResourceDir
                                + " was deleted.  Restart liberty:dev mode for it to take effect.");
                        webResourceMap.put(webResourceDir, false);
                    	runTestThread(false, executor, -1, false, false);
                    }
                }

                if (isMultiModuleProject()) {
                    for (ProjectModule p : upstreamProjects) {

                        // check if resource directory of an upstream project has been added/deleted
                        for (File resourceDir : p.getResourceDirs()) {
                            if (!p.getResourceMap().get(resourceDir) && resourceDir.exists()) {
                                registerAll(resourceDir.getCanonicalFile().toPath(), executor);
                                p.getResourceMap().put(resourceDir, true);
                            } else if (p.getResourceMap().get(resourceDir) && !resourceDir.exists()) {
                                // deleted resource directory
                                warn("The resource directory " + resourceDir
                                        + " was deleted.  Restart dev mode for it to take effect.");
                                p.getResourceMap().put(resourceDir, false);
                            }
                        }

                        // Adding a src/main/java dir to an upstream project is not currently supported
                        // for multi module projects. See
                        // https://github.com/OpenLiberty/ci.maven/issues/1202
                        if (shouldIncludeSources(p.getPackagingType())) {
                            if (!p.sourceDirRegistered && p.getSourceDirectory().exists()
                                    && p.getSourceDirectory().listFiles().length > 0) {
                                p.sourceDirRegistered = true;
                                warn("The source directory " + p.getSourceDirectory()
                                        + " was added.  This may result in compilation errors between dependent modules.  Restart dev mode for it to take effect.");
                            } else if (p.sourceDirRegistered && !p.getSourceDirectory().exists()) {
                                p.sourceDirRegistered = false;
                                warn("The source directory " + p.getSourceDirectory()
                                        + " was deleted.  This may result in compilation errors between dependent modules.  Restart dev mode for it to take effect.");
                            }
                        }

                        // check if test directory of an upstream project has been added/deleted
                        if (!p.testSourceDirRegistered && p.getTestSourceDirectory().exists()
                                && p.getTestSourceDirectory().listFiles().length > 0) {
                            compile(p.getTestSourceDirectory(), p);
                            registerAll(p.getTestSourceDirectory().getCanonicalFile().toPath(), executor);
                            p.testSourceDirRegistered = true;
                            debug("Registering Java test directory: " + p.getTestSourceDirectory());
                            // compile all tests in downstream modules
                            for (File dependentBuildFile : p.getDependentModules()) {
                                ProjectModule depModule = getProjectModule(dependentBuildFile);
                                if (depModule != null) {
                                    compile(depModule.getTestSourceDirectory(), depModule);
                                } else {
                                    // main module
                                    compile(this.testSourceDirectory);
                                }
                            }
                            runTestThread(false, executor, -1, false, getAllBuildFiles(p));
                        } else if (p.testSourceDirRegistered && !p.getTestSourceDirectory().exists()) {
                            cleanTargetDir(p.getTestOutputDirectory());
                            p.testSourceDirRegistered = false;
                        }
                    }
                }

                if (trackingMode == FileTrackMode.FILE_WATCHER || trackingMode == FileTrackMode.NOT_SET) {
                    try {
                        final WatchKey wk = watcher.poll(100, TimeUnit.MILLISECONDS);
                        final Watchable watchable = wk.watchable();
                        final Path directory = (Path) watchable;

                        List<WatchEvent<?>> events = wk.pollEvents();

                        for (WatchEvent<?> event : events) {
                            if (trackingMode == FileTrackMode.NOT_SET) {
                                trackingMode = FileTrackMode.FILE_WATCHER;
                                disablePolling();
                            }
                            final Path changed = (Path) event.context();
                            debug("Processing events for watched directory: " + directory);

                            File fileChanged = new File(directory.toString(), changed.toString());
                            if (ignoreFileOrDir(fileChanged)) {
                                // skip this file or directory, and continue to the next file or directory
                                continue;
                            }
                            debug("Changed: " + changed + "; " + event.kind());

                            ChangeType changeType = null;
                            if (event.kind() == StandardWatchEventKinds.ENTRY_CREATE) {
                                changeType = ChangeType.CREATE;
                            } else if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) {
                                changeType = ChangeType.MODIFY;
                            } else if (event.kind() == StandardWatchEventKinds.ENTRY_DELETE) {
                                changeType = ChangeType.DELETE;
                            }
                            processFileChanges(executor, fileChanged, outputDirectory, false, changeType);

                        }
                        // reset the key
                        boolean valid = wk.reset();
                        if (!valid) {
                            debug("WatchService key has been unregistered for " + directory);
                        }
                    } catch (InterruptedException | NullPointerException e) {
                        // do nothing let loop continue
                    }
                }
                if (trackingMode == FileTrackMode.POLLING || trackingMode == FileTrackMode.NOT_SET) {
                    synchronized (newFileObservers) {
                        consolidateFileObservers();
                    }
                    // iterate through file observers
                    for (FileAlterationObserver observer : fileObservers) {
                        if (!cancelledFileObservers.contains(observer)) {
                            observer.checkAndNotify();
                        }
                    }
                    synchronized (cancelledFileObservers) {
                        removeCancelledFileObservers();
                    }

                    Thread.sleep(pollingInterval);
                }
            }
        } finally {
            if (watcher != null) {
                try {
                    watcher.close();
                } catch (IOException e) {
                    error("An error occurred attempting to close the file watcher. " + e.getMessage(), e);
                }
            }
        }
    }

    /**
     * 
     * @param classFiles javaSourceClassFiles that have been modified
     * @return {@code Collection<String>} of class paths
     * @throws IOException
     */
    private Collection<String> getClassPaths(Collection<File> classFiles) throws IOException {
        Collection<String> classPaths = new HashSet<String>();
        for (File classPath : classFiles) {
            classPaths.add(classPath.getCanonicalPath());
        }
        return classPaths;
    }

    /**
     * Whether source files should be watched/included for compilation for this packaging type.
     */
    private boolean shouldIncludeSources(String packaging) {
        return !("ear".equals(packaging) || "pom".equals(packaging));
    }

    /**
     * Consolidate new file observers into the main observers set
     */
    private void consolidateFileObservers() {
        fileObservers.addAll(newFileObservers);
        newFileObservers.clear();
    }

    /**
     * Remove cancelled file observers from the main observers set
     */
    private void removeCancelledFileObservers() {
        fileObservers.removeAll(cancelledFileObservers);
        cancelledFileObservers.clear();
    }

    private void registerSingleFile(final File registerFile, final ThreadPoolExecutor executor) throws IOException {
        registerSingleFile(registerFile, executor, false);
    }

    /**
     * Register a single file with the WatchService using a file filter.
     * 
     * @param registerFile             the file of interest
     * @param executor                 the test thread executor
     * @param removeOnContainerRebuild whether the files should be unwatched if the container is rebuilt
     * @throws IOException unable to read the canonical path name
     */
    private void registerSingleFile(final File registerFile, final ThreadPoolExecutor executor, boolean removeOnContainerRebuild) throws IOException {
        if (trackingMode == FileTrackMode.POLLING || trackingMode == FileTrackMode.NOT_SET) {
            String parentPath = registerFile.getParentFile().getCanonicalPath();

            debug("Registering single file polling for " + registerFile.toString());

            // synchronize on the new observer set since only those are being updated in separate threads
            synchronized (newFileObservers) {
                Set<FileAlterationObserver> tempCombinedObservers = new HashSet<FileAlterationObserver>();
                tempCombinedObservers.addAll(fileObservers);
                tempCombinedObservers.addAll(newFileObservers);

                FileAlterationObserver existingObserver = null;

                // if this path is already observed, ignore it
                for (FileAlterationObserver observer : tempCombinedObservers) {
                    if (parentPath.equals(observer.getDirectory().getCanonicalPath())) {
                        debug("Updating file polling for " + registerFile.toString() + " since its parent directory is already being observed");
                        existingObserver = observer;
                    }
                }

                FileFilter singleFileFilter = new FileFilter() {
                    @Override
                    public boolean accept(File file) {
                        try {
                            if (file.getCanonicalFile().equals(registerFile.getCanonicalFile())) {
                                return true;
                            }
                        } catch (IOException e) {
                            if (file.equals(registerFile)) {
                                return true;
                            }
                        }
                        return false;
                    }
                };

                try {
                    FileAlterationObserver observer = existingObserver;
                    if (observer != null) {
                        debug("Updating parent file observer for: " + registerFile.toString());
                        observer = addFileAlterationObserver(executor, observer, parentPath, singleFileFilter);
                    } else {
                        debug("Adding single file observer for: " + registerFile.toString());
                        observer = addFileAlterationObserver(executor, parentPath, singleFileFilter);
                    }
                    if (removeOnContainerRebuild) {
                        debug("Adding file to containerfileDirectoriesFileObservers: " + registerFile.toString());
                        containerfileDirectoriesFileObservers.add(observer);
                    }
                } catch (Exception e) {
                    error("Could not observe single file " + registerFile.toString(), e);
                }
            }
        }
        
        if (trackingMode == FileTrackMode.FILE_WATCHER || trackingMode == FileTrackMode.NOT_SET) {
            debug("Adding directory to WatchService " + registerFile.getParentFile().toPath() + " for single file " + registerFile.getName());
            WatchKey key = registerFile.getParentFile().toPath().register(
                watcher, 
                new WatchEvent.Kind[] { StandardWatchEventKinds.ENTRY_MODIFY,
                        StandardWatchEventKinds.ENTRY_DELETE, StandardWatchEventKinds.ENTRY_CREATE },
                SensitivityWatchEventModifier.HIGH);
            if (removeOnContainerRebuild) {
                debug("Adding file to containerfileDirectoriesWatchKeys: " + registerFile.getName());
                containerfileDirectoriesWatchKeys.add(key);
            }
        }
    }

    private FileAlterationObserver addFileAlterationObserver(final ThreadPoolExecutor executor, FileAlterationObserver observer, String parentPath, FileFilter filter)
            throws Exception {
        // create new observer for filter
        FileAlterationObserver newObserver = getFileAlterationObserver(executor, parentPath, filter);
        // iterate through existing listeners on observer and add them to newObserver
        for (FileAlterationListener nextListener: observer.getListeners()) {
            newObserver.addListener(nextListener);
        }
        newObserver.initialize();
        // clean up previous observer
        fileObservers.remove(observer);
        newFileObservers.remove(observer);
        try {
            // destroy the observer
            observer.destroy();
        } catch (Exception e) {
            debug("Could not destroy file observer", e);
        }
        // add new observer with the combined listeners
        newFileObservers.add(newObserver);
        return newObserver;
    }

    private FileAlterationObserver addFileAlterationObserver(final ThreadPoolExecutor executor, String parentPath, FileFilter filter)
            throws Exception {
        FileAlterationObserver observer = getFileAlterationObserver(executor, parentPath, filter);
        observer.initialize();
        newFileObservers.add(observer);
        return observer;
    }

    private FileAlterationObserver getFileAlterationObserver(final ThreadPoolExecutor executor, final String parentPath, FileFilter filter) throws Exception {
        FileAlterationObserver observer = new FileAlterationObserver(parentPath, filter);
        addFileAlterationListener(executor, observer, parentPath, filter);
        return observer;
    }

    private void addFileAlterationListener(final ThreadPoolExecutor executor, FileAlterationObserver observer, final String parentPath, FileFilter filter) {
        observer.addListener(new FileAlterationListenerAdaptor() {
            @Override
            public void onDirectoryCreate(File file) {
                onAlteration(executor, parentPath, file, true, ChangeType.CREATE);
            }

            @Override
            public void onDirectoryDelete(File file) {
                onAlteration(executor, parentPath, file, true, ChangeType.DELETE);
            }

            @Override
            public void onDirectoryChange(File file) {
                onAlteration(executor, parentPath, file, true, ChangeType.MODIFY);
            }

            @Override
            public void onFileCreate(File file) {
                onAlteration(executor, parentPath, file, false, ChangeType.CREATE);
            }

            @Override
            public void onFileDelete(File file) {
                onAlteration(executor, parentPath, file, false, ChangeType.DELETE);
            }

            @Override
            public void onFileChange(File file) {
                onAlteration(executor, parentPath, file, false, ChangeType.MODIFY);
            }

            private void onAlteration(final ThreadPoolExecutor executor, final String parentPath, File file,
                    boolean isDirectory, ChangeType changeType) {
                if (trackingMode == FileTrackMode.NOT_SET) {
                    try {
                        WatchKey wk = null;
                        if (watcher != null) {
                            // use bigger timeout on first file change to determine FILE_WATCHER vs POLLING (issue 1741)
                            wk = watcher.poll(1000, TimeUnit.MILLISECONDS);   
                        }
                        List<WatchEvent<?>> events = null;
                        if (wk != null) {
                            events = wk.pollEvents();
                        }
                        if ((events == null) || events.isEmpty()) {
                            debug("Setting file track mode to POLLING since no file watcher events were found.");
                            trackingMode = FileTrackMode.POLLING;
                            if (watcher != null) {
                                watcher.close();
                            }
                        } else {
                            debug("Setting file track mode to FILE_WATCHER.");
                            trackingMode = FileTrackMode.FILE_WATCHER;
                            disablePolling();
                            Watchable watchable = wk.watchable(); // for debug msg below
                            boolean valid = wk.reset();
                            if (!valid) {
                                debug("WatchService key has been unregistered for " + watchable);
                            }
                        }
                    } catch (Exception e) {
                        error("An error occurred attempting to retrieve the watch key or close the file watcher. " + e.getMessage(), e);
                    }
                }
                try {
                    processFileChanges(executor, file, outputDirectory, isDirectory, changeType);
                } catch(Exception e) {
                    debug(e);
                    error("Could not file process changes for " + file.getAbsolutePath() + ": " + e.getMessage());
                }
            }
        });
    }

    /**
     * Given the loose app file and the source directory path, return a list of
     * files that are specified in the loose app file and are in the source
     * directory and should be omitted from watching.
     * 
     * @param looseAppFile Loose Application configuration file
     * @param srcDirectoryPath the source directory path
     * @return a list of files that should be omitted from watching as they are on
     *         the source directory path and exist in the loose app config file
     */
    protected Collection<File> getOmitFilesList(File looseAppFile, String srcDirectoryPath) {
        Collection<File> omitFiles = new ArrayList<File>();
        try {
            if (looseAppFile != null && looseAppFile.exists()) {
                DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
                dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false); 
                dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
                dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
                dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
                dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
                dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
                dbf.setXIncludeAware(false);
                dbf.setExpandEntityReferences(false);
                DocumentBuilder db = dbf.newDocumentBuilder();
                Document document = db.parse(looseAppFile);
                NodeList archiveList = document.getElementsByTagName("archive");
                for (int i = 0; i < archiveList.getLength(); i++) {
                    NodeList ar = archiveList.item(i).getChildNodes();
                    for (int j = 0; j < ar.getLength(); j++) {
                        Node node = ar.item(j);
                        if (node.getNodeName().equals("dir") || node.getNodeName().equals("file")) {
                            String srcOnDiskNodeText = node.getAttributes().getNamedItem("sourceOnDisk").getTextContent();
                            if (container) {
                                srcOnDiskNodeText = srcOnDiskNodeText.replace("${" + DEVMODE_PROJECT_ROOT + "}",
                                        getLooseAppProjectRoot(projectDirectory, multiModuleProjectDirectory)
                                                .getCanonicalPath());
                            }
                            File srcOnDiskFile = new File(srcOnDiskNodeText);
                            if (srcOnDiskFile.getCanonicalPath().startsWith(srcDirectoryPath)) {
                                omitFiles.add(srcOnDiskFile);
                            }
                        }
                    }
                }
            }
        } catch (ParserConfigurationException | SAXException | IOException e) {
            error("Unable to read loose application configuration file: " + looseAppFile.toString());
            return null;
        }
        return omitFiles;
    }

    private boolean processUpstreamJavaCompilation(List<ProjectModule> upstreamProjects, final ThreadPoolExecutor executor)
            throws PluginExecutionException, IOException {
        boolean change = false;
        // process java source files if no changes detected after the compile wait time
        boolean processSources = System.currentTimeMillis() > lastJavaSourceChange + compileWaitMillis;
        boolean processTests = System.currentTimeMillis() > lastJavaTestChange + compileWaitMillis;

        if (processSources) {
            if (triggerUpstreamJavaSourceRecompile) { // this is triggered from build file change
                compileFailingProjects(null, false, executor);
                // compile main project
                if (!failedCompilationJavaSources.isEmpty()) {
                    triggerJavaSourceRecompile = true;
                }
                if (!failedCompilationJavaTests.isEmpty()) {
                    triggerJavaTestRecompile = true;
                }
                change = true;
            }
            for (ProjectModule project : upstreamProjects) {
                boolean successfulCompilation = true;
                boolean compileDownstreamSrc = false;
                boolean compileDownstreamTest = false;
                boolean pastBuildFileWaitPeriod = true;
                if (lastBuildFileChange.get(project.getBuildFile()) != null) {
                    pastBuildFileWaitPeriod = System
                            .currentTimeMillis() > lastBuildFileChange.get(project.getBuildFile()) + compileWaitMillis;
                }
                if (!pastBuildFileWaitPeriod) {
                    continue;
                }

                // delete before recompiling, so if a file is in both lists, its class
                // will be deleted then recompiled
                if (!project.deleteJavaSources.isEmpty()) {
                    debug("Deleting Java source files: " + project.deleteJavaSources);
                    for (File file : project.deleteJavaSources) {
                        deleteJavaFile(file, project.getOutputDirectory(), project.getSourceDirectory());
                    }
                    change = true;
                }
                if (!project.recompileJavaSources.isEmpty()) {
                    if (!project.failedCompilationJavaSources.isEmpty()) {
                        project.recompileJavaSources.addAll(project.failedCompilationJavaSources);
                    }
                    // try recompiling failing project modules that are not dependent on the current
                    // module (upstream of the current module)
                    if (shouldRecompileDependencies(project) || !recompileDependencies) {
                        compileDownstreamSrc = true;
                        if (!compileFailingProjects(project, false, executor)) {
                            successfulCompilation = false;
                        }
                    }
                    debug("Recompiling Java source files: " + project.recompileJavaSources);

                    // always skip running tests through recompileJavaSource on upstream projects
                    // since tests need to run on all dependent projects, runTestThread is called
                    // directly in logic below
                    if (recompileJavaSource(project.recompileJavaSources, project.getCompileArtifacts(), executor,
                            project.getOutputDirectory(), project.getTestOutputDirectory(), project.getProjectName(),
                            project.getBuildFile(), project.getCompilerOptions(), project.skipUTs(), true)) {
                        // successful compilation so we can clear failedCompilation list
                        project.failedCompilationJavaSources.clear();
                        change = true;
                        if (modifiedSrcBuildFile != null && 
                                project.getBuildFile().equals(modifiedSrcBuildFile)) {
                            // The module with the latest src file change has compiled successfully
                            debug("Setting lastChangeCompiled to true");
                            lastChangeCompiled = true;
                        }
                    } else {
                        successfulCompilation = false;
                        project.failedCompilationJavaSources.addAll(project.recompileJavaSources);
                    }
                }
                // additionally, process java test files if no changes detected after a
                // different timeout
                // (but source timeout takes precedence i.e. don't recompile tests if someone
                // keeps changing the source)
                if (processTests) {
                    // delete before recompiling, so if a file is in both lists, its class will be
                    // deleted then recompiled
                    if (!project.deleteJavaTests.isEmpty()) {
                        debug("Deleting Java test files: " + project.deleteJavaTests);
                        for (File file : project.deleteJavaTests) {
                            deleteJavaFile(file, project.getTestOutputDirectory(), project.getTestSourceDirectory());
                        }
                    }
                    if (!project.recompileJavaTests.isEmpty() || project.triggerJavaTestRecompile) {
                        if (!project.failedCompilationJavaTests.isEmpty()) {
                            project.recompileJavaTests.addAll(project.failedCompilationJavaTests);
                        }
                        // try recompiling failing project modules that are not dependent on the current
                        // module (upstream of the current module)
                        if (shouldRecompileDependencies(project) || !recompileDependencies) {
                            compileDownstreamTest = true;
                            if (!compileFailingProjects(project, true, executor)) {
                                successfulCompilation = false;
                            }
                        }
                        debug("Recompiling Java test files: " + project.recompileJavaTests);

                        // always skip running tests through recompileJavaTest on upstream projects
                        // since tests need to run on all dependent projects, runTestThread is called
                        // directly in logic below
                        if (recompileJavaTest(project.recompileJavaTests, project.getTestArtifacts(), executor,
                                project.getOutputDirectory(), project.getTestOutputDirectory(),
                                project.getProjectName(), project.getBuildFile(), project.getCompilerOptions(),
                                project.skipUTs(), true)) {
                            // successful compilation so we can clear failedCompilation list
                            project.failedCompilationJavaTests.clear();
                        } else {
                            successfulCompilation = false;
                            project.failedCompilationJavaTests.addAll(project.recompileJavaTests);
                        }
                    }
                }

                if (compileDownstreamSrc) { // compile downstream modules' source and test classes
                    // Count the messages before the compile.
                    int numApplicationUpdatedMessages = countApplicationUpdatedMessages();
                    for (File dependentModule : project.getDependentModules()) {
                        if (!recompileDependencies) {
                            // recompileDependencies = false, only compile failing classes from dependent
                            // modules
                            if (!compileFailingClasses(getProjectModule(dependentModule), false, executor)) {
                                successfulCompilation = false;
                            }
                        } else {
                            // recompileDependencies = true, compile all of the dependent module's classes
                            if (!compileModuleForBuildFile(dependentModule, false, executor)) {
                                successfulCompilation = false;
                            }
                        }
                    }
                    if (successfulCompilation && !generateFeatures) {
                        // do not run tests if generateFeatures = true, tests will run after generated-features.xml is updated
                        // run tests on current module and dependent modules
                        runTestThread(true, executor, numApplicationUpdatedMessages, false, getAllBuildFiles(project));
                    }
                } else if (compileDownstreamTest) { // compile downstream modules' test classes
                    for (File dependentModule : project.getDependentModules()) {
                        if (!recompileDependencies) {
                            // recompileDependencies = false, only compile failing test classes from
                            // dependent modules
                            if (!compileFailingClasses(getProjectModule(dependentModule), true, executor)) {
                                successfulCompilation = false;
                            }
                        } else {
                            // recompileDependencies = true, compile all of the dependent module's test
                            // classes
                            if (!compileModuleForBuildFile(dependentModule, true, executor)) {
                                successfulCompilation = false;
                            }
                        }
                    }
                    if (successfulCompilation) {
                        // run tests on current module and dependent modules without waiting for app
                        // update since only tests changed
                        runTestThread(false, executor, -1, false, getAllBuildFiles(project));
                    }
                }

                // run tests if files were deleted without any other changes, since
                // recompileJavaSource won't run (which normally handles tests)
                if (!project.deleteJavaSources.isEmpty() && project.recompileJavaSources.isEmpty()) {
                    // run tests after waiting for app update since app changed
                    int numApplicationUpdatedMessages = countApplicationUpdatedMessages();
                    runTestThread(true, executor, numApplicationUpdatedMessages, false, getAllBuildFiles(project));
                } else if (processTests && !project.deleteJavaTests.isEmpty() && project.recompileJavaTests.isEmpty()) {
                    // run tests without waiting for app update since only tests changed
                    runTestThread(false, executor, -1, false, getAllBuildFiles(project));
                }

                project.deleteJavaSources.clear();
                project.recompileJavaSources.clear();
                project.triggerJavaTestRecompile = false;
                if (processTests) {
                    project.deleteJavaTests.clear();
                    project.recompileJavaTests.clear();
                }
                if (project.disableDependencyCompile) {
                    project.disableDependencyCompile = false;
                }
            }
            triggerUpstreamJavaSourceRecompile = false;
        }
        return change;
    }

    /**
     * Compile failing source or test classes from upstream projects
     * 
     * @param currentProject Compile all projects of upstream of this project, if
     *                       null it is referring to main project
     * @param testsOnly      True if only tests need to be compiled
     * @param executor       ThreadPoolExecutor
     * @return true if all compilations were successful
     * @throws PluginExecutionException
     */
    private boolean compileFailingProjects(ProjectModule currentProject, boolean testsOnly, ThreadPoolExecutor executor)
            throws PluginExecutionException {
        // skip recompiling failed projects on initial loop to avoid repetitive
        // compilation calls
        if (initialCompile) {
            return false;
        }
        boolean successfulCompilation = true;
        // compile failing upstream projects
        for (int i = 0; i < upstreamProjects.size(); i++) {
            ProjectModule project = upstreamProjects.get(i);
            // stop at current project
            if (currentProject != null && upstreamProjects.get(i).equals(currentProject)) {
                break;
            }
            if (!compileFailingClasses(project, testsOnly, executor)) {
                successfulCompilation = false;
            }
        }
        return successfulCompilation;
    }

    private boolean compileFailingClasses(ProjectModule currentProject, boolean testsOnly, ThreadPoolExecutor executor) throws PluginExecutionException {
        if (initialCompile) {
            return false;
        }
        boolean successfulSrcCompile = true;
        boolean successfulTestCompile = true;

        // main project
        if (currentProject == null) {
            if (!testsOnly && !failedCompilationJavaSources.isEmpty()) {
                if (recompileJavaSource(failedCompilationJavaSources, compileArtifactPaths, executor, outputDirectory,
                        testOutputDirectory, getProjectName(), buildFile, compilerOptions, skipUTs, true)) {
                    // successful compilation so we can clear failedCompilation list
                    failedCompilationJavaSources.clear();
                } else {
                    successfulSrcCompile = false;
                }
            }
            if (!failedCompilationJavaTests.isEmpty()) {
                if (recompileJavaTest(failedCompilationJavaTests, testArtifactPaths, executor, outputDirectory,
                        testOutputDirectory, getProjectName(), buildFile, compilerOptions, skipUTs, true)) {
                    // successful compilation so we can clear failedCompilation list
                    failedCompilationJavaTests.clear();
                } else {
                    successfulTestCompile = false;
                }
            }
            return (successfulSrcCompile && successfulTestCompile);
        }

        // compile failing source
        if (!testsOnly && !currentProject.failedCompilationJavaSources.isEmpty()) {
            if (recompileJavaSource(currentProject.failedCompilationJavaSources, currentProject.getCompileArtifacts(),
                    executor, currentProject.getOutputDirectory(), currentProject.getTestOutputDirectory(),
                    currentProject.getProjectName(), currentProject.getBuildFile(), currentProject.getCompilerOptions(),
                    currentProject.skipUTs(), true)) {
                // successful compilation so we can clear failedCompilation list
                currentProject.failedCompilationJavaSources.clear();
            } else {
                successfulSrcCompile = false;
            }
        }
        // compile failing tests
        if (!currentProject.failedCompilationJavaTests.isEmpty()) {
            if (recompileJavaTest(currentProject.failedCompilationJavaTests, currentProject.getTestArtifacts(),
                    executor, currentProject.getOutputDirectory(), currentProject.getTestOutputDirectory(),
                    currentProject.getProjectName(), currentProject.getBuildFile(), currentProject.getCompilerOptions(),
                    currentProject.skipUTs(), true)) {
                // successful compilation so we can clear failedCompilation list
                currentProject.failedCompilationJavaTests.clear();
            } else {
                successfulTestCompile = false;
            }
        }
        return (successfulSrcCompile && successfulTestCompile);
    }

    private void processJavaCompilation(File outputDirectory, File testOutputDirectory, final ThreadPoolExecutor executor,
            Set<String> compileArtifactPaths, Set<String> testArtifactPaths, String projectName, boolean upstreamBuilt) throws IOException, PluginExecutionException {
        // process java source files if no changes detected after the compile wait time
        boolean processSources = System.currentTimeMillis() > lastJavaSourceChange + compileWaitMillis;
        boolean processTests = System.currentTimeMillis() > lastJavaTestChange + compileWaitMillis;
        boolean pastBuildFileWaitPeriod = true;
        if (lastBuildFileChange.get(buildFile) != null) {
            pastBuildFileWaitPeriod = System
                    .currentTimeMillis() > lastBuildFileChange.get(buildFile) + compileWaitMillis;
        }

        if (processSources && pastBuildFileWaitPeriod) {
            // Count the messages before the compile.
            int numApplicationUpdatedMessages = 0;
            // delete before recompiling, so if a file is in both lists, its class will be
            // deleted then recompiled
            if (!deleteJavaSources.isEmpty()) {
                debug("Deleting Java source files: " + deleteJavaSources);
                numApplicationUpdatedMessages = countApplicationUpdatedMessages();
                for (File file : deleteJavaSources) {
                    deleteJavaFile(file, outputDirectory, this.sourceDirectory);
                }
            }
            if (!recompileJavaSources.isEmpty() || triggerJavaSourceRecompile) {
                numApplicationUpdatedMessages = countApplicationUpdatedMessages();
                // try to recompile java files that previously did not compile successfully
                if (!failedCompilationJavaSources.isEmpty()) {
                    recompileJavaSources.addAll(failedCompilationJavaSources);
                }
                boolean skipRunningTests = false;
                if (initialCompile || disableDependencyCompile) {
                    // skip running tests when disableDependencyCompile is set as tests will be run
                    // through upstream project logic
                    skipRunningTests = true;
                }
                // if upstream projects exist try recompiling any failed projects
                if (isMultiModuleProject() && (!disableDependencyCompile || !recompileDependencies)) {
                    if (!compileFailingProjects(null, false, executor)) {
                        skipRunningTests = true;
                    }
                }
                debug("Recompiling Java source files: " + recompileJavaSources);
                if (recompileJavaSource(recompileJavaSources, compileArtifactPaths, executor, outputDirectory,
                        testOutputDirectory, projectName, buildFile, compilerOptions, skipUTs, skipRunningTests)) {
                    // successful compilation so we can clear failedCompilation list
                    failedCompilationJavaSources.clear();
                    if (modifiedSrcBuildFile != null && modifiedSrcBuildFile.equals(buildFile)) {
                        // The module with the latest src file change has compiled successfully
                        debug("Setting lastChangeCompiled to true");
                        lastChangeCompiled = true;
                    }
                } else {
                    failedCompilationJavaSources.addAll(recompileJavaSources);
                }
            }

            // additionally, process java test files if no changes detected after a
            // different timeout
            // (but source timeout takes precedence i.e. don't recompile tests if someone
            // keeps changing the source)
            if (processTests) {
                // delete before recompiling, so if a file is in both lists, its class will be
                // deleted then recompiled
                if (!deleteJavaTests.isEmpty()) {
                    debug("Deleting Java test files: " + deleteJavaTests);
                    for (File file : deleteJavaTests) {
                        deleteJavaFile(file, testOutputDirectory, this.testSourceDirectory);
                    }
                }
                if (!recompileJavaTests.isEmpty() || triggerJavaTestRecompile) {
                    if (!failedCompilationJavaTests.isEmpty()) {
                        recompileJavaTests.addAll(failedCompilationJavaTests);
                    }
                    boolean skipRunningTests = false;
                    if (initialCompile || disableDependencyCompile) {
                        // skip running tests when disableDependencyCompile is set as tests will be run
                        // through upstream project logic
                        skipRunningTests = true;
                    }
                    // if upstream projects exist try recompiling any failing upstream tests
                    if (isMultiModuleProject() && (!disableDependencyCompile || !recompileDependencies)) {
                        if (!compileFailingProjects(null, true, executor)) {
                            skipRunningTests = true;
                        }
                    }
                    debug("Recompiling Java test files: " + recompileJavaTests);
                    if (recompileJavaTest(recompileJavaTests, testArtifactPaths, executor, outputDirectory,
                            testOutputDirectory, projectName, buildFile, compilerOptions, skipUTs, skipRunningTests)) {
                        // successful compilation so we can clear failedCompilation list
                        failedCompilationJavaTests.clear();
                    } else {
                        failedCompilationJavaTests.addAll(recompileJavaTests);
                    }
                }
            }

            // run tests if files were deleted without any other changes, since
            // recompileJavaSource won't run (which normally handles tests)
            if (!deleteJavaSources.isEmpty() && recompileJavaSources.isEmpty()) {
                // run tests after waiting for app update since app changed
                runTestThread(true, executor, numApplicationUpdatedMessages, false, buildFile);
            } else if (processTests && !deleteJavaTests.isEmpty() && recompileJavaTests.isEmpty()) {
                // run all tests without waiting for app update since only tests changed
                runTestThread(false, executor, -1, false, buildFile);
            }

            deleteJavaSources.clear();
            recompileJavaSources.clear();
            triggerJavaTestRecompile = false;
            triggerJavaSourceRecompile = false;

            if (processTests) {
                deleteJavaTests.clear();
                recompileJavaTests.clear();
            }
            if (disableDependencyCompile) {
                disableDependencyCompile = false;
            }
            if (initialCompile) {
                debug("Setting initialCompile to false");
                initialCompile = false;
                if (hotTests) {
                    // if hot testing, run tests on startup
                    if (isMultiModuleProject()) {
                        // force run tests across all modules in multi module scenario
                        runTestThread(false, executor, -1, true, getAllBuildFiles());
                    } else {
                        runTestThread(false, executor, -1, false, buildFile);
                    }
                }
            }
        }
    }
 
    private void checkStopDevMode(boolean skipOnRestart) throws PluginScenarioException {
        // stop dev mode if the server has been stopped by another process
        if (serverThread == null || serverThread.getState().equals(Thread.State.TERMINATED)) {
            // server is restarting if devStop was set to true and we have not called the shutdown hook
            boolean restarting = devStop.get() && !calledShutdownHook.get();
            if (skipOnRestart && restarting && !externalContainerShutdown.get()) {
                debug("Server is restarting. Allowing dev mode to continue.");
                return;
            }
            if (!devStop.get()) {
                // an external situation caused the server to stop
                if (container) {
                    throw new PluginScenarioException("The container has stopped. Exiting dev mode.");
                }
                throw new PluginScenarioException("The server has stopped. Exiting dev mode.");
            } else {
                // server was stopped by dev mode
                throw new PluginScenarioException();
            }
        }
    }

    private void initWatchLoop() throws IOException {
        recompileJavaSources = new HashSet<File>();
        recompileJavaTests = new HashSet<File>();
        deleteJavaSources = new HashSet<File>();
        deleteJavaTests = new HashSet<File>();
        failedCompilationJavaSources = new HashSet<File>();
        failedCompilationJavaTests = new HashSet<File>();
        lastJavaSourceChange = System.currentTimeMillis();
        lastJavaTestChange = System.currentTimeMillis();
        triggerJavaSourceRecompile = false;
        triggerJavaTestRecompile = false;
        triggerUpstreamJavaSourceRecompile = false;
        lastBuildFileChange = new HashMap<File, Long>();
        modifiedClasses = new HashSet<File>();
        failedToGenerateClasses = new HashSet<File>();

        // initial source and test compile of upstream projects
        if (isMultiModuleProject()) {
            for (ProjectModule project : upstreamProjects) {
                // for ci.gradle, map will be null always
                // we can always trigger recompile for gradle, as we gradle is using gradle task state to recompile
                if (projectRecompileMap == null) {
                    info("Recompile " + project.getProjectName());
                    triggerUpstreamModuleCompile(project, false);
                    // build file tracking of upstream projects, this update is needed for auto test invocation
                    lastBuildFileChange.put(project.getBuildFile(), System.currentTimeMillis());
                } else if (Boolean.TRUE.equals(projectRecompileMap.get(project.getProjectName()))) {
                    info("Recompile " + project.getProjectName() + " due to an earlier compilation error");
                    triggerUpstreamModuleCompile(project, false);
                    // build file tracking of upstream projects, this update is needed for auto test invocation
                    lastBuildFileChange.put(project.getBuildFile(), System.currentTimeMillis());
                } else {
                    info("Recompile skipped for " + project.getProjectName() + " since earlier compilation is successful");
                }
            }
        }

        // for ci.gradle, map will be null always
        // we can always trigger recompile for gradle, as we gradle is using gradle task state to recompile
        if (projectRecompileMap == null) {
            info("Recompile " + getProjectName());
            triggerMainModuleCompile(false);
            // build file tracking of upstream projects, this update is needed for auto test invocation
            lastBuildFileChange.put(buildFile, System.currentTimeMillis());
        } else if (Boolean.TRUE.equals(projectRecompileMap.get(getProjectName()))) {
            info("Recompile " + getProjectName() + " due to an earlier compilation error");
            triggerMainModuleCompile(false);
            // build file tracking of upstream projects, this update is needed for auto test invocation
            lastBuildFileChange.put(buildFile, System.currentTimeMillis());
        } else {
            info("Recompile skipped for " + getProjectName() + " since earlier compilation is successful");
        }

    }

    private void processFileChanges(
        final ThreadPoolExecutor executor, File fileChanged, File outputDirectory,
        boolean isDirectory, ChangeType changeType) throws IOException, PluginExecutionException {

        if (ignoreFileOrDir(fileChanged)) {
            // skip this file or directory, and continue to the next file or directory
            return;
        }

        debug("Processing file changes for " + fileChanged + ", change type " + changeType);

        Path srcPath = this.sourceDirectory.getCanonicalFile().toPath();
        Path testSrcPath = this.testSourceDirectory.getCanonicalFile().toPath();
        Path configPath = this.configDirectory.getCanonicalFile().toPath();
        Path outputPath = this.outputDirectory.getCanonicalFile().toPath();
        Path gfTmpDirPath = this.generateFeaturesTmpDir.getCanonicalFile().toPath();

        Path directory = fileChanged.getParentFile().getCanonicalFile().toPath();

        // resource file check
        File resourceParent = null;
        for (File resourceDir : resourceDirs) {
            if (directory.startsWith(resourceDir.getCanonicalFile().toPath())) {
                resourceParent = resourceDir;
                break;
            }
        }
        
        // webResource file check
        Path webResourceParent = null;
        for (Path webResourceDir : monitoredWebResourceDirs) {
            if (directory.startsWith(webResourceDir)) {
                webResourceParent = webResourceDir;
                break;
            }
        }

        if (fileChanged.isDirectory()) {
            // if new directory added, watch the entire directory
            if (changeType == ChangeType.CREATE) {
                if (!isUpstreamSourceDir(fileChanged)) { // adding a src/main/java dir to an upstream project is not currently supported
                    registerAll(fileChanged.toPath(), executor);
                    // TODO process ALL files in newly registered directory, not just the generated features file https://github.com/OpenLiberty/ci.maven/issues/1548
                    // check if the generated features file exists in any of the newly registered directories
                    Iterator<File> it = FileUtils.iterateFiles(fileChanged,
                            new NameFileFilter(FeatureGeneratorUtil.GENERATED_FEATURES_FILE_NAME),
                            TrueFileFilter.INSTANCE);
                    if (it.hasNext()) {
                        File newlyRegisteredFile = it.next();
                        // confirm that the newly registered file is generated features file in configDropins/overrides
                        if (newlyRegisteredFile.equals(generateFeaturesFile)) {
                            // process file changes for the generated features file so that newly generated features are installed
                            debug("Registered configDropins/overrides directory, processing file changes for generated features file: "
                                    + newlyRegisteredFile);
                            processFileChanges(executor, newlyRegisteredFile, outputDirectory, false,
                                    ChangeType.CREATE);
                        }
                    }
                }
                
            }
            // otherwise if a directory was modified, just continue to the next entry
            // (if delete, can't tell if it was a directory since it doesn't exist anymore)
            return;
        }

        int numApplicationUpdatedMessages = countApplicationUpdatedMessages();

        // reset this property in case it had been set to true
        System.setProperty(SKIP_BETA_INSTALL_WARNING, Boolean.FALSE.toString());

        // parent build file changed
        if (!this.parentBuildFiles.isEmpty() && this.parentBuildFiles.containsKey(fileChanged.getCanonicalPath())
                && changeType == ChangeType.MODIFY) {
            debug("Change detected in parent build file: " + fileChanged + ". Updating compile artifact paths.");
            updateArtifactPaths(fileChanged);
            // compile child modules
            if (recompileDependencies) {
                // upstream projects are in build order
                // search for first upstream project that is a child module
                if (isMultiModuleProject()) {
                    ProjectModule project = getFirstProjectModule(fileChanged);
                    if (project != null) {
                        triggerUpstreamModuleCompile(project, false);
                    } else {
                        triggerMainModuleCompile(false);
                    }
                } else {
                    triggerMainModuleCompile(false);
                }
            } else {
                // trigger recompile of failing source and test
                compileFailingProjects(null, false, executor);
            }
        }

        // upstream project source file changed
        if (this.upstreamProjects != null && !this.upstreamProjects.isEmpty()) {
            for (ProjectModule project : this.upstreamProjects) {
                // resource file check
                File upstreamResourceParent = null;
                for (File resourceDir : project.getResourceDirs()) {
                    if (directory.startsWith(resourceDir.getCanonicalFile().toPath())) {
                        upstreamResourceParent = resourceDir;
                        break;
                    }
                }

                // src/main/java directory
                if (directory.startsWith(project.getSourceDirectory().getCanonicalPath())) {
                    if (fileChanged.exists() && fileChanged.getName().endsWith(".java")
                            && (changeType == ChangeType.MODIFY || changeType == ChangeType.CREATE)) {
                        debug("Java source file modified: " + fileChanged.getName()
                                + ". Adding to list for processing.");
                        lastJavaSourceChange = System.currentTimeMillis();
                        if (recompileDependencies) {
                            if (generateFeatures) {
                                // When recompileDependencies=true, we now check for source code changes and not class file changes
                                // We track what module each change occurred in and generate features for that entire module once compilation for that module is complete
                                // set modifiedSrcBuildFile to know what module to generate features for
                                modifiedSrcBuildFile = project.getBuildFile();
                                debug("Multi-module - setting modifiedSrcBuildFile to: " + modifiedSrcBuildFile);
                                ProjectModule modifiedModule = getProjectModule(modifiedSrcBuildFile);
                                File outputDir = modifiedModule.getOutputDirectory();
                                // Add output dir for processing generate features
                                if (outputDir != null) {
                                    modifiedClasses.add(outputDir);
                                }
                                // New src file change, reset lastChangeCompiled
                                lastChangeCompiled = false;
                            }
                            triggerUpstreamModuleCompile(project, false);
                        } else {
                            project.recompileJavaSources.add(fileChanged);
                        }
                    } else if (changeType == ChangeType.DELETE) {
                        debug("Java file deleted: " + fileChanged.getName() + ". Adding to list for processing.");
                        lastJavaSourceChange = System.currentTimeMillis();
                        project.deleteJavaSources.add(fileChanged);
                        if (recompileDependencies) {
                            triggerUpstreamModuleCompile(project, false);
                        }
                    }
                } else if (directory.startsWith(project.getTestSourceDirectory().getCanonicalPath())) {
                    if (fileChanged.exists() && fileChanged.getName().endsWith(".java")
                            && (changeType == ChangeType.MODIFY || changeType == ChangeType.CREATE)) {
                        debug("Java test file modified: " + fileChanged.getName() + ". Adding to list for processing.");
                        lastJavaTestChange = System.currentTimeMillis();
                        if (recompileDependencies) {
                            triggerUpstreamModuleCompile(project, true);
                        } else {
                            project.recompileJavaTests.add(fileChanged);
                        }
                    } else if (changeType == ChangeType.DELETE) {
                        debug("Java test file deleted: " + fileChanged.getName() + ". Adding to list for processing.");
                        lastJavaTestChange = System.currentTimeMillis();
                        project.deleteJavaTests.add(fileChanged);
                        if (recompileDependencies) {
                            triggerUpstreamModuleCompile(project, true);
                        }
                    }
                } else if (!recompileDependencies && generateFeatures && directory.startsWith(project.getOutputDirectory().getCanonicalPath())) {
                    // track Java source class files if generateFeatures=true && recompileDependencies=false
                    // When recompileDependencies=true and class files are tracked, there are too many recompiled class files and feature generation runs too often
                    // Therefore, we only track class files if recompileDependencies=false
                    if (fileChanged.exists() && fileChanged.getName().endsWith(".class")
                            && (changeType == ChangeType.MODIFY || changeType == ChangeType.CREATE)) {
                        debug("Java source class file modified: " + fileChanged.getName()
                                + ". Adding to list for processing.");
                        modifiedClasses.add(fileChanged);
                    } else if (changeType == ChangeType.DELETE) {
                        debug("Java source class deleted: " + fileChanged.getName()
                                + ". Adding to list for processing.");
                        modifiedClasses.remove(fileChanged); // remove if class file is already stored in list
                    }
                } else if (fileChanged.equals(project.getBuildFile())
                        && directory.startsWith(project.getBuildFile().getParentFile().getCanonicalFile().toPath())
                        && changeType == ChangeType.MODIFY) { // pom.xml
                    debug("Change detected in: " + project.getBuildFile() + ". Updating compile artifact paths.");
                    lastBuildFileChange.put(project.getBuildFile(), System.currentTimeMillis());
                    // when an upstream project build file changes, get the updated artifact paths
                    boolean updatedArtifactPaths = updateArtifactPaths(project, true, generateFeatures, executor);
                    if (updatedArtifactPaths) {
                        if (recompileDependencies) {
                            // recompile the entire module
                            if (!project.getSourceDirectory().exists() && !project.getTestSourceDirectory().exists()
                                    && !project.getDependentModules().isEmpty()) {
                                // parent project, trigger compile on first dependent module
                                triggerUpstreamModuleCompile(getProjectModule(project.getDependentModules().get(0)),
                                        false);
                            } else {
                                triggerUpstreamModuleCompile(project, false);
                            }
                        } else {
                            // trigger java source recompile of all projects if there are compilation errors
                            // in this project
                            if (!project.failedCompilationJavaSources.isEmpty()) {
                                triggerUpstreamJavaSourceRecompile = true;
                            }
                            // trigger java test recompile of just this project if there are existing test
                            // compilation errors
                            if (!project.failedCompilationJavaTests.isEmpty()) {
                                project.triggerJavaTestRecompile = true;
                            }
                        }
                    }
                } else if (upstreamResourceParent != null
                        && directory.startsWith(upstreamResourceParent.getCanonicalFile().toPath())) { // resources
                    debug("Resource dir: " + upstreamResourceParent.toString());
                    if (fileChanged.exists() && (changeType == ChangeType.MODIFY || changeType == ChangeType.CREATE)) {
                        copyFile(fileChanged, upstreamResourceParent, project.getOutputDirectory(), null);

                        // run all tests on resource change
                        runTestThread(true, executor, numApplicationUpdatedMessages, false,
                                getAllBuildFiles(project));
                    } else if (changeType == ChangeType.DELETE) {
                        debug("Resource file deleted: " + fileChanged.getName());
                        deleteFile(fileChanged, upstreamResourceParent, project.getOutputDirectory(), null);

                        // run all tests on resource change
                        runTestThread(true, executor, numApplicationUpdatedMessages, false, getAllBuildFiles(project));
                    }
                }
            }
        }

        if (directory.startsWith(srcPath)) {
            ArrayList<File> javaFilesChanged = new ArrayList<File>();
            javaFilesChanged.add(fileChanged);
            if (fileChanged.exists() && fileChanged.getName().endsWith(".java")
                    && (changeType == ChangeType.MODIFY
                            || changeType == ChangeType.CREATE)) {
                debug("Java source file modified: " + fileChanged.getName()
                        + ". Adding to list for processing.");
                lastJavaSourceChange = System.currentTimeMillis();
                if (recompileDependencies) {
                    if (generateFeatures) {
                        // When recompileDependencies=true, we now check for source code changes and not class file changes
                        // We track what module each change occurred in and generate features for that entire module once compilation for that module is complete
                        // set modifiedSrcBuildFile to know what module to generate features for
                        modifiedSrcBuildFile = buildFile;
                        debug("Single module - setting modifiedSrcBuildFile to: " + modifiedSrcBuildFile);
                        // Add output dir for processing generate features
                        if (outputDirectory != null) {
                            modifiedClasses.add(outputDirectory);
                        }
                        // New src file change, reset lastChangeCompiled
                        lastChangeCompiled = false;
                    }
                    triggerMainModuleCompile(false);
                } else {
                    recompileJavaSources.add(fileChanged);
                }
            } else if (changeType == ChangeType.DELETE) {
                debug("Java file deleted: " + fileChanged.getName()
                        + ". Adding to list for processing.");
                lastJavaSourceChange = System.currentTimeMillis();
                deleteJavaSources.add(fileChanged);
                if (recompileDependencies) {
                    triggerMainModuleCompile(false);
                }
            }
        } else if (directory.startsWith(testSrcPath)) { // src/test/java
            ArrayList<File> javaFilesChanged = new ArrayList<File>();
            javaFilesChanged.add(fileChanged);
            if (fileChanged.exists() && fileChanged.getName().endsWith(".java")
                    && (changeType == ChangeType.MODIFY
                            || changeType == ChangeType.CREATE)) {
                debug("Java test file modified: " + fileChanged.getName()
                        + ". Adding to list for processing.");
                lastJavaTestChange = System.currentTimeMillis();
                if (recompileDependencies) {
                    triggerMainModuleCompile(true);
                } else {
                    recompileJavaTests.add(fileChanged);
                }
            } else if (changeType == ChangeType.DELETE) {
                debug("Java test file deleted: " + fileChanged.getName()
                        + ". Adding to list for processing.");
                lastJavaTestChange = System.currentTimeMillis();
                deleteJavaTests.add(fileChanged);
                if (recompileDependencies) {
                    triggerMainModuleCompile(true);
                }
            }
        } else if (serverXmlFileParent != null
                && directory.equals(serverXmlFileParent.getCanonicalFile().toPath())
                && fileChanged.getCanonicalPath().endsWith(serverXmlFile.getName())) {
            // This is for server.xml specified by the configuration parameter
            // server will load new properties
            processConfigFileChange(fileChanged, changeType, executor, numApplicationUpdatedMessages, true);
        } else if ((directory.startsWith(configPath)
                || directory.startsWith(gfTmpDirPath))
                && !isGeneratedConfigFile(fileChanged, configDirectory, serverDirectory)) {
            // configuration file or generate-features.xml in temp directory
            processConfigFileChange(fileChanged, changeType, executor, numApplicationUpdatedMessages, false);
        } else if (bootstrapPropertiesFileParent != null
                   && directory.equals(bootstrapPropertiesFileParent.getCanonicalFile().toPath())
                   && fileChanged.getCanonicalPath().endsWith(bootstrapPropertiesFile.getName())) {
            // This is for bootstrap.properties outside of the config folder
            // restart server to load new properties
            if (isContainerfileDirectoryChanged(fileChanged)) {
                untrackContainerfileDirectoriesAndRestart();
            } else {
                restartServer(false);
            }
        } else if (jvmOptionsFileParent != null
                && directory.equals(jvmOptionsFileParent.getCanonicalFile().toPath())
                && fileChanged.getCanonicalPath().endsWith(jvmOptionsFile.getName())) {
            // This is for jvm.options outside of the config folder
            // restart server to load new options
            if (isContainerfileDirectoryChanged(fileChanged)) {
                untrackContainerfileDirectoriesAndRestart();
            } else {
                restartServer(false);
            }
        } else if (resourceParent != null
                && directory.startsWith(resourceParent.getCanonicalFile().toPath())) { // resources
            debug("Resource dir: " + resourceParent.toString());
            if (fileChanged.exists() && (changeType == ChangeType.MODIFY
                    || changeType == ChangeType.CREATE)) {
            	resourceModifiedOrCreated(fileChanged, resourceParent, outputDirectory);

                // run all tests on resource change
                runTestThread(true, executor, numApplicationUpdatedMessages, skipUTs, false, buildFile);
            } else if (changeType == ChangeType.DELETE) {
                debug("Resource file deleted: " + fileChanged.getName());
                resourceDeleted(fileChanged, resourceParent, outputDirectory);
                // run all tests on resource change
                runTestThread(true, executor, numApplicationUpdatedMessages, skipUTs, false, buildFile);
            }
        } else if (webResourceParent != null && directory.startsWith(webResourceParent)) { // webResources
                debug("webResource dir: " + webResourceParent.toString());
                updateLooseApp();
                runTestThread(true, executor, numApplicationUpdatedMessages, false, false);
        } else if (fileChanged.equals(buildFile)
                && directory.startsWith(buildFile.getParentFile().getCanonicalFile().toPath())
                && changeType == ChangeType.MODIFY) { // pom.xml
            lastBuildFileChange.put(buildFile, System.currentTimeMillis());
            boolean recompiledBuild = recompileBuildFile(buildFile, compileArtifactPaths, testArtifactPaths, generateFeatures, executor);
            
            // run all tests on build file change
            if (recompiledBuild) {
                if (recompileDependencies) {
                    triggerMainModuleCompile(false);
                } else {
                    // trigger java source recompile if there are compilation errors
                    if (!failedCompilationJavaSources.isEmpty()) {
                        triggerJavaSourceRecompile = true;
                    }
                    // trigger java test recompile if there are compilation errors
                    if (!failedCompilationJavaTests.isEmpty()) {
                        triggerJavaTestRecompile = true;
                    }
                }
                runTestThread(true, executor, numApplicationUpdatedMessages, skipUTs, false, buildFile);
            }
        } else if (fileChanged.equals(containerfileUsed)
                && directory.startsWith(containerfileUsed.getParentFile().getCanonicalFile().toPath())
                && changeType == ChangeType.MODIFY) { // containerfile
            untrackContainerfileDirectoriesAndRestart(); // untrack all Containerfile directories, then rebuild container and restart
        } else if (propertyFilesMap != null && propertyFilesMap.keySet().contains(fileChanged)) { // properties file
            boolean reloadedPropertyFile = reloadPropertyFile(fileChanged);
            // run all tests on properties file change
            if (reloadedPropertyFile) {
                runTestThread(true, executor, numApplicationUpdatedMessages, skipUTs, false, buildFile);
            }
        } else if (isContainerfileDirectoryChanged(fileChanged)) {
            // If contents within a directory specified in a Containerfile COPY command were changed, and not already processed by one of the other conditions above.
            untrackContainerfileDirectoriesAndRestart();
        } else if (!recompileDependencies && generateFeatures && directory.startsWith(outputPath)) {
            // track Java source class files if generateFeatures=true && recompileDependencies=false
            // When recompileDependencies=true and class files are tracked, there are too many recompiled class files and feature generation runs too often
            // Therefore, we only track class files if recompileDependencies=false
            if (fileChanged.exists() && fileChanged.getName().endsWith(".class")
                    && (changeType == ChangeType.MODIFY
                            || changeType == ChangeType.CREATE)) {
                debug("Java source class file modified: " + fileChanged.getName()
                        + ". Adding to list for processing.");
                modifiedClasses.add(fileChanged);
            } else if (changeType == ChangeType.DELETE) {
                debug("Java source class deleted: " + fileChanged.getName() + ". Adding to list for processing.");
                modifiedClasses.remove(fileChanged); // remove if class file is already stored in list
            }
        }
    }

    /**
     * Process a configuration file change.
     * 
     * Typical sequence of events when config file is created/modified:
     * 1. Generate features
     * 2. Install features if features were modified
     * 3. Copy fileChanged (and generated features file if modified in step 1) to
     * server directory
     * 4. Update existing features list if features were modified
     * 5. Redeploy or restart server if needed (devc or server properties modified)
     * 
     * Typical sequence of events when config file is deleted:
     * 1. Delete corresponding configuration file from server directory
     * 2. Generate features
     * 3. Restart server if needed (devc or Linux)
     * 
     * @param fileChanged                   modified file
     * @param changeType                    file change event CREATE, MODIFY or
     *                                      DELETE
     * @param executor                      ThreadPoolExecutor for running tests
     * @param numApplicationUpdatedMessages number of application updated
     *                                      messages, used when running tests
     * @param configuredServerXml           true if fileChanged is serverXmlFile
     *                                      specified by the configuration parameter
     * @throws IOException
     * @throws PluginExecutionException
     */
    private void processConfigFileChange(File fileChanged, ChangeType changeType, ThreadPoolExecutor executor,
            int numApplicationUpdatedMessages, boolean configuredServerXml)
            throws IOException, PluginExecutionException {
        boolean isGeneratedFeaturesFile = configuredServerXml ? false : fileChanged.equals(generateFeaturesFile);
        String targetFileName = configuredServerXml ? "server.xml" : null; // if null file will retain the same name when copied
        // three possible values for the parent directory
        File fileChangedParentDir = configuredServerXml ? serverXmlFileParent :
            isGeneratedFeaturesFile ? generateFeaturesOutputDir : configDirectory;

        if (fileChanged.exists() && (changeType == ChangeType.MODIFY || changeType == ChangeType.CREATE)) {
            debug("Config file modified: " + fileChanged);
            boolean generateFeaturesSuccess = true; // default to true in case feature generation is disabled
            boolean serverFeaturesModified = serverFeaturesModified();

            // generate features whenever features have changed and an XML file is modified,
            // excluding the generated features file
            // if generateToSrc is false then we must copy generated file to serverDir after install to temp
            if (generateFeatures && (fileChanged.getName().endsWith(".xml")
                    && !isGeneratedFeaturesFile)
                    && serverFeaturesModified) {
                // If generating to server dir we use the server dir config files and also the
                // modified xml file in src dir. Copy them all to the gen. feat. temp dir to
                // combine them for feature generation.
                if (!generateToSrc) {
                    // Deleting generateFeaturesTmpDir also deletes generateFeaturesFile which we "watch" in
                    // dev mode. This causes a deletion event and we are counting on the handler (this method,
                    // below) not to call generate features and recreate the file.
                    cleanUpTempConfig(generateFeaturesTmpDir.toPath());
                    // copy config files to temp dir
                    copyToTempDir(serverDirectory, generateFeaturesTmpDir);
                    // copy changed file to temp dir
                    copyFile(fileChanged, fileChangedParentDir, generateFeaturesTmpDir, targetFileName);
                }
                generateFeaturesSuccess = optimizeGenerateFeatures(!generateToSrc, !generateToSrc);
            }
            if (serverFeaturesModified) {
                // suppress install feature warning - property must be set before installing using temp dir
                System.setProperty(SKIP_BETA_INSTALL_WARNING, Boolean.TRUE.toString());
                installFeaturesToTempDir(fileChanged, fileChangedParentDir, targetFileName, generateFeaturesSuccess);
            }
            // Copy the config file which was changed to the server directory unless it is
            // the generated features file. The generated features file may have been copied 
            // to the server directory already as a result of a change to the build file (pom.xml)
            // or the server.xml.
            if (!isGeneratedFeaturesFile) { // all other config files
                copyFile(fileChanged, fileChangedParentDir, serverDirectory, targetFileName);
            } else {
                if (!generatedFeaturesCopied) {
                    copyGeneratedFeaturesFile(serverDirectory);
                }
                generatedFeaturesCopied = false;
            }
            // If the generated features file was modified as a result of another config file modification
            // (usually server.xml), copy it over to target so the server picks up the two changes together
            if (generateFeaturesSuccess && generatedFeaturesModified && !isGeneratedFeaturesFile) {
                // This logic is not entered if the fileChanged is the generated features file.
                // Copy generated features file to server dir and set generatedFeaturesCopied true
                // (not referring to generatedFeaturesModified).
                // Leave generatedFeaturesCopied true because the call to optimize/incrementalGenerateFeatures that was
                // required to get into this IF block will also generate a file change event for generatedFeaturesFile
                copyGeneratedFeaturesFile(serverDirectory);
                generatedFeaturesModified = false;
            }
            if (serverFeaturesModified) {
                updateExistingFeatures();
            }

            if (isContainerfileDirectoryChanged(serverDirectory, fileChanged)) {
                untrackContainerfileDirectoriesAndRestart();
            } else {
                if (changeType == ChangeType.CREATE) {
                    redeployApp();
                }
                if (fileChanged.getName().equals("server.env")) {
                    // re-enable debug variables in server.env
                    enableServerDebug(false);
                } else if ((fileChanged.getName().equals("bootstrap.properties")
                        && bootstrapPropertiesFileParent == null)
                        || (fileChanged.getName().equals("jvm.options") && jvmOptionsFileParent == null)) {
                    // restart server to load new properties
                    restartServer(false);
                }
            }
            if (isGeneratedFeaturesFile && generateFeatures) {
                // if generateFeatures is true, run UTs and ITs as tests would have been skipped
                // during recompileJava()
                if (isMultiModuleProject()) {
                    runTestThread(true, executor, numApplicationUpdatedMessages, false, getAllBuildFiles());
                } else {
                    runTestThread(true, executor, numApplicationUpdatedMessages, false, false, buildFile);
                }
            } else {
                // always skip UTs
                runTestThread(true, executor, numApplicationUpdatedMessages, true, false, buildFile);
            }
        } else if (changeType == ChangeType.DELETE) {
            info("Config file deleted: " + fileChanged.getName());
            deleteFile(fileChanged, fileChangedParentDir, serverDirectory, targetFileName);
            // generate features whenever features have changed and an XML file is deleted,
            // excluding the generated-features.xml file. This is important also when we delete the
            // generateFeaturesTmpDir in the process of handling an xml config modicifcation.
            // Deleting that directory could cause generated-features.xml to be deleted and we
            // need to be careful how to handle that event e.g. don't call optimizeGenerateFeatures().
            // Another scenario, when we toggle generateToSrc option we delete the old file and change the
            // value of generateFeaturesFile. Therefore we must only use the base name of generateFeaturesFile
            // in this file name check.
            if (generateFeatures && (fileChanged.getName().endsWith(".xml")
                    && !fileChanged.getName().equals(generateFeaturesFile.getName()))
                    && serverFeaturesModified()) {
                optimizeGenerateFeatures(!generateToSrc);
            }
            // Let this restart if needed for container mode.  Otherwise, nothing else needs to be done for config file delete.
            if (isContainerfileDirectoryChanged(serverDirectory, fileChanged)) {
                untrackContainerfileDirectoriesAndRestart();
            } else {
                if (fileChanged.getName().equals("server.env")) {
                    // re-enable debug variables in server.env
                    enableServerDebug(false);
                }
                if (container && OSUtil.isLinux()) {
                    info("Restarting the container for this change to take effect.");
                    // Allow a 1 second grace period to replace the file in case the user changes the file with a script or a tool like vim.
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                        debug("Unexpected InterruptedException handling config file deletion.", e);
                    }
                    restartServer(false);
                }    
            }
            // always skip UTs
            runTestThread(true, executor, numApplicationUpdatedMessages, true, false, buildFile);
        }
    }

    // given the parent build file changed, return the first dependent project in
    // the reactor build
    private ProjectModule getFirstProjectModule(File buildFileChanged) throws IOException {
        for (ProjectModule project : this.upstreamProjects) {
            ProjectModule match = getFirstProjectModule(project, buildFileChanged);
            if (match != null) {
                return match;
            }
        }
        return null;
    }

    private ProjectModule getFirstProjectModule(ProjectModule project, File buildFileChanged) throws IOException {
        List<String> childBuildFiles = this.parentBuildFiles.get(buildFileChanged.getCanonicalPath());
        for (String childBuildPath : childBuildFiles) {
            if (childBuildPath.equals(project.getBuildFile().getCanonicalPath())) {
                // found match
                return project;
            } else if (this.parentBuildFiles.containsKey(childBuildPath)) {
                return getFirstProjectModule(project, new File(childBuildPath));
            }
        }
        return null;
    }
    
    protected abstract void resourceDirectoryCreated() throws IOException;

    protected abstract void resourceModifiedOrCreated(File fileChanged, File resourceParent, File outputDirectory) throws IOException;

    protected abstract void resourceDeleted(File fileChanged, File resourceParent, File outputDirectory) throws IOException;

    /**
     * Unwatches all directories that were specified in Containerfile COPY commands, then does a container
     * rebuild and restart.
     * 
     * @throws PluginExecutionException
     */
    private void untrackContainerfileDirectoriesAndRestart() throws PluginExecutionException {
        // Cancel and clear any WatchKeys that were added for to the Containerfile directories
        for (WatchKey key : containerfileDirectoriesWatchKeys) {
            key.cancel();
        }
        containerfileDirectoriesWatchKeys.clear();

        // Cancel and clear any FileAlterationObservers that were added for the Containerfile directories
        synchronized (cancelledFileObservers) {
            for (FileAlterationObserver observer : containerfileDirectoriesFileObservers) {
                // add the observer to be cancelled 
                cancelledFileObservers.add(observer);
                try {
                    // destroy the observer
                    observer.destroy();
                } catch (Exception e) {
                    debug("Could not destroy file observer", e);
                }
            }
        }
        containerfileDirectoriesFileObservers.clear();

        // Untrack the directories
        containerfileDirectoriesTracked.clear();
        restartServer(true);
    }

    /**
     * If container mode, check if any of the files are within a directory specified in one of the Containerfile's
     * COPY commands.  If not container mode, does nothing.
     * 
     * @param files The files to check, in the same order.
     * @return true if container mode and any of the files are within a directory specified in one of the Containerfile's COPY commands.
     * @throws IOException if there was an error getting canonical paths
     */
    private boolean isContainerfileDirectoryChanged(File... files) throws IOException {
        // Check for directory content changes from directories specified in Containerfile
        if (container && !containerfileDirectoriesTracked.isEmpty()) {
            for (Path trackedPath : containerfileDirectoriesTracked) {
                Path logsPath = new File(serverDirectory, "logs").getCanonicalFile().toPath();

                for (File file : files) {
                    // if the file's path is a child of the tracked path, except for the server logs folder or if it's the loose application itself
                    Path filePath = file.getCanonicalFile().toPath();
                    if (filePath.startsWith(trackedPath) && !filePath.startsWith(logsPath) && !filePath.toString().endsWith(".war.xml") && !filePath.toString().endsWith(".ear.xml")) {
                        debug("isContainerfileDirectoryChanged=true for directory " + trackedPath + " with file " + file);
                        return true;
                    }
                }
            }
        }
        return false;
    }

    /**
     * Determines if the specified config file in target dir was generated by a Liberty
     * plugin: bootstrap.properties or jvm.options
     * 
     * @param fileChanged the file that was changed
     * @param srcDir      the directory of the file changed
     * @param targetDir   the target directory
     * @throws IOException unable to resolve canonical path
     */
    protected boolean isGeneratedConfigFile(File fileChanged, File srcDir, File targetDir) throws IOException {
        return (fileChanged.getName().equals("bootstrap.properties") || fileChanged.getName().equals("jvm.options"))
                && isGeneratedTargetFile(fileChanged, srcDir, targetDir);
    }

    private boolean isGeneratedTargetFile(File fileChanged, File srcDir, File targetDir) throws IOException {
        File targetFile = getTargetFile(fileChanged, srcDir, targetDir, null);
        try (FileReader fileReader = new FileReader(targetFile);
                BufferedReader bufferedReader = new BufferedReader(fileReader)) {
            String line = bufferedReader.readLine();
            return line == null ? false : line.matches(GENERATED_HEADER_REGEX);
        } catch (IOException e) {
            // If the target file could not be read, assume it was not a generated file
            debug("Could not read the target file " + targetFile + ". It will be replaced by the contents of "
                    + fileChanged, e);
        }
        return false;
    }

    /**
     * Reads the file to a String
     * 
     * @param file
     * @return String representation of the file
     * @throws IOException unable to read file to string
     */
    public String readFile(File file) throws IOException {
        return FileUtils.readFileToString(file, StandardCharsets.UTF_8);
    }

    /**
     * Creates a temporary copy of the configuration file and checks the configFile
     * in the temporary directory to avoid install-feature timing issues. Installs
     * features if new features are detected.
     * 
     * @param fileChanged             the file that was changed
     * @param srcDir                  the directory of the file changed
     * @param targetFileName          if not null renames the fileChanged to
     *                                targetFileName in the targetDir
     * @param generateFeaturesSuccess if features were successfully generated, skip
     *                                install features if false. Defaults to true if
     *                                generateFeatures is off
     * @throws IOException creating and copying to tempConfig directory
     */
    public void installFeaturesToTempDir(File fileChanged, File srcDir, String targetFileName, boolean generateFeaturesSuccess) throws IOException {
        if (generateFeatures && !generateFeaturesSuccess) {
            return; // skip creating temp dir and installing features if feature generation failed
        }
        this.tempConfigPath = Files.createTempDirectory("tempConfig");
        File tempConfig = tempConfigPath.toFile();
        debug("Temporary configuration folder created: " + tempConfig);

        copyToTempDir(serverDirectory, tempConfig);
        copyFile(fileChanged, srcDir, tempConfig, targetFileName);
        if (generateFeatures && generateFeaturesSuccess && !fileChanged.equals(generateFeaturesFile)) {
            copyGeneratedFeaturesFile(tempConfig);
        }
        installFeatures(fileChanged, tempConfig, generateFeatures);
        cleanUpTempConfig(this.tempConfigPath);
    }

    /**
     * Copy the liberty config in the sourceDir directory into the supplied temp directory.
     * Filter out certain directories used in Liberty configuration: workarea, logs, messaging
     * and also the files dev mode usually ignores e.g. .dir, .file, xxx.dmp etc
     * 
     * @param sourceDir  copy files from this directory
     * @param tempDir    target directory to which files are copied
     */
    public File copyToTempDir(File sourceDir, File tempConfig) throws IOException {
        FileUtils.copyDirectory(sourceDir, tempConfig, new FileFilter() {
            public boolean accept(File pathname) {
                String name = pathname.getName();
                String parent = pathname.getParentFile().getName();
                String sourceDirName = sourceDir.getName();
                // skip:
                // - ignore list
                // - workarea, messaging, and logs dirs from the server directory, since those can be
                // changing
                boolean skip = ignoreFileOrDir(pathname) || (pathname.isDirectory() && 
                    (name.equals("workarea") || name.equals("logs") || (name.equals("messaging") && parent.equals(sourceDirName))));
                return !skip;
            }
        }, true);

        return tempConfig;
    }

    /**
     * Whether dev mode should ignore a file or directory.
     * 
     * @param file File or directory
     * @return true if the file or directory should be ignored, false otherwise
     */
    private boolean ignoreFileOrDir(File file) {
        String name = file.getName();
        if (file.isDirectory()) {
            for (String prefix : IGNORE_DIRECTORY_PREFIXES) {
                if (name.startsWith(prefix)) {
                    debug("Ignoring " + name);
                    return true;
                }
            }
        } else {
            for (String prefix : IGNORE_FILE_PREFIXES) {
                if (name.startsWith(prefix)) {
                    debug("Ignoring " + name);
                    return true;
                }
            }
            for (String postfix : IGNORE_FILE_POSTFIXES) {
                if (name.endsWith(postfix)) {
                    debug("Ignoring " + name);
                    return true;
                }
            }
        }
        return false;
    }

    /**
     * Copies the fileChanged from the srcDir to the targetDir.
     * 
     * @param fileChanged    the file that was changed
     * @param srcDir         the directory of the file changed
     * @param targetDir      the target directory
     * @param targetFileName if not null renames the fileChanged to targetFileName
     *                       in the targetDir
     * @throws IOException unable to resolve canonical path
     */
    public void copyFile(File fileChanged, File srcDir, File targetDir, String targetFileName) throws IOException {
        File targetResource = getTargetFile(fileChanged, srcDir, targetDir, targetFileName);

        try {
            FileUtils.copyFile(fileChanged, targetResource);
            info("Copied file: " + fileChanged.getCanonicalPath() + " to: " + targetResource.getCanonicalPath());
        } catch (FileNotFoundException ex) {
            debug("Failed to copy file: " + fileChanged.getCanonicalPath());
        } catch (Exception ex) {
            debug(ex);
        }
    }

    private File getTargetFile(File fileChanged, File srcDir, File targetDir, String targetFileName)
            throws IOException {
        int srcDirIndex = fileChanged.getCanonicalPath().indexOf(srcDir.getCanonicalPath());
        if (srcDirIndex < 0 ) { // warn plugin developer this input is invalid, should not happen in production
            new IllegalArgumentException("fileChanged " + fileChanged + " is not in srcDir " + srcDir).printStackTrace();
        }
        String relPath = fileChanged.getCanonicalPath().substring(srcDirIndex + srcDir.getCanonicalPath().length());
        if (targetFileName != null) {
            relPath = relPath.substring(0, relPath.indexOf(fileChanged.getName())) + targetFileName;
        }
        File targetResource = new File(targetDir.getCanonicalPath() + relPath);
        return targetResource;
    }

    /**
     * Deletes the corresponding file in the targetDir.
     * 
     * @param deletedFile    the file that was deleted
     * @param dir            the directory of the deletedFile
     * @param targetDir      the corresponding targetDir of the deletedFile
     * @param targetFileName if not null deletes the targetFile with this name
     * @throws IOException unable to resolve canonical path
     */
    protected void deleteFile(File deletedFile, File dir, File targetDir, String targetFileName) throws IOException {
        File targetFile = getTargetFile(deletedFile, dir, targetDir, targetFileName);
        if (targetFile.exists()) {
            if (targetFile.isDirectory()) {
                try {
                    FileUtils.deleteDirectory(targetFile);
                    info("The directory " + targetFile.getCanonicalPath() + " was deleted.");
                } catch (IllegalArgumentException e) {
                    debug("Could not delete the directory " + targetFile.getCanonicalPath() + ". " + e.getMessage());
                } catch (IOException e) {
                    error("An error encountered while deleting the directory " + targetFile.getCanonicalPath()
                            + ". " + e.getMessage());
                }
            } else {
                if (targetFile.delete()) {
                    info("The file " + targetFile.getCanonicalPath() + " was deleted.");
                } else {
                    error("Could not delete the file " + targetFile.getCanonicalPath() + ".");
                }
            }
        }
    }

    /**
     * Delete all the Java class files within the specified directory. If the
     * directory is empty, deletes the directory as well.
     * 
     * @param outputDirectory the directory for compiled classes
     */
    protected void cleanTargetDir(File outputDirectory) {
        File[] fList = outputDirectory.listFiles();
        if (fList != null) {
            for (File file : fList) {
                if (file.isFile() && file.getName().toLowerCase().endsWith(".class")) {
                    file.delete();
                    info("Deleted Java class file: " + file);
                } else if (file.isDirectory()) {
                    cleanTargetDir(file);
                }
            }
        }
        if (outputDirectory.listFiles().length == 0) {
            outputDirectory.delete();
        }
    }

    private enum ChangeType {
        CREATE,
        DELETE,
        MODIFY
    };

    /**
     * Register the parent directory and all sub-directories with the WatchService
     * 
     * @param start   parent directory
     * @param executor the test thread executor
     * @throws IOException unable to walk through file tree
     */
    protected void registerAll(final Path start, final ThreadPoolExecutor executor) throws IOException {
        registerAll(start, executor, false);
    }

    /**
     * Register the parent directory and all sub-directories with the WatchService
     * 
     * @param start   parent directory
     * @param executor the test thread executor
     * @param removeOnContainerRebuild whether the files should be unwatched if the container is rebuilt
     * @throws IOException unable to walk through file tree
     */
    protected void registerAll(final Path start, final ThreadPoolExecutor executor,
            final boolean removeOnContainerRebuild) throws IOException {

        debug("Registering all files in directory: " + start.toString());

        // register directory and sub-directories
        Files.walkFileTree(start, new SimpleFileVisitor<Path>() {

            @Override
            public FileVisitResult preVisitDirectory(final Path dir, BasicFileAttributes attrs) throws IOException {
                if (trackingMode == FileTrackMode.POLLING || trackingMode == FileTrackMode.NOT_SET) {
                    // synchronize on the new observer set since only those are being updated in separate threads
                    synchronized (newFileObservers) {
                        Set<FileAlterationObserver> tempCombinedObservers = new HashSet<FileAlterationObserver>();
                        tempCombinedObservers.addAll(fileObservers);
                        tempCombinedObservers.addAll(newFileObservers);
        
                        // if this path is already observed, ignore it
                        for (FileAlterationObserver observer : tempCombinedObservers) {
                            if (dir.equals(observer.getDirectory().getCanonicalFile().toPath())) {
                                debug("Skipping subdirectory " + dir.toString() + " since it already being observed");
                                return FileVisitResult.CONTINUE;
                            }
                        }
                        for (File omitFile : omitWatchingFiles) {
                            if (dir.startsWith(omitFile.getCanonicalPath() + File.separator)) {
                                debug("Skipping subdirectory " + dir.toString() + " since it is in the omit files list");
                                return FileVisitResult.CONTINUE;
                            }
                        }
                        FileFilter singleDirectoryFilter = new FileFilter() {
                            @Override
                            public boolean accept(File file) {
                                try {
                                    // if it's a direct child of this path
                                    if (dir.equals(file.getParentFile().getCanonicalFile().toPath())) {
                                        return true;
                                    }
                                } catch (IOException e) {
                                    return false;
                                }
                                return false;
                            }
                        };
            
                        try {
                            debug("Adding subdirectory to file observers: " + dir.toString());
                            FileAlterationObserver observer = addFileAlterationObserver(executor, dir.toString(), singleDirectoryFilter);
                            if (removeOnContainerRebuild) {
                                debug("Adding to containerfileDirectoriesFileObservers: " + dir);
                                containerfileDirectoriesFileObservers.add(observer);
                            }
                        } catch (Exception e) {
                            error("Could not observe directory " + dir.toString(), e);
                        }
                    }
                } 
                if (trackingMode == FileTrackMode.FILE_WATCHER || trackingMode == FileTrackMode.NOT_SET) {
                    debug("Adding subdirectory to WatchService: " + dir.toString());
                    WatchKey key = dir.register(watcher,
                            new WatchEvent.Kind[] { StandardWatchEventKinds.ENTRY_MODIFY,
                                    StandardWatchEventKinds.ENTRY_DELETE, StandardWatchEventKinds.ENTRY_CREATE },
                            SensitivityWatchEventModifier.HIGH);
                    if (removeOnContainerRebuild) {
                        debug("Adding to containerfileDirectoriesWatchKeys: " + dir);
                        containerfileDirectoriesWatchKeys.add(key);
                    }
                }
                return FileVisitResult.CONTINUE;
            }
        });

    }

    /**
     * Get the file from the configDirectory if it exists
     * 
     * @param file 
     * @return file or null if it does not exist
     */
    protected File getFileFromConfigDirectory(String file) {
        File f = new File(configDirectory, file);
        if (configDirectory != null && f.exists()) {
            return f;
        }
        return null;
    }

    /**
     * Given the fileChanged delete the corresponding Java class or directory
     * 
     * @param fileChanged       Java file changed
     * @param classesDir        the directory for compiled classes
     * @param compileSourceRoot the source directory for the Java classes
     * @throws IOException unable to resolve canonical path
     */
    protected void deleteJavaFile(File fileChanged, File classesDir, File compileSourceRoot) throws IOException {
        String fileName = fileChanged.getName();
        File parentFile = fileChanged.getParentFile();

        boolean javaFile = fileName.endsWith(".java");
        String relPath;
        if (javaFile) {
            fileName = fileName.substring(0, fileChanged.getName().indexOf(".java"));
            relPath = parentFile.getCanonicalPath()
                    .substring(parentFile.getCanonicalPath().indexOf(compileSourceRoot.getCanonicalPath())
                            + compileSourceRoot.getCanonicalPath().length())
                    + "/" + fileName + ".class";
        } else {
            relPath = parentFile.getCanonicalPath()
                    .substring(parentFile.getCanonicalPath().indexOf(compileSourceRoot.getCanonicalPath())
                            + compileSourceRoot.getCanonicalPath().length())
                    + "/" + fileName;
        }

        File targetFile = new File(classesDir.getCanonicalPath() + relPath);
        if (targetFile.exists()) {
            if (targetFile.isDirectory()) {
                try {
                    FileUtils.deleteDirectory(targetFile);
                    info("The target directory " + targetFile.getCanonicalPath() + " was deleted.");
                } catch (IllegalArgumentException e) {
                    debug("Could not delete directory " + targetFile.getCanonicalPath() + ". " + e.getMessage());
                } catch (IOException e) {
                    error("There was an error encountered while deleting the directory " + targetFile.getCanonicalPath()
                            + ". " + e.getMessage());
                }
            } else {
                if (targetFile.delete()) {
                    info("The java class " + targetFile.getCanonicalPath() + " was deleted.");
                } else {
                    error("Could not delete the file " + targetFile.getCanonicalPath() + ". ");
                }
            }
        } else {
            debug("File deleted but could not find corresponding file or folder in the target directory: "
                    + fileChanged.getCanonicalPath() + ".");
        }
    }

    /**
     * Recompile Java source files and run tests after application update
     * 
     * @param javaFilesChanged collection of Java files changed
     * @param artifactPaths list of project artifact paths for building the classpath
     * @param executor the test thread executor
     * @param outputDirectory the directory for compiled classes
     * @param testOutputDirectory the directory for compiled test classes
     * @param projectName the name of the current project (artifactId), null if only one project exists
     * @param projectBuildFile the build file of the current project
     * @param projectCompilerOptions the Java compiler options of the current project
     * @param forceSkipUTs whether to force skipping the unit tests
     * @param skipRunningTests whether to skip running tests, takes precedence over the forceSkipUTs param
     * @throws PluginExecutionException if the classes output directory doesn't exist and can't be created
     */
    protected boolean recompileJavaSource(Collection<File> javaFilesChanged, Set<String> artifactPaths,
            ThreadPoolExecutor executor, File outputDirectory, File testOutputDirectory, String projectName,
            File projectBuildFile, JavaCompilerOptions projectCompilerOptions, boolean forceSkipUTs,
            boolean skipRunningTests) throws PluginExecutionException {
        return recompileJava(javaFilesChanged, artifactPaths, executor, false, outputDirectory, testOutputDirectory,
                projectName, projectBuildFile, projectCompilerOptions, forceSkipUTs, skipRunningTests);
    }

    /**
     * Recompile test source files and run tests immediately
     * 
     * @param javaFilesChanged collection of Java files changed
     * @param artifactPaths list of project artifact paths for building the classpath
     * @param executor the test thread executor
     * @param outputDirectory the directory for compiled classes
     * @param testOutputDirectory the directory for compiled test classes
     * @param projectName the name of the current project (artifactId), null if only one project exists
     * @param projectBuildFile the build file of the current project
     * @param projectCompilerOptions the Java compiler options of the current project
     * @param forceSkipUTs whether to force skipping the unit tests
     * @param skipRunningTests whether to skip running tests, takes precedence over the forceSkipUTs param
     * @throws PluginExecutionException if the classes output directory doesn't exist and can't be created
     */
    protected boolean recompileJavaTest(Collection<File> javaFilesChanged, Set<String> artifactPaths,
            ThreadPoolExecutor executor, File outputDirectory, File testOutputDirectory, String projectName,
            File projectBuildFile, JavaCompilerOptions projectCompilerOptions, boolean forceSkipUTs,
            boolean skipRunningTests) throws PluginExecutionException {
        return recompileJava(javaFilesChanged, artifactPaths, executor, true, outputDirectory, testOutputDirectory,
                projectName, projectBuildFile, projectCompilerOptions, forceSkipUTs, skipRunningTests);
    }

    /**
     * Recompile source files
     * 
     * @param javaFilesChanged       collection of Java files changed
     * @param artifactPaths          list of project artifact paths for building the
     *                               classpath
     * @param executor               the test thread executor
     * @param tests                  indicates whether the files changed were test
     *                               files
     * @param outputDirectory        the directory for compiled classes
     * @param testOutputDirectory    the directory for compiled test classes
     * @param projectName            the name of the current project (artifactId),
     *                               null if only one project exists
     * @param projectBuildFile       the build file of the current project
     * @param projectCompilerOptions the Java compiler options of the current
     *                               project
     * @param forceSkipUTs           whether to force skipping the unit tests
     * @param skipRunningTests       whether to skip running tests, takes
     *                               precedence over the forceSkipUTs param
     * @throws PluginExecutionException if the classes output directory doesn't
     *                                  exist and can't be created
     */
    protected boolean recompileJava(Collection<File> javaFilesChanged, Set<String> artifactPaths,
            ThreadPoolExecutor executor, boolean tests, File outputDirectory, File testOutputDirectory,
            String projectName, File projectBuildFile, JavaCompilerOptions projectCompilerOptions, boolean forceSkipUTs,
            boolean skipRunningTests) throws PluginExecutionException {
        try {
            int messageOccurrences = countApplicationUpdatedMessages();
            boolean compileResult;

            if (useBuildRecompile) {
                compileResult = compile(tests ? testSourceDirectory : sourceDirectory);
            } else {
                // source root is src/main/java or src/test/java
                File classesDir = tests ? testOutputDirectory : outputDirectory;
                if (!classesDir.exists()) {
                    if (!classesDir.mkdirs()) {
                        throw new PluginExecutionException("The classes output directory " + classesDir.getAbsolutePath()
                                + " does not exist and cannot be created.");
                    } else if (classesDir.exists() && Objects.equals(classesDir.getCanonicalFile(), outputDirectory.getCanonicalFile())) {
                        // redeploy application when class directory has been created
                        redeployApp();
                    }
                }

                List<String> combinedCompilerOptions = new ArrayList<>(Arrays.asList(DEFAULT_COMPILER_OPTIONS));
                if (projectCompilerOptions != null) {
                    combinedCompilerOptions.addAll(projectCompilerOptions.getOptions());
                }
                debug("Compiler options: " + combinedCompilerOptions);

                List<File> outputDirs = new ArrayList<File>();

                if (tests) {
                    outputDirs.add(outputDirectory);
                    outputDirs.add(testOutputDirectory);
                } else {
                    outputDirs.add(outputDirectory);
                }

                Set<File> classPathElems = getClassPath(artifactPaths, outputDirs);

                JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
                StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null);

                fileManager.setLocation(StandardLocation.CLASS_PATH, classPathElems);
                fileManager.setLocation(StandardLocation.CLASS_OUTPUT, Collections.singleton(classesDir));

                Collection<JavaFileObject> compilationUnits = new HashSet<JavaFileObject>();
                for (File file : javaFilesChanged) {
                    if (file.exists() && file.isFile()) {
                        for (JavaFileObject o : fileManager.getJavaFileObjects(file)) {
                            compilationUnits.add(o);
                        }    
                    } else {
                        debug("The Java file " + file + " does not exist and will not be compiled.");
                    }
                }

                JavaCompiler.CompilationTask task = compiler.getTask(null, fileManager, null, combinedCompilerOptions, null,
                        compilationUnits);

                compileResult = task.call();
            }
            if (compileResult) {
                if (tests) {
                    if (projectName != null) {
                        info(projectName + " tests compilation was successful.");
                    } else {
                        info("Tests compilation was successful.");
                    }
                } else {
                    // redeploy app after compilation if not loose application
                    if (!isLooseApplication()) {
                        redeployApp();
                    }
                    if (projectName != null) {
                        info(projectName + " source compilation was successful.");
                    } else {
                        info("Source compilation was successful.");
                    }
                }
                // run tests after successful compile
                if (!skipRunningTests) {
                    if (tests) {
                        // if only tests were compiled, don't need to wait for
                        // app to update
                        runTestThread(false, executor, -1, skipUTs, false, projectBuildFile);
                    } else if (!generateFeatures) {
                        // do not run tests if generateFeatures = true, tests will run after generated-features.xml is updated
                        runTestThread(true, executor, messageOccurrences, skipUTs, false, projectBuildFile);
                    }
                }
                return true;
            } else {
                if (tests) {
                    if (projectName != null) {
                        info(projectName + " tests compilation had errors.");
                    } else {
                        info("Tests compilation had errors.");
                    }
                } else {
                    if (projectName != null) {
                        info(projectName + " source compilation had errors.");
                    } else {
                        info("Source compilation had errors.");
                    }
                }
                return false;
            }
        } catch (Exception e) {
            if (projectName != null) {
                error(projectName + " error compiling Java files: " + e.getMessage());
            } else {
                error("Error compiling Java files: " + e.getMessage());
            }
            debug(e);
            return false;
        }
    }

    /**
     * Gets the class path for the specified artifactPaths and outputDirs.
     * 
     * @param artifactPaths list of artifacts for the current project
     * @param outputDirs list of output directories for the current project
     * @return set of classpath files
     * @throws IOException unable to resolve canonical path
     */
    protected Set<File> getClassPath(Set<String> artifactPaths, List<File> outputDirs) throws IOException {
        Set<String> parsedFiles = new HashSet<>();
        Deque<String> toParse = new ArrayDeque<>();

        for (String artifactPath : artifactPaths) {
            toParse.add(new File(artifactPath).getCanonicalPath());
        }

        Set<File> classPathElements = new HashSet<>();
        classPathElements.addAll(outputDirs);
        while (!toParse.isEmpty()) {
            String s = toParse.poll();
            if (!parsedFiles.contains(s)) {
                parsedFiles.add(s);
                File file = new File(s);
                if (file.exists() && file.getName().endsWith(".jar")) {
                    classPathElements.add(file);
                    if (!file.isDirectory()) {
                        try (JarFile jar = new JarFile(file)) {
                            Manifest mf = jar.getManifest();
                            if (mf == null || mf.getMainAttributes() == null) {
                                continue;
                            }
                            Object classPath = mf.getMainAttributes().get(Attributes.Name.CLASS_PATH);
                            if (classPath != null) {
                                for (String i : classPath.toString().split(" ")) {
                                    File f;
                                    try {
                                        URL u = new URL(i);
                                        f = new File(u.getPath());
                                    } catch (MalformedURLException e) {
                                        f = new File(file.getParentFile(), i);
                                    }
                                    if (f.exists()) {
                                        toParse.add(f.getCanonicalPath());
                                    }
                                }
                            }
                        } catch (Exception e) {
                            throw new RuntimeException("Failed to open class path file " + file, e);
                        }
                    }
                } else if (file.isDirectory() && !classPathElements.contains(file)) {
                    debug("Adding directory to compile class path: " + file);
                    classPathElements.add(file);
                }
            }
        }
        return classPathElements;
    }

    /**
     * Run tests in a new thread.
     * 
     * @param waitForApplicationUpdate whether it should wait for the application to
     *                                 update before running integration tests
     * @param executor                 the thread pool executor
     * @param messageOccurrences       how many times the application updated
     *                                 message has occurred in the log
     * @param manualInvocation         whether the tests were manually invoked
     * @param currentBuildFiles        the build file(s) to run tests against
     */
    public void runTestThread(boolean waitForApplicationUpdate, ThreadPoolExecutor executor, int messageOccurrences,
            boolean manualInvocation, File... currentBuildFiles) {
        // always pass in skip unit tests value for main project
        runTestThread(waitForApplicationUpdate, executor, messageOccurrences, this.skipUTs, manualInvocation,
                currentBuildFiles);
    }

    /**
     * Run tests in a new thread. Only call this method directly if you want to
     * override the skip unit tests value.
     * 
     * @param waitForApplicationUpdate whether it should wait for the application to
     *                                 update before running integration tests
     * @param executor                 the thread pool executor
     * @param messageOccurrences       how many times the application updated
     *                                 message has occurred in the log
     * @param skipUnitTests            whether to skip the unit tests
     * @param manualInvocation         whether the tests were manually invoked
     * @param currentBuildFiles        the build file(s) to run tests against
     */
    public void runTestThread(boolean waitForApplicationUpdate, ThreadPoolExecutor executor, int messageOccurrences,
            boolean skipUnitTests, boolean manualInvocation, File... currentBuildFiles) {
        try {
            if (manualInvocation || hotTests) {
                executor.execute(new TestJob(waitForApplicationUpdate, messageOccurrences, executor, skipUnitTests,
                        manualInvocation, currentBuildFiles));
            }
        } catch (RejectedExecutionException e) {
            debug("Cannot add thread since max threads reached", e);
        }
    }

    public class TestJob implements Runnable {
        private boolean waitForApplicationUpdate;
        private int messageOccurrences;
        private ThreadPoolExecutor executor;
        private boolean skipUnitTests;
        private boolean manualInvocation;
        private File[] currentBuildFiles;

        public TestJob(boolean waitForApplicationUpdate, int messageOccurrences, ThreadPoolExecutor executor,
                boolean skipUnitTests, boolean manualInvocation, File... currentBuildFiles) {
            this.waitForApplicationUpdate = waitForApplicationUpdate;
            this.messageOccurrences = messageOccurrences;
            this.executor = executor;
            this.skipUnitTests = skipUnitTests;
            this.manualInvocation = manualInvocation;
            this.currentBuildFiles = currentBuildFiles;
        }

        @Override
        public void run() {
            try {
                if (currentBuildFiles.length >= 1) {
                    for (File currentBuildFile : currentBuildFiles) {
                        boolean currentProjForceSkipUTs = skipUnitTests;
                        boolean currentProjForceSkipITs = skipITs;
                        boolean currentProjSkipTests = skipTests;
                        String currentProjName = applicationId;
                        if (isMultiModuleProject()) {
                            // multi module project, match the build file to the upstream project to honour
                            // the skipUTs value
                            for (ProjectModule upstreamProject : upstreamProjects) {
                                if (upstreamProject.getBuildFile().equals(currentBuildFile)) {
                                    currentProjForceSkipUTs = upstreamProject.skipUTs();
                                    currentProjForceSkipITs = upstreamProject.skipITs();
                                    currentProjSkipTests = upstreamProject.skipTests();
                                    currentProjName = upstreamProject.getProjectName();
                                    break;
                                }
                            }
                            runTests(waitForApplicationUpdate, messageOccurrences, executor, currentProjSkipTests,
                                    currentProjForceSkipUTs, currentProjForceSkipITs, currentBuildFile,
                                    currentProjName);
                        } else {
                            // single module project, do not need to pass in project name
                            runTests(waitForApplicationUpdate, messageOccurrences, executor, currentProjSkipTests,
                                    currentProjForceSkipUTs, currentProjForceSkipITs, currentBuildFile, null);
                        }
                    }
                } else {
                    // build file was not indicated, run tests for the main (default) project
                    runTests(waitForApplicationUpdate, messageOccurrences, executor, skipTests, skipUTs, skipITs, null,
                            null);
                }
            } finally {
                // start watching for hotkey presses if not already started, or re-print message
                // if thread already running
                runHotkeyReaderThread(executor);
            }
        }

        public boolean isManualInvocation() {
            return manualInvocation;
        }
    }

    /**
     * Gets the Liberty server's host name.
     * @return hostName the host name, or null if the server is not started
     */
    public String getHostName() {
        return hostName;
    }

    /**
     * Gets the Liberty server's http port.
     * @return httpPort the http port, or null if the server is not started or there is no http port bound
     */
    public String getHttpPort() {
        return httpPort;
    }

    /**
     * Gets the Liberty server's https port.
     * @return httpsPort the https port, or null if the server is not started or there is no https port bound
     */
    public String getHttpsPort() {
        return httpsPort;
    }

    /**
     * Sets the preferred debug port.
     * 
     * @param libertyDebugPort the preferred debug port
     */
    public void setLibertyDebugPort(int libertyDebugPort) {
        this.libertyDebugPort = libertyDebugPort;
    }

    /**
     * Reload the property file by restarting the server if there were changes.
     * 
     * @param propertyFile The property file that was changed.
     * @throws PluginExecutionException if there was an error when reloading the file
     * @return true if the property file was reloaded with changes
     */
    private boolean reloadPropertyFile(File propertyFile) throws PluginExecutionException {
        Properties properties = readPropertiesFromFile(propertyFile);
        if (!Objects.equals(properties, propertyFilesMap.get(propertyFile))) {
            debug("Properties file " + propertyFile.getAbsolutePath() + " has changed. Restarting server...");
            propertyFilesMap.put(propertyFile, properties);    
            restartServer();
            return true;
        } else {
            debug("No changes detected in properties file " + propertyFile.getAbsolutePath());
            return false;
        }
    }

    /**
     * This is needed for Gradle only.
     * 
     * Sets additional property files that may be used by the build.
     * Loads the properties for later comparison of changes.
     * 
     * @param propertyFiles list of property files
     */
    public void setPropertyFiles(List<File> propertyFiles) {
        if (propertyFiles == null) {
            return;
        }
        if (propertyFilesMap == null) {
            propertyFilesMap = new HashMap<File, Properties>(propertyFiles.size());
        }
        for (File propertyFile : propertyFiles) {
            Properties properties = readPropertiesFromFile(propertyFile);
            propertyFilesMap.put(propertyFile, properties);
        }
    }

    /**
     * Read properties from file.  If file does not exist or an IO exception occurred, returns null.
     */
    private Properties readPropertiesFromFile(File propertyFile) {
        Properties properties = null;
        if (propertyFile.exists()) {
            InputStream inputStream = null;
            try {
                debug("Loading properties from file: " + propertyFile);
                inputStream = new FileInputStream(propertyFile);
                properties = new Properties();
                properties.load(inputStream);
            } catch (IOException e) {
                error("Could not read properties file " + propertyFile.getAbsolutePath(), e);
            } finally {
                if (inputStream != null) {
                    try {
                        inputStream.close();
                    } catch (IOException e) {
                        // nothing to do
                    }
                }
            }
        }
        return properties;
    }

    public String getContainerName() {
        return containerName;
    }

    public Set<String> getCompileArtifacts() {
        return compileArtifactPaths;
    }

    public Set<String> getTestArtifacts() {
        return testArtifactPaths;
    }

    /**
     * Updates the Java compiler options
     * @param updatedCompilerOptions
     */
    public void updateJavaCompilerOptions(JavaCompilerOptions updatedCompilerOptions) {
        compilerOptions = updatedCompilerOptions;
    }

    /**
     * Given a build file returns the corresponding ProjectModule otherwise
     * returns null
     * 
     * @param buildFile
     * @return Project module
     * @throws IOException
     */
    public ProjectModule getProjectModule(File buildFile) throws IOException {
        if (isMultiModuleProject()) {
            for (ProjectModule p : upstreamProjects) {
                if (p.getBuildFile().getCanonicalPath().equals(buildFile.getCanonicalPath())) {
                    return p;
                }
            }
        }
        return null;
    }

    /**
     * Checks if upstreamProjects exist, indicating that this is a multi module
     * project
     * 
     * @return true if this is a multi module project, false if not
     */
    public boolean isMultiModuleProject() {
        return (upstreamProjects != null && !upstreamProjects.isEmpty());
    }

    /**
     * Returns the value of changeOnDemandTestsAction boolean.
     * 
     * @return value of changeOnDemandTestsAction
     */
    public boolean isChangeOnDemandTestsAction() {
        return this.changeOnDemandTestsAction;
    }

    /**
     * Gets an array of all the buildFiles in a multi module project
     * 
     * @return File[] array of buildFiles in multi module project, in build order
     */
    private File[] getAllBuildFiles() {
        File[] buildFiles = new File[upstreamProjects.size() + 1];
        int count = 0;
        for (ProjectModule project : upstreamProjects) {
            buildFiles[count] = project.getBuildFile();
            count++;
        }
        buildFiles[count] = buildFile;
        return buildFiles;
    }

    /**
     * Gets an array of all the build files for the specified project module and
     * dependent projects
     * 
     * @param currentProject ProjectModule
     * @return File[] array of buildFiles for specified project, in build order
     */
    private File[] getAllBuildFiles(ProjectModule currentProject) {
        List<File> dependentModuleBuildFiles = currentProject.getDependentModules();
        File[] buildFiles = new File[dependentModuleBuildFiles.size() + 1];
        buildFiles[0] = currentProject.getBuildFile();
        int count = 1;
        for (File buildFile : dependentModuleBuildFiles) {
            buildFiles[count] = buildFile;
            count++;
        }
        return buildFiles;
    }

    /**
     * Trigger a compile of the entire main module. The main module is the module
     * with the Liberty configuration.Adds all Java files to the to be compiled list
     * so that they will be compiled on next watch loop.
     * 
     * @param testsOnly true if ONLY tests should be compiled
     * @throws IOException
     */
    protected void triggerMainModuleCompile(boolean testsOnly) throws IOException {
        triggerProjectCompile(this.sourceDirectory, recompileJavaSources, this.testSourceDirectory, recompileJavaTests,
                testsOnly, packagingType);
    }

    /**
     * Trigger a compile of the entire specified module. This is only used in a
     * multi-module scenario. Adds all Java files to the to be compiled list so that
     * they will be compiled on next watch loop.
     * 
     * @param project   ProjectModule, the module to be compiled
     * @param testsOnly true if ONLY tests should be compiled
     * @throws IOException
     */
    protected void triggerUpstreamModuleCompile(ProjectModule project, boolean testsOnly) throws IOException {
        triggerProjectCompile(project.getSourceDirectory(), project.recompileJavaSources,
                project.getTestSourceDirectory(), project.recompileJavaTests, testsOnly, project.getPackagingType());
    }

    private void triggerProjectCompile(File sourceDir, Collection<File> recompileJavaSourceSet, File testSourceDir,
            Collection<File> recompileJavaTestSet, boolean testsOnly, String packagingType) throws IOException {

        // recompile source
        if (!testsOnly && shouldIncludeSources(packagingType)) {
            if (sourceDir.exists()) {
                Collection<File> allJavaSources = FileUtils.listFiles(sourceDir.getCanonicalFile(),
                        new String[] { "java" }, true);
                recompileJavaSourceSet.addAll(allJavaSources);
            }
        }

        // recompile tests
        if (testSourceDir.exists()) {
            Collection<File> allJavaTestSources = FileUtils.listFiles(testSourceDir.getCanonicalFile(),
                    new String[] { "java" }, true);
            recompileJavaTestSet.addAll(allJavaTestSources);
        }
    }

    /**
     * Compile all classes in the module corresponding to the build file.
     * 
     * @param moduleBuildFile build file
     * @param testsOnly       true if test classes should be compiled
     * @param executor        ThreadPoolExecutor
     * @return true if compilation is successful
     * @throws IOException
     * @throws PluginExecutionException
     */
    private boolean compileModuleForBuildFile(File moduleBuildFile, boolean testsOnly, ThreadPoolExecutor executor)
            throws IOException, PluginExecutionException {
        if (moduleBuildFile.getCanonicalPath().equals(buildFile.getCanonicalPath())) {
            debug("recompileDependencies is set to true, recompiling the entire module for "
                    + moduleBuildFile.getCanonicalPath());
            disableDependencyCompile = true;
            return compileAllClasses(testsOnly, executor);
        }
        for (ProjectModule project : upstreamProjects) {
            if (moduleBuildFile.getCanonicalPath().equals(project.getBuildFile().getCanonicalPath())) {
                debug("recompileDependencies is set to true, recompiling the entire module for "
                        + moduleBuildFile.getCanonicalPath());
                project.disableDependencyCompile = true;
                return compileAllClasses(project, testsOnly, executor);
            }
        }
        return false;
    }

    /**
     * Compiles all the Java files in the source directory of the main module.
     * Compilation occurs immediately (not on next watch loop).
     * 
     * @param tests    true if Java files in the test source directory should also
     *                 be compiled
     * @param executor ThreadPoolExecutor
     * @return true if compilation is successful
     * @throws PluginExecutionException
     * @throws IOException
     */
    private boolean compileAllClasses(boolean tests, ThreadPoolExecutor executor)
            throws PluginExecutionException, IOException {
        return compileAllClasses(this.sourceDirectory, this.testSourceDirectory, packagingType, compileArtifactPaths,
                testArtifactPaths, outputDirectory, testOutputDirectory, getProjectName(), buildFile, compilerOptions,
                skipUTs, failedCompilationJavaSources, failedCompilationJavaTests, tests, executor);
    }

    /**
     * Compiles all the Java files in the source directory of the corresponding
     * module. Compilation occurs immediately (not on next watch loop).
     * 
     * @param project  ProjectModule, the module to be compiled
     * @param tests    true if Java files in the test source directory should also
     *                 be compiled
     * @param executor ThreadPoolExecutor
     * @return true if compilation is successful
     * @throws IOException
     * @throws PluginExecutionException
     */
    private boolean compileAllClasses(ProjectModule project, boolean tests, ThreadPoolExecutor executor)
            throws IOException, PluginExecutionException {
        return compileAllClasses(project.getSourceDirectory(), project.getTestSourceDirectory(),
                project.getPackagingType(), project.getCompileArtifacts(), project.getTestArtifacts(),
                project.getOutputDirectory(), project.getTestOutputDirectory(), project.getProjectName(),
                project.getBuildFile(), project.getCompilerOptions(), project.skipUTs(),
                project.failedCompilationJavaSources, project.failedCompilationJavaTests, tests, executor);
    }

    private boolean compileAllClasses(File sourceDir, File testSourceDir, String packagingType,
            Set<String> compileArtifactPaths, Set<String> testArtifactPaths, File outputDir, File testOutputDir,
            String projectName, File buildFile, JavaCompilerOptions compilerOptions, boolean skipUTs,
            Collection<File> failedCompilationJavaSources, Collection<File> failedCompilationJavaTests, boolean tests,
            ThreadPoolExecutor executor) throws IOException, PluginExecutionException {
        boolean successfulCompilation = true;

        // recompile source
        if (!tests && shouldIncludeSources(packagingType)) {
            if (sourceDir.exists()) {
                Collection<File> allJavaSources = FileUtils.listFiles(sourceDir.getCanonicalFile(),
                        new String[] { "java" }, true);
                debug("Recompiling Java source files: " + allJavaSources);
                if (recompileJavaSource(allJavaSources, compileArtifactPaths, executor, outputDir, testOutputDir,
                        projectName, buildFile, compilerOptions, skipUTs, true)) {
                    // successful compilation so we can clear failedCompilation list
                    failedCompilationJavaSources.clear();
                } else {
                    successfulCompilation = false;
                    failedCompilationJavaSources.addAll(recompileJavaSources);
                }
            }
        }
        // recompile tests
        if (testSourceDir.exists()) {
            Collection<File> allJavaTestSources = FileUtils.listFiles(testSourceDir.getCanonicalFile(),
                    new String[] { "java" }, true);
            debug("Recompiling Java test files: " + allJavaTestSources);
            if (recompileJavaTest(allJavaTestSources, testArtifactPaths, executor, outputDir, testOutputDir,
                    projectName, buildFile, compilerOptions, skipUTs, true)) {
                // successful compilation so we can clear failedCompilation list
                failedCompilationJavaTests.clear();
            } else {
                successfulCompilation = false;
                failedCompilationJavaTests.addAll(recompileJavaTests);
            }
        }
        return successfulCompilation;
    }

    // indicates whether to recompile dependencies for the given project module
    private boolean shouldRecompileDependencies(ProjectModule project) {
        return (!project.disableDependencyCompile && recompileDependencies && !initialCompile);
    }

    // returns true if directory is a src/main/java dir of an upstream module
    private boolean isUpstreamSourceDir(File dirAdded) {
        try {
            if (isMultiModuleProject()) {
                for (ProjectModule p : upstreamProjects) {
                    if (p.getSourceDirectory().getCanonicalPath().startsWith(dirAdded.getCanonicalPath() + File.separator)) {
                        return true;
                    }
                }
            }
            return false;
        } catch (IOException e) {
            return false;
        }
    }

    // returns true if there are classes in the project that are currently failing
    // to compile
    private boolean classesFailingToCompile() {
        boolean failingClasses = false;
        if (isMultiModuleProject()) {
            for (ProjectModule p : upstreamProjects) {
                if (!p.failedCompilationJavaSources.isEmpty()) {
                    failingClasses = true;
                }
            }
        }
        if (!failedCompilationJavaSources.isEmpty()) {
            failingClasses = true;
        }
        return failingClasses;
    }

    // Returns true if features have been modified in the configuration directory
    private boolean serverFeaturesModified() {
        ServerFeatureUtil servUtil = getServerFeatureUtilObj();
        Set<String> generatedFeatureSet = null;
        Set<String> featuresExcludingGenerated = null;

        if (generateFeatures) {
            // generateFeatures scenario: check if a generated feature has been manually added to other config files
            // Here we pass generated-features.xml instead of server.xml to calculate the generated ones
            // The second parameter will only be used if generated-features.xml contains an <include> element and 
            // the current design does not allow this.
        	FeaturesPlatforms fp = servUtil.getServerXmlFeatures(new FeaturesPlatforms(), serverDirectory,
                    generateFeaturesFile, null, null);
        	if (fp != null)
        		generatedFeatureSet = fp.getFeatures();

            // Calculate the features specified in the config excluding those in generated-features
            Set<String> generatedFiles = new HashSet<String>();
            generatedFiles.add(generateFeaturesFile.getName());
            // if serverXmlFile is null, getServerFeatures will use the default server.xml
            // in the configDirectory
            fp = servUtil.getServerFeatures(configDirectory, serverXmlFile,
                    new HashMap<String, File>(), generatedFiles);
            if (fp != null)
            	featuresExcludingGenerated = fp.getFeatures();
            if (featuresExcludingGenerated != null && generatedFeatureSet != null
                    && !Collections.disjoint(featuresExcludingGenerated, generatedFeatureSet)) {
                // indicates a generated feature has been manually added to other config files
                return true;
            }
        } else {
        	FeaturesPlatforms fp = servUtil.getServerFeatures(configDirectory, serverXmlFile,
                    new HashMap<String, File>(), null);
        	if (fp != null)
        		featuresExcludingGenerated = fp.getFeatures();
        }
        
        // compare current feature list to existing feature list
        Set<String> features = featuresExcludingGenerated != null ? new HashSet<String>(featuresExcludingGenerated)
                : new HashSet<String>();
        if (generatedFeatureSet != null) {
            features.addAll(generatedFeatureSet);
        }
        return !features.equals(getExistingFeatures());
    }

    // returns true and updates the generatedFeaturesSet if the feature set in the
    // generated-features.xml file has changed
    private boolean generatedFeaturesModified() {
        Set<String> updatedGeneratedFeatures = getGeneratedFeatures();
        if (!updatedGeneratedFeatures.equals(generatedFeaturesSet)) {
            generatedFeaturesSet = updatedGeneratedFeatures; // update generated features set list
            return true;
        }
        return false;
    }

    // returns the features specified in the generated-features.xml file
    // generated-features.xml has a <server> element so it is also a "serverFile"
    // The second parameter will not be used for generated-features.xml.
    private Set<String> getGeneratedFeatures() {
        ServerFeatureUtil servUtil = getServerFeatureUtilObj();
        FeaturesPlatforms fp = servUtil.getServerXmlFeatures(new FeaturesPlatforms(), configDirectory,
                generateFeaturesFile, null, null);
        return fp!=null ? fp.getFeatures() : new HashSet<String>(); 
    }

    /**
     * Create metadata when running devc mode and containers
     * Language server then uses metadata file to connect
     */
    public void writeDevcMetadata(boolean alive) {
        File metaFile = new File(buildDirectory, serverDirectory.getName() + "-liberty-devc-metadata.xml");

        FileWriter metaFileWriter = null;
        XMLStreamWriter metadataWriter = null;
        try {
            metaFileWriter = new FileWriter(metaFile);
            metadataWriter = XMLOutputFactory.newInstance().createXMLStreamWriter(metaFileWriter) ;
            metadataWriter.writeStartDocument();
            metadataWriter.writeStartElement("devcModeMetaData");
            writeElement(metadataWriter, "containerType", isDocker ? DEVC_CONTAINER_DOCKER : DEVC_CONTAINER_PODMAN);
            writeElement(metadataWriter, "containerName", containerName != null ? containerName : DEVMODE_CONTAINER_BASE_NAME);
            writeElement(metadataWriter, "imageName", imageName);
            if (containerfile != null) {
                writeElement(metadataWriter, "containerfile", containerfile.getCanonicalPath());
            }
            if (containerBuildContext != null) {
                writeElement(metadataWriter, "containerBuildContext", containerBuildContext.getCanonicalPath());
            }
            writeElement(metadataWriter, "containerAlive", String.valueOf(alive));
            writeElement(metadataWriter, "containerBuildTimeout", Integer.toString(containerBuildTimeout));
            writeElement(metadataWriter, "containerRunOpts", containerRunOpts);
            metadataWriter.writeEndElement();
            metadataWriter.writeEndDocument();
        } catch (Exception e) {
            warn("Failed to write metadata.\n" + e.getMessage());
        } finally {
            try {
                if (metadataWriter != null) {
                    metadataWriter.flush();
                    metadataWriter.close();
                }
                if (metaFileWriter != null) {
                    metaFileWriter.flush();
                    metaFileWriter.close();
                }
            } catch (Exception e) {
                warn("Failed to close metadata writer due to an error.\n" + e.getMessage());
            }
        }
    }

    private void writeElement(XMLStreamWriter writer, String element, String optional) throws XMLStreamException {
        writer.writeStartElement(element);
        if (optional != null) writer.writeCharacters(optional);
        writer.writeEndElement();
    }
}