Gradle multi module, changes to watch child project

.github/workflows/gradle.yml

@@ -154,13 +154,13 @@ jobs:
     - name: Run tests that require a minimum of Java 17 or later
       if: ${{ matrix.java == '17'  || matrix.java == '21' }}
       run:
-        ./gradlew clean install check -P"test.include"="**/TestSpringBootApplication30*,**/TestCompileJSPSource17*" -Druntime=${{ matrix.RUNTIME }} -DruntimeVersion="${{ matrix.RUNTIME_VERSION }}" --stacktrace --info --no-daemon
+        ./gradlew clean install check -P"test.include"="**/TestSpringBootApplication30*,**/TestCompileJSPSource17*,**/TestMultiModuleLooseEarWithPages*" -Druntime=${{ matrix.RUNTIME }} -DruntimeVersion="${{ matrix.RUNTIME_VERSION }}" --stacktrace --info --no-daemon
     # Run tests
     - name: Run tests with Gradle on Windows
       working-directory: C:/ci.gradle
       # LibertyTest is excluded because test0_run hangs
       # For Java 8, TestCreateWithConfigDir is excluded because test_micro_clean_liberty_plugin_variable_config runs out of memory
-      run: ./gradlew clean install check -P"test.exclude"="${{env.TEST_EXCLUDE}}" -Druntime=${{ matrix.RUNTIME }} -DruntimeVersion="${{ matrix.RUNTIME_VERSION }}" --stacktrace --info --no-daemon
+      run: ./gradlew clean install check -P"test.include"="**" -Druntime=${{ matrix.RUNTIME }} -DruntimeVersion="${{ matrix.RUNTIME_VERSION }}" --stacktrace --info --no-daemon
       timeout-minutes: 75
     # Copy build reports and upload artifact if build failed
     - name: Copy build/report/tests/test for upload

src/main/groovy/io/openliberty/tools/gradle/tasks/DevTask.groovy

@@ -15,42 +15,42 @@
  */
 package io.openliberty.tools.gradle.tasks
 
+import groovy.xml.XmlParser
+import io.openliberty.tools.ant.ServerTask
+import io.openliberty.tools.common.plugins.util.BinaryScannerUtil
+import io.openliberty.tools.common.plugins.util.DevUtil
+import io.openliberty.tools.common.plugins.util.InstallFeatureUtil
+import io.openliberty.tools.common.plugins.util.JavaCompilerOptions
+import io.openliberty.tools.common.plugins.util.PluginExecutionException
+import io.openliberty.tools.common.plugins.util.PluginScenarioException
+import io.openliberty.tools.common.plugins.util.ProjectModule
+import io.openliberty.tools.common.plugins.util.ServerFeatureUtil
+import io.openliberty.tools.common.plugins.util.ServerFeatureUtil.FeaturesPlatforms
+import io.openliberty.tools.common.plugins.util.ServerStatusUtil
+import io.openliberty.tools.gradle.utils.DevTaskHelper
+import io.openliberty.tools.gradle.utils.LooseWarApplication
 import org.gradle.api.GradleException
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.internal.file.DefaultFilePropertyFactory
+import org.gradle.api.logging.LogLevel
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.Optional
-import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.SourceSet
-import org.gradle.api.logging.LogLevel
+import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.options.Option
 import org.gradle.testfixtures.ProjectBuilder
 import org.gradle.tooling.BuildException
 import org.gradle.tooling.BuildLauncher
-import org.gradle.tooling.ProjectConnection
 import org.gradle.tooling.GradleConnector
+import org.gradle.tooling.ProjectConnection
 
+import java.nio.file.Path
+import java.util.Map.Entry
 import java.util.concurrent.ArrayBlockingQueue
 import java.util.concurrent.ThreadPoolExecutor
-
-import io.openliberty.tools.ant.ServerTask
-
-import io.openliberty.tools.common.plugins.util.DevUtil
-import io.openliberty.tools.common.plugins.util.InstallFeatureUtil
-import io.openliberty.tools.common.plugins.util.PluginExecutionException
-import io.openliberty.tools.common.plugins.util.PluginScenarioException
-import io.openliberty.tools.common.plugins.util.ServerFeatureUtil
-import io.openliberty.tools.common.plugins.util.ServerFeatureUtil.FeaturesPlatforms
-import io.openliberty.tools.common.plugins.util.ServerStatusUtil
-import io.openliberty.tools.common.plugins.util.ProjectModule
-import io.openliberty.tools.common.plugins.util.BinaryScannerUtil
-
 import java.util.concurrent.TimeUnit
-import java.util.Map.Entry
-import java.nio.file.Path;
-import groovy.xml.XmlParser
 
 class DevTask extends AbstractFeatureTask {
 
@@ -371,23 +371,22 @@ class DevTask extends AbstractFeatureTask {
         Map<String, File> libertyDirPropertyFiles = new HashMap<String, File> ();
 
         private ServerTask serverTask = null;
-
         DevTaskUtil(File buildDir, File installDirectory, File userDirectory, File serverDirectory, File sourceDirectory, File testSourceDirectory,
                     File configDirectory, File projectDirectory, List<File> resourceDirs, boolean changeOnDemandTestsAction,
                     boolean  hotTests, boolean  skipTests, boolean skipInstallFeature, String artifactId, int serverStartTimeout,
                     int verifyAppStartTimeout, int appUpdateTimeout, double compileWait,
                     boolean libertyDebug, boolean pollingTest, boolean container, File containerfile, File containerBuildContext,
                     String containerRunOpts, int containerBuildTimeout, boolean skipDefaultPorts, boolean keepTempContainerfile, 
-                    String mavenCacheLocation, String packagingType, File buildFile, boolean generateFeatures
+                    String mavenCacheLocation, String packagingType, File buildFile, boolean generateFeatures, List<Path> webResourceDirs,
+                    List<ProjectModule> projectModuleList
         ) throws IOException, PluginExecutionException {
             super(buildDir, serverDirectory, sourceDirectory, testSourceDirectory, configDirectory, projectDirectory, /* multi module project directory */ projectDirectory,
                     resourceDirs, changeOnDemandTestsAction, hotTests, skipTests, false /* skipUTs */, false /* skipITs */, skipInstallFeature, artifactId,  serverStartTimeout,
                     verifyAppStartTimeout, appUpdateTimeout, ((long) (compileWait * 1000L)), libertyDebug,
                     true /* useBuildRecompile */, true /* gradle */, pollingTest, container, containerfile, containerBuildContext, containerRunOpts, containerBuildTimeout, skipDefaultPorts,
-                    null /* compileOptions not needed since useBuildRecompile is true */, keepTempContainerfile, mavenCacheLocation, null /* multi module upstream projects */,
-                    false /* recompileDependencies only supported in ci.maven */, packagingType, buildFile, null /* parent build files */, generateFeatures, null /* compileArtifactPaths */, null /* testArtifactPaths */, new ArrayList<Path>() /* webResources */
+                    null /* compileOptions not needed since useBuildRecompile is true */, keepTempContainerfile, mavenCacheLocation, projectModuleList /* multi module upstream projects */,
+                    false /* recompileDependencies only supported in ci.maven */, packagingType, buildFile, null /* parent build files */, generateFeatures, null /* compileArtifactPaths */, null /* testArtifactPaths */, webResourceDirs /* webResources */
                 );
-
             this.libertyDirPropertyFiles = AbstractServerTask.getLibertyDirectoryPropertyFiles(installDirectory, userDirectory, serverDirectory);
             ServerFeatureUtil servUtil = getServerFeatureUtil(true, libertyDirPropertyFiles);
             FeaturesPlatforms fp = servUtil.getServerFeatures(serverDirectory, libertyDirPropertyFiles);
@@ -1262,15 +1261,19 @@ class DevTask extends AbstractFeatureTask {
 
         File buildFile = project.getBuildFile();
 
+        List<Path> webResourceDirs = DevTaskHelper.getWebSourceDirectoriesToMonitor(project);
         // Instantiate util before any child gradle tasks launched so it can help find available port if needed
+        // Project modules contain all child modules. This project modules will be present only for multi-module
+        // used to watch sub project src and test source files
+        List<ProjectModule> projectModules = getProjectModules()
         try {
             this.util = new DevTaskUtil(project.getLayout().getBuildDirectory().getAsFile().get(), serverInstallDir, getUserDir(project, serverInstallDir),
                 serverDirectory, sourceDirectory, testSourceDirectory, configDirectory, project.getRootDir(),
                 resourceDirs, changeOnDemandTestsAction.booleanValue(), hotTests.booleanValue(), skipTests.booleanValue(), skipInstallFeature.booleanValue(), artifactId, serverStartTimeout.intValue(),
                 verifyAppStartTimeout.intValue(), verifyAppStartTimeout.intValue(), compileWait.doubleValue(),
                 libertyDebug.booleanValue(), pollingTest.booleanValue(), container.booleanValue(), containerfile, containerBuildContext, containerRunOpts,
                 containerBuildTimeout, skipDefaultPorts.booleanValue(), keepTempContainerfile.booleanValue(), localMavenRepoForFeatureUtility,
-                getPackagingType(), buildFile, generateFeatures.booleanValue()
+                DevTaskHelper.getPackagingType(project), buildFile, generateFeatures.booleanValue(), webResourceDirs, projectModules
             );
         } catch (IOException | PluginExecutionException e) {
             throw new GradleException("Error initializing dev mode.", e)
@@ -1410,6 +1413,81 @@ class DevTask extends AbstractFeatureTask {
         }
     }
 
+    private List<ProjectModule> getProjectModules() {
+        List<ProjectModule> upstreamProjects = new ArrayList<ProjectModule>();
+        for (Project dependencyProject : DevTaskHelper.getAllUpstreamProjects(project)) {
+            // TODO get compiler options for upstream project
+            // JavaCompilerOptions upstreamCompilerOptions = getMavenCompilerOptions(p);
+            JavaCompilerOptions upstreamCompilerOptions = new JavaCompilerOptions();
+            SourceSet mainSourceSet = dependencyProject.sourceSets.main;
+            SourceSet testSourceSet = dependencyProject.sourceSets.test;
+            DefaultFilePropertyFactory.DefaultDirectoryVar outputDirectory = mainSourceSet.java.classesDirectory;
+            DefaultFilePropertyFactory.DefaultDirectoryVar testOutputDirectory = testSourceSet.java.classesDirectory;
+            Set<String> compileArtifacts = new HashSet<String>();
+            Set<String> testArtifacts = new HashSet<String>();
+            File upstreamSourceDir = new File(mainSourceSet.java.sourceDirectories.asPath)
+            File upstreamOutputDir = outputDirectory.asFile.get();
+            File upstreamTestSourceDir = new File(testSourceSet.java.sourceDirectories.asPath);
+            File upstreamTestOutputDir = testOutputDirectory.asFile.get();
+            // resource directories
+            List<File> upstreamResourceDirs = mainSourceSet.resources.srcDirs.toList();
+            /* TODO all gradle items
+            // properties that are set in the pom file
+            Properties props = dependencyProject.getProperties();
+
+            // properties that are set by user via CLI parameters
+            Properties userProps = session.getUserProperties();
+
+            Plugin libertyPlugin = getLibertyPluginForProject(p);
+            // use "dev" goal, although we don't expect the skip tests flags to be bound to any goal
+            Xpp3Dom config = ExecuteMojoUtil.getPluginGoalConfig(libertyPlugin, "dev", getLog());
+
+            boolean upstreamSkipTests = DevHelper.getBooleanFlag(config, userProps, props, "skipTests");
+            boolean upstreamSkipITs = DevHelper.getBooleanFlag(config, userProps, props, "skipITs");
+            boolean upstreamSkipUTs = DevHelper.getBooleanFlag(config, userProps, props, "skipUTs");
+
+            // only force skipping unit test for ear modules otherwise honour existing skip
+            // test params
+
+
+            // build list of dependent modules
+            List<MavenProject> dependentProjects = graph.getDownstreamProjects(p, true);
+            List<File> dependentModules = new ArrayList<File>();
+            for (MavenProject depProj : dependentProjects) {
+                dependentModules.add(depProj.getFile());
+            }
+            */
+            boolean upstreamSkipTests = false
+            boolean upstreamSkipITs = false
+            boolean upstreamSkipUTs = false
+
+            if (DevTaskHelper.getPackagingType(dependencyProject).equals("ear")) {
+                upstreamSkipUTs = true;
+            }
+            // build list of dependent modules
+            List<File> dependentModules = new ArrayList<File>();
+            ProjectModule upstreamProject = new ProjectModule(dependencyProject.getBuildFile(),
+                    dependencyProject.getName(),
+                    DevTaskHelper.getPackagingType(dependencyProject),
+                    compileArtifacts,
+                    testArtifacts,
+                    upstreamSourceDir,
+                    upstreamOutputDir,
+                    upstreamTestSourceDir,
+                    upstreamTestOutputDir,
+                    upstreamResourceDirs,
+                    upstreamSkipTests,
+                    upstreamSkipUTs,
+                    upstreamSkipITs,
+                    upstreamCompilerOptions,
+                    dependentModules);
+
+            upstreamProjects.add(upstreamProject);
+        }
+
+        return upstreamProjects;
+    }
+
     private boolean isInstallDirChanged(Project project, File currentInstallDir) {
         if (project.getLayout().getBuildDirectory().getAsFile().get().exists() && new File(project.getLayout().getBuildDirectory().getAsFile().get(), 'liberty-plugin-config.xml').exists()) {
             XmlParser pluginXmlParser = new XmlParser()

src/main/groovy/io/openliberty/tools/gradle/utils/DevTaskHelper.groovy

@@ -0,0 +1,153 @@
+/**
+ * (C) Copyright IBM Corporation 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openliberty.tools.gradle.utils
+
+import org.apache.commons.io.FilenameUtils
+import org.apache.tools.ant.taskdefs.Jar
+import org.gradle.api.GradleException
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.artifacts.Configuration
+import org.gradle.api.artifacts.Dependency
+import org.gradle.api.artifacts.ProjectDependency
+import org.gradle.api.tasks.bundling.War
+import org.gradle.plugins.ear.Ear
+
+import java.nio.file.Path
+
+public class DevTaskHelper {
+
+    /**
+     * <p> Get packaging type of Gradle project
+     * @param project gradle project
+     * @return
+     * @throws Exception
+     */
+    public static String getPackagingType(Project project) throws Exception{
+        if (project.plugins.hasPlugin("war") || !project.tasks.withType(War).isEmpty()) {
+            if (project.plugins.hasPlugin("org.springframework.boot")) {
+                return "springboot"
+            }
+            return "war"
+        }
+        else if (project.plugins.hasPlugin("ear") || !project.tasks.withType(Ear).isEmpty()) {
+            return "ear"
+        }
+        else if (project.plugins.hasPlugin("org.springframework.boot") ) {
+            return "springboot"
+        } else if (project.plugins.hasPlugin("java") ||!project.tasks.withType(Jar).isEmpty()) {
+            return "jar"
+        }
+        else {
+            throw new GradleException("Archive path not found. Supported formats are jar, war, ear, and springboot jar.")
+        }
+    }
+
+    /**
+     * get deploy dependencies for gradle project
+     * @param project project
+     * @return
+     */
+    public static Map<File, Dependency> getDeployDependencies(Project project) {
+        Map<File, Dependency> completeDeployDeps = new HashMap<File, Dependency>();
+        File[] filesAsDeps = project.configurations.deploy.getFiles().toArray()
+        Dependency[] deployDeps = project.configurations.deploy.getAllDependencies().toArray()
+
+        if (filesAsDeps.size() == deployDeps.size()) {
+            for (int i = 0; i < filesAsDeps.size(); i++) {
+                completeDeployDeps.put(filesAsDeps[i], deployDeps[i])
+            }
+        }
+        return completeDeployDeps
+    }
+
+    /**
+     * get all upstream projects for a gradle project. Iterate through all projects and recursively find all dependent projects
+     * @param project
+     * @return
+     */
+    public static Set<Project> getAllUpstreamProjects(Project project) {
+        Set<Project> allDependentProjects = new HashSet<>()
+
+        for (Iterator<Configuration> iter = project.getConfigurations().iterator(); iter.hasNext(); ) {
+            Configuration element = iter.next();
+            if (element.canBeResolved) {
+                Dependency[] deployDeps = element.getAllDependencies().toArray()
+                for (Dependency dependency1: deployDeps) {
+                    if (dependency1 instanceof ProjectDependency) {
+                        Project dependencyProject = dependency1.getDependencyProject()
+                        allDependentProjects.add(dependencyProject)
+                        allDependentProjects.addAll(getAllUpstreamProjects(dependencyProject))
+                    }
+                }
+            }
+        }
+        return allDependentProjects;
+    }
+
+    /**
+     * get web app source directories
+     * @param project
+     * @return
+     */
+    public static List<Path> getWebSourceDirectoriesToMonitor(Project project) {
+        List<Path> retVal = new ArrayList<Path>();
+        Task warTask = project.getTasks().findByName('war')
+        if (warTask != null) {
+            setWarSourceDir(warTask, retVal)
+        } else if (project.configurations.deploy != null) {
+            setWarSourceDirForDeployDependencies(project, retVal)
+        } else {
+            retVal.add("src/main/webapp")
+        }
+        return retVal;
+    }
+    /**
+     * find war deploy dependencies and add source dir
+     * @param project
+     * @param retVal
+     */
+    private static void setWarSourceDirForDeployDependencies(Project project, ArrayList<Path> retVal) {
+        Task warTask
+        HashMap<File, Dependency> completeDeployDeps = DevTaskHelper.getDeployDependencies(project)
+        for (Map.Entry<File, Dependency> entry : completeDeployDeps) {
+            Dependency dependency = entry.getValue();
+            File dependencyFile = entry.getKey();
+
+            if (dependency instanceof ProjectDependency) {
+                Project dependencyProject = dependency.getDependencyProject()
+                String projectType = FilenameUtils.getExtension(dependencyFile.toString())
+                switch (projectType) {
+                    case "war":
+                        warTask = dependencyProject.getTasks().findByName('war')
+                        if (warTask != null) {
+                            setWarSourceDir(warTask, retVal)
+                        }
+                        break;
+                    default:
+                        break;
+                }
+            }
+        }
+    }
+
+    private static void setWarSourceDir(Task warTask, ArrayList<Path> retVal) {
+        War war = (War) warTask.getProject().war
+        if (war.getWebAppDirectory().getAsFile().get() != null) {
+            retVal.add(war.getWebAppDirectory().get().asFile.toPath())
+        }
+    }
+}