[JENKINS-69135] Add a "Versions to include" field to the Global Library Cache feature

src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java

@@ -52,6 +52,7 @@
 import edu.umd.cs.findbugs.annotations.NonNull;
 import java.util.regex.Pattern;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
 import org.jenkinsci.plugins.workflow.cps.CpsFlowExecution;
 import org.jenkinsci.plugins.workflow.cps.GlobalVariable;
 import org.jenkinsci.plugins.workflow.cps.GlobalVariableSet;
@@ -210,7 +211,10 @@ static List<URL> retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev
             shouldCache = false;
         }
 
-        if(shouldCache) {
+        //If the included versions is blank/null, cache irrespective
+        //else check if that version is included and then cache only that version
+
+        if((shouldCache && cachingConfiguration.isIncluded(version)) || (shouldCache && StringUtils.isBlank(cachingConfiguration.getIncludedVersionsStr()))) {
             retrieveLock.readLock().lockInterruptibly();
             try {
                 CacheStatus cacheStatus = getCacheStatus(cachingConfiguration, versionCacheDir);
@@ -220,8 +224,8 @@ static List<URL> retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev
                     try {
                       boolean retrieve = false;
                       switch (getCacheStatus(cachingConfiguration, versionCacheDir)) {
-                          case VALID: 
-                              listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); 
+                          case VALID:
+                              listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home.");
                                 break;
                           case DOES_NOT_EXIST:
                               retrieve = true;
@@ -236,20 +240,20 @@ static List<URL> retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev
                                 retrieve = true;
                                 break;
                       }
-                            
+
                         if (retrieve) {
-                            listener.getLogger().println("Caching library " + name + "@" + version);                            
+                            listener.getLogger().println("Caching library " + name + "@" + version);
                             versionCacheDir.mkdirs();
                             retriever.retrieve(name, version, changelog, versionCacheDir, run, listener);
                         }
                         retrieveLock.readLock().lock();
                     } finally {
-                        retrieveLock.writeLock().unlock(); 
+                        retrieveLock.writeLock().unlock();
                     }
                 } else {
-                    listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home.");  
+                    listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home.");
                 }
-  
+
                 lastReadFile.touch(System.currentTimeMillis());
                 versionCacheDir.withSuffix("-name.txt").write(name, "UTF-8");
                 versionCacheDir.copyRecursiveTo(libDir);

src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java

@@ -29,16 +29,19 @@ public final class LibraryCachingConfiguration extends AbstractDescribableImpl<L
 
     private int refreshTimeMinutes;
     private String excludedVersionsStr;
+    private String includedVersionsStr;
 
     private static final String VERSIONS_SEPARATOR = " ";
     public static final String GLOBAL_LIBRARIES_DIR = "global-libraries-cache";
     public static final String LAST_READ_FILE = "last_read";
 
-    @DataBoundConstructor public LibraryCachingConfiguration(int refreshTimeMinutes, String excludedVersionsStr) {
+    @DataBoundConstructor public LibraryCachingConfiguration(int refreshTimeMinutes, String excludedVersionsStr, String includedVersionsStr) {
         this.refreshTimeMinutes = refreshTimeMinutes;
         this.excludedVersionsStr = excludedVersionsStr;
+        this.includedVersionsStr = includedVersionsStr;
     }
 
+
     public int getRefreshTimeMinutes() {
         return refreshTimeMinutes;
     }
@@ -54,6 +57,13 @@ public Boolean isRefreshEnabled() {
     public String getExcludedVersionsStr() {
         return excludedVersionsStr;
     }
+    public String getIncludedVersionsStr() {
+        if(StringUtils.isBlank(includedVersionsStr)){
+            return null;
+        }
+            return includedVersionsStr;
+    }
+
 
     private List<String> getExcludedVersions() {
         if (excludedVersionsStr == null) {
@@ -62,6 +72,13 @@ private List<String> getExcludedVersions() {
         return Arrays.asList(excludedVersionsStr.split(VERSIONS_SEPARATOR));
     }
 
+    private List<String> getIncludedVersions() {
+        if (includedVersionsStr == null) {
+            return Collections.emptyList();
+        }
+        return Arrays.asList(includedVersionsStr.split(VERSIONS_SEPARATOR));
+    }
+
     public Boolean isExcluded(String version) {
         // exit early if the version passed in is null or empty
         if (StringUtils.isBlank(version)) {
@@ -78,6 +95,22 @@ public Boolean isExcluded(String version) {
         return false;
     }
 
+    public Boolean isIncluded(String version) {
+        // exit early if the version passed in is null or empty
+        if (StringUtils.isBlank(version)) {
+            return false;
+        }
+        for (String it : getIncludedVersions()) {
+            // works on empty or null included versions
+            // and if the version contains the inclusion thus it can be
+            // anywhere in the string.
+            if (StringUtils.isNotBlank(it) && version.contains(it)){
+                return true;
+            }
+        }
+        return false;
+    }
+
     @Override public String toString() {
         return "LibraryCachingConfiguration{refreshTimeMinutes=" + refreshTimeMinutes + ", excludedVersions="
                 + excludedVersionsStr + '}';
@@ -129,6 +162,5 @@ public FormValidation doClearCache(@QueryParameter String name, @QueryParameter
             }
             return FormValidation.ok("The cache dir was deleted successfully.");
         }
-
     }
 }

src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly

@@ -31,6 +31,9 @@ THE SOFTWARE.
     <f:entry title="${%Versions to exclude}" field="excludedVersionsStr">
         <f:textbox />
     </f:entry>
+     <f:entry title="${%Versions to include}" field="includedVersionsStr">
+        <f:textbox />
+     </f:entry>
     <j:if test="${h.hasPermission(app.ADMINISTER)}">
         <f:entry title="${%Force clear cache}" field="forceDelete">
           <f:checkbox/>

src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html

@@ -0,0 +1,6 @@
+<div>
+    Space separated list of versions to include to allow caching via substring search using .contains() method. Ex: "release/ master".
+</div>
+<div>
+    Note: Excluded versions will always take precedence over included versions
+</div>

src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java

@@ -420,7 +420,7 @@ public class LibraryAdderTest {
                 new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true)));
         globalLib.setDefaultVersion("master");
         globalLib.setImplicit(true);
-        globalLib.setCachingConfiguration(new LibraryCachingConfiguration(60, ""));
+        globalLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "", ""));
         GlobalLibraries.get().setLibraries(Collections.singletonList(globalLib));
         // Create a folder library with the same name and which is also set up to enable caching.
         sampleRepo2.write("vars/folderLibVar.groovy", "def call() { jenkins.model.Jenkins.get().setSystemMessage('folder library') }");
@@ -430,7 +430,7 @@ public class LibraryAdderTest {
                 new SCMSourceRetriever(new GitSCMSource(null, sampleRepo2.toString(), "", "*", "", true)));
         folderLib.setDefaultVersion("master");
         folderLib.setImplicit(true);
-        folderLib.setCachingConfiguration(new LibraryCachingConfiguration(60, ""));
+        folderLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "", ""));
         Folder f = r.jenkins.createProject(Folder.class, "folder1");
         f.getProperties().add(new FolderLibraries(Collections.singletonList(folderLib)));
         // Create a job that uses the folder library, which will take precedence over the global library, since they have the same name.
@@ -493,7 +493,7 @@ public void parallelBuildsDontInterfereWithExpiredCache() throws Throwable {
                 new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true)));
         config.setDefaultVersion("master");
         config.setImplicit(true);
-        config.setCachingConfiguration(new LibraryCachingConfiguration(30, null));
+        config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null));
         GlobalLibraries.get().getLibraries().add(config);
         WorkflowJob p1 = r.createProject(WorkflowJob.class);
         WorkflowJob p2 = r.createProject(WorkflowJob.class);

src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java

@@ -59,7 +59,7 @@ public void smokes() throws Throwable {
                 new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true)));
         config.setDefaultVersion("master");
         config.setImplicit(true);
-        config.setCachingConfiguration(new LibraryCachingConfiguration(30, null));
+        config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null));
         GlobalLibraries.get().getLibraries().add(config);
         // Run build and check that cache gets created.
         WorkflowJob p = r.createProject(WorkflowJob.class);

src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java

@@ -61,31 +61,57 @@ public class LibraryCachingConfigurationTest {
     private static int NO_REFRESH_TIME_MINUTES = 0;
 
     private static String NULL_EXCLUDED_VERSION = null;
+    private static String NULL_INCLUDED_VERSION = null;
+
     private static String ONE_EXCLUDED_VERSION = "branch-1";
 
+    private static String ONE_INCLUDED_VERSION = "branch-1i";
+
+
     private static String MULTIPLE_EXCLUDED_VERSIONS_1 = "main";
+
+    private static String MULTIPLE_INCLUDED_VERSIONS_1 = "master";
+
     private static String MULTIPLE_EXCLUDED_VERSIONS_2 = "branch-2";
+
+    private static String MULTIPLE_INCLUDED_VERSIONS_2 = "branch-2i";
+
     private static String MULTIPLE_EXCLUDED_VERSIONS_3 = "branch-3";
+    private static String MULTIPLE_INCLUDED_VERSIONS_3 = "branch-3i";
+
 
     private static String SUBSTRING_EXCLUDED_VERSIONS_1 = "feature/test-substring-exclude";
+
+    private static String SUBSTRING_INCLUDED_VERSIONS_1 = "feature_include/test-substring";
+
     private static String SUBSTRING_EXCLUDED_VERSIONS_2 = "test-other-substring-exclude";
+    private static String SUBSTRING_INCLUDED_VERSIONS_2 = "test-other-substring-include";
+
 
     private static String MULTIPLE_EXCLUDED_VERSIONS =
         MULTIPLE_EXCLUDED_VERSIONS_1 + " " +
         MULTIPLE_EXCLUDED_VERSIONS_2 + " " +
         MULTIPLE_EXCLUDED_VERSIONS_3;
 
+    private static String MULTIPLE_INCLUDED_VERSIONS =
+            MULTIPLE_INCLUDED_VERSIONS_1 + " " +
+            MULTIPLE_INCLUDED_VERSIONS_2 + " " +
+            MULTIPLE_INCLUDED_VERSIONS_3;
+
     private static String SUBSTRING_EXCLUDED_VERSIONS =
         "feature/ other-substring";
 
+    private static String SUBSTRING_INCLUDED_VERSIONS =
+            "feature_include/ other-substring";
+
     private static String NEVER_EXCLUDED_VERSION = "never-excluded-version";
 
     @Before
     public void createCachingConfiguration() {
-        nullVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, NULL_EXCLUDED_VERSION);
-        oneVersionConfig = new LibraryCachingConfiguration(NO_REFRESH_TIME_MINUTES, ONE_EXCLUDED_VERSION);
-        multiVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, MULTIPLE_EXCLUDED_VERSIONS);
-        substringVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, SUBSTRING_EXCLUDED_VERSIONS);
+        nullVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, NULL_EXCLUDED_VERSION, NULL_INCLUDED_VERSION);
+        oneVersionConfig = new LibraryCachingConfiguration(NO_REFRESH_TIME_MINUTES, ONE_EXCLUDED_VERSION, ONE_INCLUDED_VERSION);
+        multiVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, MULTIPLE_EXCLUDED_VERSIONS, MULTIPLE_INCLUDED_VERSIONS);
+        substringVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, SUBSTRING_EXCLUDED_VERSIONS, SUBSTRING_INCLUDED_VERSIONS);
     }
 
     @Issue("JENKINS-66045") // NPE getting excluded versions
@@ -125,6 +151,15 @@ public void getExcludedVersionsStr() {
         assertThat(substringVersionConfig.getExcludedVersionsStr(), is(SUBSTRING_EXCLUDED_VERSIONS));
     }
 
+    @Test
+    @WithoutJenkins
+    public void getIncludedVersionsStr() {
+        assertThat(nullVersionConfig.getIncludedVersionsStr(), is(NULL_INCLUDED_VERSION));
+        assertThat(oneVersionConfig.getIncludedVersionsStr(), is(ONE_INCLUDED_VERSION));
+        assertThat(multiVersionConfig.getIncludedVersionsStr(), is(MULTIPLE_INCLUDED_VERSIONS));
+        assertThat(substringVersionConfig.getIncludedVersionsStr(), is(SUBSTRING_INCLUDED_VERSIONS));
+    }
+
     @Test
     @WithoutJenkins
     public void isExcluded() {
@@ -155,6 +190,24 @@ public void isExcluded() {
         assertFalse(substringVersionConfig.isExcluded(null));
     }
 
+    @Issue("JENKINS-69135") //"Versions to include" feature for caching
+    @Test
+    @WithoutJenkins
+    public void isIncluded() {
+        assertFalse(nullVersionConfig.isIncluded(NULL_INCLUDED_VERSION));
+        assertFalse(nullVersionConfig.isIncluded(""));
+
+        assertTrue(oneVersionConfig.isIncluded(ONE_INCLUDED_VERSION));
+
+        assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_1));
+        assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_2));
+        assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_3));
+
+        assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_1));
+        assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_2));
+
+    }
+
     @Test
     public void clearCache() throws Exception {
         sampleRepo.init();
@@ -165,7 +218,7 @@ public void clearCache() throws Exception {
                 new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true)));
         config.setDefaultVersion("master");
         config.setImplicit(true);
-        config.setCachingConfiguration(new LibraryCachingConfiguration(30, null));
+        config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null));
         GlobalLibraries.get().getLibraries().add(config);
         // Run build and check that cache gets created.
         WorkflowJob p = r.createProject(WorkflowJob.class);
@@ -182,4 +235,72 @@ public void clearCache() throws Exception {
         assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile()));
     }
 
+    //Test similar substrings in "Versions to include" & "Versions to exclude"
+    //Exclusion takes precedence
+    @Issue("JENKINS-69135") //"Versions to include" feature for caching
+    @Test
+    public void clearCacheConflict() throws Exception {
+        sampleRepo.init();
+        sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }");
+        sampleRepo.git("add", "vars");
+        sampleRepo.git("commit", "--message=init");
+        LibraryConfiguration config = new LibraryConfiguration("library",
+                new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true)));
+        config.setDefaultVersion("master");
+        config.setImplicit(true);
+        // Same version specified in both include and exclude version
+        //Exclude takes precedence
+        config.setCachingConfiguration(new LibraryCachingConfiguration(30, "master", "master"));
+        GlobalLibraries.get().getLibraries().add(config);
+        // Run build and check that cache gets created.
+        WorkflowJob p = r.createProject(WorkflowJob.class);
+        p.setDefinition(new CpsFlowDefinition("foo()", true));
+        WorkflowRun b = r.buildAndAssertSuccess(p);
+        LibrariesAction action = b.getAction(LibrariesAction.class);
+        LibraryRecord record = action.getLibraries().get(0);
+        FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName());
+        // Cache should not get created since the version is included in "Versions to exclude"
+        assertThat(new File(cache.getRemote()), not(anExistingDirectory()));
+        assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile()));
+    }
+
+    @Issue("JENKINS-69135") //"Versions to include" feature for caching
+    @Test
+    public void clearCacheIncludedVersion() throws Exception {
+        sampleRepo.init();
+        sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }");
+        sampleRepo.git("add", "vars");
+        sampleRepo.git("commit", "--message=init");
+        sampleRepo.git("branch", "test/include");
+        LibraryConfiguration config = new LibraryConfiguration("library",
+                new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true)));
+        config.setDefaultVersion("master");
+        config.setAllowVersionOverride(true);
+        config.setImplicit(false);
+        config.setCachingConfiguration(new LibraryCachingConfiguration(30, "", "test/include"));
+        GlobalLibraries.get().getLibraries().add(config);
+        // Run build and check that cache gets created.
+        WorkflowJob p = r.createProject(WorkflowJob.class);
+        p.setDefinition(new CpsFlowDefinition("library identifier: 'library', changelog:false\n\nfoo()", true));
+        WorkflowRun b = r.buildAndAssertSuccess(p);
+        WorkflowJob p2 = r.createProject(WorkflowJob.class);
+        p2.setDefinition(new CpsFlowDefinition("library identifier: 'library@test/include', changelog:false\n\nfoo()", true));
+        WorkflowRun b2 = r.buildAndAssertSuccess(p2);
+        LibrariesAction action = b.getAction(LibrariesAction.class);
+        LibraryRecord record = action.getLibraries().get(0);
+        LibrariesAction action2 = b2.getAction(LibrariesAction.class);
+        LibraryRecord record2 = action2.getLibraries().get(0);
+        FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName());
+        FilePath cache2 = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record2.getDirectoryName());
+        assertThat(new File(cache.getRemote()), not(anExistingDirectory()));
+        assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile()));
+        assertThat(new File(cache2.getRemote()), anExistingDirectory());
+        assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), anExistingFile());
+        // Clears cache for the entire library, until the "Delete specific cache version" feature in merged
+        // Clear the cache. TODO: Would be more realistic to set up security and use WebClient.
+        ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", false);
+        assertThat(new File(cache2.getRemote()), not(anExistingDirectory()));
+        assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), not(anExistingFile()));
+    }
+
 }