diff --git a/src/main/java/org/jenkins/plugins/lockableresources/LockStepExecution.java b/src/main/java/org/jenkins/plugins/lockableresources/LockStepExecution.java index fb1fe1ae2..45e5ed3eb 100644 --- a/src/main/java/org/jenkins/plugins/lockableresources/LockStepExecution.java +++ b/src/main/java/org/jenkins/plugins/lockableresources/LockStepExecution.java @@ -50,57 +50,95 @@ public boolean start() throws Exception { getContext().get(FlowNode.class).addAction(new PauseAction("Lock")); PrintStream logger = getContext().get(TaskListener.class).getLogger(); - logger.println("Trying to acquire lock on [" + step + "]"); + + Run run = getContext().get(Run.class); + LockableResourcesManager.printLogs("Trying to acquire lock on [" + step + "]", Level.FINE, LOGGER, logger); List resourceHolderList = new ArrayList<>(); - for (LockStepResource resource : step.getResources()) { - List resources = new ArrayList<>(); - if (resource.resource != null) { - if (LockableResourcesManager.get().createResource(resource.resource)) { - logger.println("Resource [" + resource + "] did not exist. Created."); + LockableResourcesManager lrm = LockableResourcesManager.get(); + List available = null; + LinkedHashMap> resourceNames = new LinkedHashMap<>(); + synchronized (lrm.syncResources) { + for (LockStepResource resource : step.getResources()) { + List resources = new ArrayList<>(); + if (resource.resource != null) { + if (lrm.createResource(resource.resource)) { + LockableResourcesManager.printLogs( + "Resource [" + resource.resource + "] did not exist. Created.", + Level.INFO, + LOGGER, + logger); + } + resources.add(resource.resource); } - resources.add(resource.resource); + resourceHolderList.add(new LockableResourcesStruct(resources, resource.label, resource.quantity)); } - resourceHolderList.add(new LockableResourcesStruct(resources, resource.label, resource.quantity)); - } - // determine if there are enough resources available to proceed - List available = LockableResourcesManager.get() - .checkResourcesAvailability( - resourceHolderList, logger, null, step.skipIfLocked, resourceSelectStrategy); - Run run = getContext().get(Run.class); + // determine if there are enough resources available to proceed + available = lrm.getAvailableResources(resourceHolderList, logger, resourceSelectStrategy); + if (available == null || available.isEmpty()) { + LOGGER.fine("No available resources: " + available); + onLockFailed(logger, resourceHolderList); + return false; + } - if (available == null - || !LockableResourcesManager.get() - .lock(available, run, getContext(), step.toString(), step.variable, step.inversePrecedence)) { - // No available resources, or we failed to lock available resources - // if the resource is known, we could output the active/blocking job/build - LockableResource resource = LockableResourcesManager.get().fromName(step.resource); - boolean buildNameKnown = resource != null && resource.getBuildName() != null; - if (step.skipIfLocked) { - if (buildNameKnown) { - logger.println( - "[" + step + "] is locked by " + resource.getBuildName() + ", skipping execution..."); - } else { - logger.println("[" + step + "] is locked, skipping execution..."); - } - getContext().onSuccess(null); + final boolean lockFailed = (lrm.lock(available, run) == false); + + if (lockFailed) { + // this here is very defensive code, and you will probably never hit it. (hopefully) + LOGGER.warning("Internal program error: Can not lock resources: " + available); + onLockFailed(logger, resourceHolderList); return true; - } else { - if (buildNameKnown) { - logger.println("[" + step + "] is locked by " + resource.getBuildName() + ", waiting..."); - } else { - logger.println("[" + step + "] is locked, waiting..."); - } - LockableResourcesManager.get() - .queueContext(getContext(), resourceHolderList, step.toString(), step.variable); } - } // proceed is called inside lock if execution is possible + + // since LockableResource contains transient variables, they cannot be correctly serialized + // hence we use their unique resource names and properties + for (LockableResource resource : available) { + resourceNames.put(resource.getName(), resource.getProperties()); + } + } + LockStepExecution.proceed(resourceNames, getContext(), step.toString(), step.variable, step.inversePrecedence); return false; } + // --------------------------------------------------------------------------- + /** + * Executed when the lock() function fails. No available resources, or we failed to lock available + * resources if the resource is known, we could output the active/blocking job/build + */ + private void onLockFailed(PrintStream logger, List resourceHolderList) { + + if (step.skipIfLocked) { + this.printBlockCause(logger, resourceHolderList); + LockableResourcesManager.printLogs( + "[" + step + "] is not free, skipping execution ...", Level.INFO, LOGGER, logger); + getContext().onSuccess(null); + } else { + this.printBlockCause(logger, resourceHolderList); + LockableResourcesManager.printLogs( + "[" + step + "] is not free, waiting for execution ...", Level.INFO, LOGGER, logger); + LockableResourcesManager lrm = LockableResourcesManager.get(); + lrm.queueContext(getContext(), resourceHolderList, step.toString(), step.variable); + } + } + + private void printBlockCause(PrintStream logger, List resourceHolderList) { + LockableResourcesManager lrm = LockableResourcesManager.get(); + LockableResource resource = this.step.resource != null ? lrm.fromName(this.step.resource) : null; + + if (resource != null) { + final String logMessage = resource.getLockCauseDetail(); + if (logMessage != null && !logMessage.isEmpty()) + LockableResourcesManager.printLogs(logMessage, Level.INFO, LOGGER, logger); + } else { + // looks like ordered by label + lrm.getAvailableResources(resourceHolderList, logger, null); + } + } + + // --------------------------------------------------------------------------- @SuppressFBWarnings(value = "REC_CATCH_EXCEPTION", justification = "not sure which exceptions might be catch.") public static void proceed( final LinkedHashMap> lockedResources, @@ -109,17 +147,19 @@ public static void proceed( final String variable, boolean inversePrecedence) { Run build; - FlowNode node; + FlowNode node = null; + PrintStream logger = null; try { build = context.get(Run.class); node = context.get(FlowNode.class); - context.get(TaskListener.class).getLogger().println("Lock acquired on [" + resourceDescription + "]"); + logger = context.get(TaskListener.class).getLogger(); + LockableResourcesManager.printLogs( + "Lock acquired on [" + resourceDescription + "]", Level.INFO, LOGGER, logger); } catch (Exception e) { context.onFailure(e); return; } - LOGGER.finest("Lock acquired on [" + resourceDescription + "] by " + build.getExternalizableId()); try { LockedResourcesBuildAction.updateAction(build, new ArrayList<>(lockedResources.keySet())); @@ -127,7 +167,7 @@ public static void proceed( BodyInvoker bodyInvoker = context.newBodyInvoker() .withCallback(new Callback( new ArrayList<>(lockedResources.keySet()), resourceDescription, inversePrecedence)); - if (variable != null && variable.length() > 0) { + if (variable != null && !variable.isEmpty()) { // set the variable for the duration of the block bodyInvoker.withContext( EnvironmentExpander.merge(context.get(EnvironmentExpander.class), new EnvironmentExpander() { @@ -161,6 +201,7 @@ public void expand(@NonNull EnvVars env) { } bodyInvoker.start(); } catch (IOException | InterruptedException e) { + LOGGER.warning("proceed done with failure " + resourceDescription); throw new RuntimeException(e); } } @@ -182,10 +223,11 @@ private static final class Callback extends BodyExecutionCallback.TailCall { protected void finished(StepContext context) throws Exception { LockableResourcesManager.get() .unlockNames(this.resourceNames, context.get(Run.class), this.inversePrecedence); - context.get(TaskListener.class) - .getLogger() - .println("Lock released on resource [" + resourceDescription + "]"); - LOGGER.finest("Lock released on [" + resourceDescription + "]"); + LockableResourcesManager.printLogs( + "Lock released on resource [" + resourceDescription + "]", + Level.INFO, + LOGGER, + context.get(TaskListener.class).getLogger()); } } @@ -195,8 +237,7 @@ public void stop(@NonNull Throwable cause) { if (!cleaned) { LOGGER.log( Level.WARNING, - "Cannot remove context from lockable resource waiting list. " - + "The context is not in the waiting list."); + "Cannot remove context from lockable resource waiting list. The context is not in the waiting list."); } getContext().onFailure(cause); } diff --git a/src/main/java/org/jenkins/plugins/lockableresources/LockableResourcesManager.java b/src/main/java/org/jenkins/plugins/lockableresources/LockableResourcesManager.java index 310c2fd13..0dafaa18a 100644 --- a/src/main/java/org/jenkins/plugins/lockableresources/LockableResourcesManager.java +++ b/src/main/java/org/jenkins/plugins/lockableresources/LockableResourcesManager.java @@ -19,7 +19,6 @@ import hudson.Util; import hudson.console.ModelHyperlinkNote; import hudson.model.Run; -import hudson.model.TaskListener; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; @@ -41,7 +40,6 @@ import jenkins.util.SystemProperties; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; -import org.jenkins.plugins.lockableresources.queue.LockableResourcesCandidatesStruct; import org.jenkins.plugins.lockableresources.queue.LockableResourcesStruct; import org.jenkins.plugins.lockableresources.queue.QueuedContextStruct; import org.jenkins.plugins.lockableresources.util.Constants; @@ -61,12 +59,15 @@ public class LockableResourcesManager extends GlobalConfiguration { private List resources; private transient Cache> cachedCandidates = CacheBuilder.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES).build(); + private static final Logger LOGGER = Logger.getLogger(LockableResourcesManager.class.getName()); /** * Only used when this lockable resource is tried to be locked by {@link LockStep}, otherwise * (freestyle builds) regular Jenkins queue is used. */ private List queuedContexts = new ArrayList<>(); + // remember last processed queue index + private transient int lastCheckedQueueIndex = -1; // cache to enable / disable saving lockable-resources state private int enableSave = -1; @@ -317,15 +318,44 @@ public List getResourcesMatchingScript( return found; } - public synchronized LockableResource fromName(String resourceName) { + // --------------------------------------------------------------------------- + /** Returns resource matched by name. Returns null in case, the resource does not exists. */ + @CheckForNull + @Restricted(NoExternalUse.class) + public LockableResource fromName(@CheckForNull String resourceName) { + resourceName = Util.fixEmpty(resourceName); + if (resourceName != null) { - for (LockableResource r : resources) { + + for (LockableResource r : this.getReadOnlyResources()) { if (resourceName.equals(r.getName())) return r; } + } else { + LOGGER.warning("Internal failure, fromName is empty or null:" + getStack()); } return null; } + // --------------------------------------------------------------------------- + @Restricted(NoExternalUse.class) + public List fromNames(final List names) { + return fromNames(names, false); + } + + // --------------------------------------------------------------------------- + @Restricted(NoExternalUse.class) + public List fromNames(final List names, final boolean createResource) { + List list = new ArrayList<>(); + for (String name : names) { + // be sure it exists + if (createResource) this.createResource(name); + LockableResource r = this.fromName(name); + if (r != null) // this is probably bug, but nobody know + list.add(r); + } + return list; + } + // --------------------------------------------------------------------------- private String getStack() { StringBuffer buf = new StringBuffer(); @@ -344,7 +374,7 @@ public boolean resourceExist(@CheckForNull String resourceName) { } // --------------------------------------------------------------------------- - public synchronized boolean queue(List resources, long queueItemId, String queueProjectName) { + public boolean queue(List resources, long queueItemId, String queueProjectName) { for (LockableResource r : resources) { if (r.isReserved() || r.isQueued(queueItemId) || r.isLocked()) { return false; @@ -356,15 +386,17 @@ public synchronized boolean queue(List resources, long queueIt return true; } + // --------------------------------------------------------------------------- /** - * @deprecated USe {@link + * @deprecated Use {@link * #tryQueue(org.jenkins.plugins.lockableresources.queue.LockableResourcesStruct, long, * java.lang.String, int, java.util.Map, java.util.logging.Logger)} */ @Deprecated @CheckForNull @ExcludeFromJacocoGeneratedReport - public synchronized List queue( + @Restricted(NoExternalUse.class) + public List queue( LockableResourcesStruct requiredResources, long queueItemId, String queueItemProject, @@ -383,6 +415,7 @@ public synchronized List queue( } } + // --------------------------------------------------------------------------- /** * If the lockable resource availability was evaluated before and cached to avoid frequent * re-evaluations under queued pressure when there are no resources to give, we should state that @@ -397,7 +430,7 @@ public synchronized List queue( * resource (LR instance) directly to unlock/unreserve, it has no idea to clean itself from this * cache, and may be considered busy in queuing for some time afterwards. */ - public synchronized boolean uncacheIfFreeing(LockableResource candidate, boolean unlocking, boolean unreserving) { + public boolean uncacheIfFreeing(LockableResource candidate, boolean unlocking, boolean unreserving) { if (candidate.isLocked() && !unlocking) return false; // "stolen" state helps track that a resource is currently not @@ -423,6 +456,7 @@ public synchronized boolean uncacheIfFreeing(LockableResource candidate, boolean return true; } + // --------------------------------------------------------------------------- /** * Try to acquire the resources required by the task. * @@ -434,7 +468,8 @@ public synchronized boolean uncacheIfFreeing(LockableResource candidate, boolean * @since 2.0 */ @CheckForNull - public synchronized List tryQueue( + @Restricted(NoExternalUse.class) + public List tryQueue( LockableResourcesStruct requiredResources, long queueItemId, String queueItemProject, @@ -443,68 +478,72 @@ public synchronized List tryQueue( Logger log) throws ExecutionException { List selected = new ArrayList<>(); + synchronized (this.syncResources) { + if (!checkCurrentResourcesStatus(selected, queueItemProject, queueItemId, log)) { + // The project has another buildable item waiting -> bail out + log.log( + Level.FINEST, + "{0} has another build waiting resources." + " Waiting for it to proceed first.", + new Object[] {queueItemProject}); + return null; + } - if (!checkCurrentResourcesStatus(selected, queueItemProject, queueItemId, log)) { - // The project has another buildable item waiting -> bail out - log.log( - Level.FINEST, - "{0} has another build waiting resources." + " Waiting for it to proceed first.", - new Object[] {queueItemProject}); - return null; - } + final SecureGroovyScript systemGroovyScript = requiredResources.getResourceMatchScript(); + boolean candidatesByScript = (systemGroovyScript != null); + List candidates = requiredResources.required; // default candidates - final SecureGroovyScript systemGroovyScript = requiredResources.getResourceMatchScript(); - boolean candidatesByScript = (systemGroovyScript != null); - List candidates = requiredResources.required; // default candidates + if (candidatesByScript || (requiredResources.label != null && !requiredResources.label.isEmpty())) { - if (candidatesByScript || (requiredResources.label != null && !requiredResources.label.isEmpty())) { - candidates = cachedCandidates.getIfPresent(queueItemId); - if (candidates != null) { - candidates.retainAll(resources); - } else { - candidates = (systemGroovyScript == null) - ? getResourcesWithLabel(requiredResources.label) - : getResourcesMatchingScript(systemGroovyScript, params); - cachedCandidates.put(queueItemId, candidates); + candidates = cachedCandidates.getIfPresent(queueItemId); + if (candidates != null) { + candidates.retainAll(this.resources); + } else { + candidates = (systemGroovyScript == null) + ? getResourcesWithLabel(requiredResources.label) + : getResourcesMatchingScript(systemGroovyScript, params); + cachedCandidates.put(queueItemId, candidates); + } } - } - for (LockableResource rs : candidates) { - if (number != 0 && (selected.size() >= number)) break; - if (!rs.isReserved() && !rs.isLocked() && !rs.isQueued()) selected.add(rs); - } + for (LockableResource rs : candidates) { + if (number != 0 && (selected.size() >= number)) break; + if (!rs.isReserved() && !rs.isLocked() && !rs.isQueued()) selected.add(rs); + } - // if did not get wanted amount or did not get all - final int required_amount; - if (candidatesByScript && candidates.isEmpty()) { - /* - * If the groovy script does not return any candidates, it means nothing is needed, even if a - * higher amount is specified. A valid use case is a Matrix job, when not all configurations - * need resources. - */ - required_amount = 0; - } else { - required_amount = number == 0 ? candidates.size() : number; - } + // if did not get wanted amount or did not get all + final int required_amount; + if (candidatesByScript && candidates.isEmpty()) { + /* + * If the groovy script does not return any candidates, it means nothing is needed, even if a + * higher amount is specified. A valid use case is a Matrix job, when not all configurations + * need resources. + */ + required_amount = 0; + } else { + required_amount = number == 0 ? candidates.size() : number; + } - if (selected.size() != required_amount) { - log.log( - Level.FINEST, - "{0} found {1} resource(s) to queue." + "Waiting for correct amount: {2}.", - new Object[] {queueItemProject, selected.size(), required_amount}); - // just to be sure, clean up - for (LockableResource x : resources) { - if (x.getQueueItemProject() != null && x.getQueueItemProject().equals(queueItemProject)) x.unqueue(); + if (selected.size() != required_amount) { + log.log( + Level.FINEST, + "{0} found {1} resource(s) to queue." + "Waiting for correct amount: {2}.", + new Object[] {queueItemProject, selected.size(), required_amount}); + // just to be sure, clean up + for (LockableResource x : this.resources) { + if (x.getQueueItemProject() != null + && x.getQueueItemProject().equals(queueItemProject)) x.unqueue(); + } + return null; } - return null; - } - for (LockableResource rsc : selected) { - rsc.setQueued(queueItemId, queueItemProject); + for (LockableResource rsc : selected) { + rsc.setQueued(queueItemId, queueItemProject); + } } return selected; } + // --------------------------------------------------------------------------- // Adds already selected (in previous queue round) resources to 'selected' // Return false if another item queued for this project -> bail out private boolean checkCurrentResourcesStatus( @@ -529,197 +568,169 @@ private boolean checkCurrentResourcesStatus( return true; } - public synchronized boolean lock(List resources, Run build, @Nullable StepContext context) { - return lock(resources, build, context, null, null, false); - } - - @Restricted(NoExternalUse.class) - public synchronized boolean lock(List resources, Run build) { - return lock(resources, build, null); + // --------------------------------------------------------------------------- + @Deprecated + public boolean lock(List resources, Run build, @Nullable StepContext context) { + return this.lock(resources, build); } - /** Try to lock the resource and return true if locked. */ - public synchronized boolean lock( + // --------------------------------------------------------------------------- + @Deprecated + public boolean lock( List resources, Run build, @Nullable StepContext context, @Nullable String logmessage, final String variable, boolean inversePrecedence) { - boolean needToWait = false; + return this.lock(resources, build); + } + + // --------------------------------------------------------------------------- + /** Try to lock the resource and return true if locked. */ + public boolean lock(List resources, Run build) { + + LOGGER.fine("lock it: " + resources + " for build " + build); + + if (build == null) { + LOGGER.warning("lock() will fails, because the build does not exits. " + resources); + return false; // not locked + } + + String cause = getCauses(resources); + if (!cause.isEmpty()) { + LOGGER.warning("lock() for build " + build + " will fails, because " + cause); + return false; // not locked + } for (LockableResource r : resources) { - if (r.isReserved() || r.isLocked()) { - needToWait = true; - break; - } + r.unqueue(); + r.setBuild(build); } - if (!needToWait) { - for (LockableResource r : resources) { - r.unqueue(); - r.setBuild(build); - } - if (context != null) { - // since LockableResource contains transient variables, they cannot be correctly serialized - // hence we use their unique resource names and properties - LinkedHashMap> resourceNames = new LinkedHashMap<>(); - for (LockableResource resource : resources) { - resourceNames.put(resource.getName(), resource.getProperties()); - } - LockStepExecution.proceed(resourceNames, context, logmessage, variable, inversePrecedence); + save(); + + return true; + } + + // --------------------------------------------------------------------------- + private void freeResources(List unlockResources, @Nullable Run build) { + LOGGER.fine("free it: " + unlockResources); + for (LockableResource resource : unlockResources) { + // No more contexts, unlock resource + if (build != null && build != resource.getBuild()) { + continue; // this happens, when you push the unlock button in LRM page } - save(); - } + resource.unqueue(); + resource.setBuild(null); + uncacheIfFreeing(resource, true, false); - return !needToWait; - } - - private synchronized void freeResources(List unlockResourceNames, @Nullable Run build) { - for (String unlockResourceName : unlockResourceNames) { - Iterator resourceIterator = this.resources.iterator(); - while (resourceIterator.hasNext()) { - LockableResource resource = resourceIterator.next(); - if (resource != null - && resource.getName() != null - && resource.getName().equals(unlockResourceName)) { - if (build == null - || (resource.getBuild() != null - && build.getExternalizableId() - .equals(resource.getBuild().getExternalizableId()))) { - // No more contexts, unlock resource - resource.unqueue(); - resource.setBuild(null); - uncacheIfFreeing(resource, true, false); - if (resource.isEphemeral()) { - resourceIterator.remove(); - } - } - } + if (resource.isEphemeral()) { + LOGGER.info("Remove ephemeral resource: " + resource); + this.resources.remove(resource); } } } - public synchronized void unlock(List resourcesToUnLock, @Nullable Run build) { + // --------------------------------------------------------------------------- + public void unlock(List resourcesToUnLock, @Nullable Run build) { unlock(resourcesToUnLock, build, false); } - public synchronized void unlock( + // --------------------------------------------------------------------------- + public void unlock( @Nullable List resourcesToUnLock, @Nullable Run build, boolean inversePrecedence) { - List resourceNamesToUnLock = new ArrayList<>(); - if (resourcesToUnLock != null) { - for (LockableResource r : resourcesToUnLock) { - resourceNamesToUnLock.add(r.getName()); - } - } - + List resourceNamesToUnLock = LockableResourcesManager.getResourcesNames(resourcesToUnLock); this.unlockNames(resourceNamesToUnLock, build, inversePrecedence); } + // --------------------------------------------------------------------------- @SuppressFBWarnings(value = "REC_CATCH_EXCEPTION", justification = "not sure which exceptions might be catch.") - public synchronized void unlockNames( + public void unlockNames( @Nullable List resourceNamesToUnLock, @Nullable Run build, boolean inversePrecedence) { // make sure there is a list of resource names to unlock if (resourceNamesToUnLock == null || resourceNamesToUnLock.isEmpty()) { return; } - // process as many contexts as possible - List remainingResourceNamesToUnLock = new ArrayList<>(resourceNamesToUnLock); - - QueuedContextStruct nextContext = null; - while (!remainingResourceNamesToUnLock.isEmpty()) { - // check if there are resources which can be unlocked (and shall not be unlocked) - nextContext = this.getNextQueuedContext(remainingResourceNamesToUnLock, inversePrecedence, nextContext); - - // no context is queued which can be started once these resources are free'd. - if (nextContext == null) { - this.freeResources(remainingResourceNamesToUnLock, build); - save(); - return; - } - - List requiredResourceForNextContext = - checkResourcesAvailability(nextContext.getResources(), null, remainingResourceNamesToUnLock); - - // resourceNamesToUnlock contains the names of the previous resources. - // requiredResourceForNextContext contains the resource objects which are required for the - // next context. - // It is guaranteed that there is an overlap between the two - the resources which are to be - // reused. - boolean needToWait = false; - for (LockableResource requiredResource : requiredResourceForNextContext) { - if (requiredResource.isStolen()) { - needToWait = true; - break; - } - if (!remainingResourceNamesToUnLock.contains(requiredResource.getName())) { - if (requiredResource.isReserved() || requiredResource.isLocked()) { - needToWait = true; + synchronized (this.syncResources) { + this.freeResources(this.fromNames(resourceNamesToUnLock), build); + + // process as many contexts as possible + this.lastCheckedQueueIndex = -1; + while (resourceNamesToUnLock.size() > 0 && proceedNextContext(inversePrecedence)) { + for (String resourceName : resourceNamesToUnLock) { + LockableResource r = fromName(resourceName); + if (r == null) { + // probably it was ephemeral resource and does not exists now + // therefore we need to check the whole queue later break; } - } - } - - if (!needToWait) { - // remove context from queue and process it - unqueueContext(nextContext.getContext()); - - LinkedHashMap> resourcesToLock = new LinkedHashMap<>(); - - // lock all (old and new resources) - for (LockableResource requiredResource : requiredResourceForNextContext) { - Run nextBuild = nextContext.getBuild(); - if (nextBuild == null) { - // skip this context, as the build cannot be retrieved (maybe it was deleted while - // running?) - LOGGER.warning( - "Skipping queued context for lock. Cannot get the Run object from the context to " - + "proceed with lock; this could be a legitimate state if the build waiting " - + "for the lock was deleted or hard killed. More information is logged at " - + "Level.FINE for debugging purposes."); - unlockNames(remainingResourceNamesToUnLock, null, inversePrecedence); - return; - } else { - requiredResource.setBuild(nextBuild); - resourcesToLock.put(requiredResource.getName(), requiredResource.getProperties()); + if (!r.isFree()) { + // i sno more free, that means, we does not need to check it in the queue now + resourceNamesToUnLock.remove(resourceName); + break; } } + } - // determine old resources no longer needed - List freeResources = new ArrayList<>(); - for (String resourceNameToUnlock : remainingResourceNamesToUnLock) { - boolean resourceStillNeeded = false; - for (LockableResource requiredResource : requiredResourceForNextContext) { - if (resourceNameToUnlock != null && resourceNameToUnlock.equals(requiredResource.getName())) { - resourceStillNeeded = true; - break; - } - } + save(); + } + } - if (!resourceStillNeeded) { - freeResources.add(resourceNameToUnlock); - } - } + private boolean proceedNextContext(boolean inversePrecedence) { + LOGGER.finest("inversePrecedence: " + inversePrecedence); + QueuedContextStruct nextContext = this.getNextQueuedContext(inversePrecedence); + LOGGER.finest("nextContext: " + nextContext); + // no context is queued which can be started once these resources are free'd. + if (nextContext == null) { + LOGGER.fine("No context is queued which can be started once these resources are free'd."); + return false; + } + LOGGER.finest("nextContext candidates: " + nextContext.candidates); + List requiredResourceForNextContext = + this.fromNames(nextContext.candidates, /*create un-existent resources */ true); + LOGGER.finest("nextContext real candidates: " + requiredResourceForNextContext); + // remove context from queue and process it - // keep unused resources - remainingResourceNamesToUnLock.retainAll(freeResources); + Run build = nextContext.getBuild(); + if (build == null) { + // this shall never happens + // skip this context, as the build cannot be retrieved (maybe it was deleted while + // running?) + LOGGER.warning("Skip this context, as the build cannot be retrieved"); + return true; + } + boolean locked = this.lock(requiredResourceForNextContext, build); + if (!locked) { + // defensive line, shall never happens + LOGGER.warning("Can not lock resources: " + requiredResourceForNextContext); + // to eliminate possible endless loop + return false; + } - // continue with next context - LockStepExecution.proceed( - resourcesToLock, - nextContext.getContext(), - nextContext.getResourceDescription(), - nextContext.getVariableName(), - inversePrecedence); - } + // build env vars + LinkedHashMap> resourcesToLock = new LinkedHashMap<>(); + for (LockableResource requiredResource : requiredResourceForNextContext) { + resourcesToLock.put(requiredResource.getName(), requiredResource.getProperties()); } - save(); + + this.unqueueContext(nextContext.getContext()); + + // continue with next context + LockStepExecution.proceed( + resourcesToLock, + nextContext.getContext(), + nextContext.getResourceDescription(), + nextContext.getVariableName(), + inversePrecedence); + return true; } + // --------------------------------------------------------------------------- /** Returns names (IDs) of given *resources*. */ @Restricted(NoExternalUse.class) - public static List getResourcesNames(List resources) { + public static List getResourcesNames(final List resources) { List resourceNames = new ArrayList<>(); if (resources != null) { for (LockableResource resource : resources) { @@ -738,15 +749,7 @@ public List getAllResourcesNames() { } } - /** - * @see #getNextQueuedContext(List, List, boolean, QueuedContextStruct) - */ - @CheckForNull - private QueuedContextStruct getNextQueuedContext( - List resourceNamesToUnLock, boolean inversePrecedence, QueuedContextStruct from) { - return this.getNextQueuedContext(resourceNamesToUnLock, null, inversePrecedence, from); - } - + // --------------------------------------------------------------------------- /** * Returns the next queued context with all its requirements satisfied. * @@ -759,47 +762,69 @@ private QueuedContextStruct getNextQueuedContext( * @return the context or null */ @CheckForNull - private QueuedContextStruct getNextQueuedContext( - @Nullable List resourceNamesToUnLock, - @Nullable List resourceNamesToUnReserve, - boolean inversePrecedence, - QueuedContextStruct from) { - QueuedContextStruct newestEntry = null; - int fromIndex = from != null ? this.queuedContexts.indexOf(from) + 1 : 0; - if (!inversePrecedence) { - for (int i = fromIndex; i < this.queuedContexts.size(); i++) { - QueuedContextStruct entry = this.queuedContexts.get(i); - if (checkResourcesAvailability( - entry.getResources(), null, resourceNamesToUnLock, resourceNamesToUnReserve) - != null) { - return entry; + private QueuedContextStruct getNextQueuedContext(boolean inversePrecedence) { + + LOGGER.fine("current queue size: " + this.queuedContexts.size()); + LOGGER.finest("current queue: " + this.queuedContexts); + List orphan = new ArrayList<>(); + QueuedContextStruct nextEntry = null; + if (inversePrecedence) { + // the last one added lock ist the newest one, and this wins + if (this.lastCheckedQueueIndex == -1) { + this.lastCheckedQueueIndex = this.queuedContexts.size() - 1; + } else this.lastCheckedQueueIndex++; + for (; this.lastCheckedQueueIndex >= 0 && nextEntry == null; this.lastCheckedQueueIndex--) { + QueuedContextStruct entry = this.queuedContexts.get(this.lastCheckedQueueIndex); + // check queue list first + if (!entry.isValid()) { + orphan.add(entry); + continue; } + LOGGER.finest("inversePrecedence - index: " + this.lastCheckedQueueIndex + " " + entry); + + nextEntry = getNextQueuedContextEntry(entry); } } else { - long newest = 0; - List orphan = new ArrayList<>(); - for (int i = fromIndex; i < this.queuedContexts.size(); i++) { - QueuedContextStruct entry = this.queuedContexts.get(i); - if (checkResourcesAvailability( - entry.getResources(), null, resourceNamesToUnLock, resourceNamesToUnReserve) - != null) { - - Run run = entry.getBuild(); - if (run == null) { - // skip this one, for some reason there is no Run object for this context - orphan.add(entry); - } else if (run.getStartTimeInMillis() > newest) { - newest = run.getStartTimeInMillis(); - newestEntry = entry; - } + // the first one added lock is the oldest one, and this wins + if (this.lastCheckedQueueIndex == -1) { + this.lastCheckedQueueIndex = 0; + } else this.lastCheckedQueueIndex--; + for (; + this.lastCheckedQueueIndex < this.queuedContexts.size() && nextEntry == null; + this.lastCheckedQueueIndex++) { + QueuedContextStruct entry = this.queuedContexts.get(this.lastCheckedQueueIndex); + // check queue list first + if (!entry.isValid()) { + orphan.add(entry); + continue; } + LOGGER.finest("oldest win - index: " + this.lastCheckedQueueIndex + " " + entry); + + nextEntry = getNextQueuedContextEntry(entry); } - if (!orphan.isEmpty()) { - this.queuedContexts.removeAll(orphan); - } } - return newestEntry; + if (!orphan.isEmpty()) { + this.queuedContexts.removeAll(orphan); + this.lastCheckedQueueIndex = -1; + } + + if (nextEntry == null) { + this.lastCheckedQueueIndex = -1; + } + return nextEntry; + } + + // --------------------------------------------------------------------------- + QueuedContextStruct getNextQueuedContextEntry(QueuedContextStruct entry) { + List candidates = this.getAvailableResources(entry.getResources()); + if (candidates == null || candidates.isEmpty()) { + return null; + } + + entry.candidates = getResourcesNames(candidates); + LOGGER.fine("take this: " + entry); + return entry; } // --------------------------------------------------------------------------- @@ -881,53 +906,63 @@ public boolean addResource(@Nullable final LockableResource resource, final bool return true; } + // --------------------------------------------------------------------------- /** * Reserves an available resource for the userName indefinitely (until that person, or some * explicit scripted action, decides to release the resource). */ - public synchronized boolean reserve(List resources, String userName) { - for (LockableResource r : resources) { - if (!r.isFree()) { - return false; + public boolean reserve(List resources, String userName) { + synchronized (this.syncResources) { + for (LockableResource r : resources) { + if (!r.isFree()) { + return false; + } } + for (LockableResource r : resources) { + r.reserve(userName); + } + save(); } - for (LockableResource r : resources) { - r.reserve(userName); - } - save(); return true; } + // --------------------------------------------------------------------------- /** * Reserves a resource that may be or not be locked by some job (or reserved by some user) * already, giving it away to the userName indefinitely (until that person, or some explicit * scripted action, later decides to release the resource). */ - public synchronized boolean steal(List resources, String userName) { - for (LockableResource r : resources) { - r.setReservedBy(userName); - r.setStolen(); + public boolean steal(List resources, String userName) { + synchronized (this.syncResources) { + for (LockableResource r : resources) { + r.setReservedBy(userName); + r.setStolen(); + } + unlock(resources, null, false); + save(); } - unlock(resources, null, false); - save(); return true; } + // --------------------------------------------------------------------------- /** * Reserves a resource that may be or not be reserved by some person already, giving it away to * the userName indefinitely (until that person, or some explicit scripted action, decides to * release the resource). */ - public synchronized void reassign(List resources, String userName) { - for (LockableResource r : resources) { - if (!r.isFree()) { - r.unReserve(); + public void reassign(List resources, String userName) { + synchronized (this.syncResources) { + for (LockableResource r : resources) { + if (!r.isFree()) { + r.unReserve(); + } + r.setReservedBy(userName); } - r.setReservedBy(userName); + save(); } - save(); } + // --------------------------------------------------------------------------- private void unreserveResources(@NonNull List resources) { for (LockableResource l : resources) { uncacheIfFreeing(l, false, true); @@ -936,112 +971,43 @@ private void unreserveResources(@NonNull List resources) { save(); } + // --------------------------------------------------------------------------- @SuppressFBWarnings(value = "REC_CATCH_EXCEPTION", justification = "not sure which exceptions might be catch.") - public synchronized void unreserve(List resources) { + public void unreserve(List resources) { // make sure there is a list of resources to unreserve if (resources == null || resources.isEmpty()) { return; } - List resourceNamesToUnreserve = new ArrayList<>(); - for (LockableResource r : resources) { - resourceNamesToUnreserve.add(r.getName()); - } - - // check if there are resources which can be unlocked (and shall not be unlocked) - QueuedContextStruct nextContext = this.getNextQueuedContext(null, resourceNamesToUnreserve, false, null); - // no context is queued which can be started once these resources are free'd. - if (nextContext == null) { - LOGGER.log( - Level.FINER, - () -> "No context queued for resources " - + String.join(", ", resourceNamesToUnreserve) - + " so unreserving and proceeding."); + synchronized (this.syncResources) { + LOGGER.fine("unreserve " + resources); unreserveResources(resources); - return; - } - - PrintStream nextContextLogger = null; - try { - TaskListener nextContextTaskListener = nextContext.getContext().get(TaskListener.class); - if (nextContextTaskListener != null) { - nextContextLogger = nextContextTaskListener.getLogger(); - } - } catch (IOException | InterruptedException e) { - LOGGER.log(Level.FINE, "Could not get logger for next context: " + e, e); - } - // remove context from queue and process it - List requiredResourceForNextContext = checkResourcesAvailability( - nextContext.getResources(), nextContextLogger, null, resourceNamesToUnreserve); - this.queuedContexts.remove(nextContext); - - // resourceNamesToUnreserve contains the names of the previous resources. - // requiredResourceForNextContext contains the resource objects which are required for the next - // context. - // It is guaranteed that there is an overlap between the two - the resources which are to be - // reused. - boolean needToWait = false; - for (LockableResource requiredResource : requiredResourceForNextContext) { - if (!resourceNamesToUnreserve.contains(requiredResource.getName())) { - if (requiredResource.isReserved() || requiredResource.isLocked()) { - needToWait = true; - break; - } - } - } - - if (needToWait) { - unreserveResources(resources); - return; - } else { - unreserveResources(resources); - LinkedHashMap> resourcesToLock = new LinkedHashMap<>(); - - // lock all (old and new resources) - for (LockableResource requiredResource : requiredResourceForNextContext) { - Run build = nextContext.getBuild(); - if (build == null) { - // skip this context, as the build cannot be retrieved (maybe it was deleted while - // running?) - LOGGER.log( - Level.WARNING, - "Skipping queued context for lock. Cannot get the Run object from the context to " - + "proceed with lock; this could be a legitimate state if the build waiting for " - + "the lock was deleted or hard killed. More information is logged at " - + "Level.FINE for debugging purposes."); - return; - } else { - requiredResource.setBuild(build); - resourcesToLock.put(requiredResource.getName(), requiredResource.getProperties()); - } - } + proceedNextContext(false /*inversePrecedence*/); - // continue with next context - LockStepExecution.proceed( - resourcesToLock, - nextContext.getContext(), - nextContext.getResourceDescription(), - nextContext.getVariableName(), - false); + save(); } - save(); } + // --------------------------------------------------------------------------- @NonNull @Override public String getDisplayName() { return Messages.LockableResourcesManager_displayName(); } - public synchronized void reset(List resources) { - for (LockableResource r : resources) { - uncacheIfFreeing(r, true, true); - r.reset(); + // --------------------------------------------------------------------------- + public void reset(List resources) { + synchronized (this.syncResources) { + for (LockableResource r : resources) { + uncacheIfFreeing(r, true, true); + r.reset(); + } + save(); } - save(); } + // --------------------------------------------------------------------------- /** * Make the lockable resource re-usable and notify the queue(s), if any WARNING: Do not use this * from inside the lock step closure which originally locked this resource, to avoid nasty @@ -1049,132 +1015,71 @@ public synchronized void reset(List resources) { * original closure ends and unlocks again that resource, a third consumer might then effectively * hijack it from the second one. */ - public synchronized void recycle(List resources) { - // Not calling reset() because that also un-queues the resource - // and we want to proclaim it is usable (if anyone is waiting) - this.unlock(resources, null); - this.unreserve(resources); + public void recycle(List resources) { + synchronized (this.syncResources) { + // Not calling reset() because that also un-queues the resource + // and we want to proclaim it is usable (if anyone is waiting) + this.unlock(resources, null); + this.unreserve(resources); + } } + // --------------------------------------------------------------------------- @Override public boolean configure(StaplerRequest req, JSONObject json) { - final List oldDeclaredResources = new ArrayList<>(getDeclaredResources()); - - try (BulkChange bc = new BulkChange(this)) { - // reset resources to default which are not currently locked - this.resources.removeIf(resource -> !resource.isLocked()); - req.bindJSON(this, json); - bc.commit(); - } catch (IOException exception) { - LOGGER.log(Level.WARNING, "Exception occurred while committing bulkchange operation.", exception); - return false; - } + synchronized (this.syncResources) { + final List oldDeclaredResources = new ArrayList<>(getDeclaredResources()); + + try (BulkChange bc = new BulkChange(this)) { + // reset resources to default which are not currently locked + this.resources.removeIf(resource -> !resource.isLocked()); + req.bindJSON(this, json); + bc.commit(); + } catch (IOException exception) { + LOGGER.log(Level.WARNING, "Exception occurred while committing bulkchange operation.", exception); + return false; + } - // Copy unconfigurable properties from old instances - boolean updated = false; - for (LockableResource oldDeclaredResource : oldDeclaredResources) { - final LockableResource updatedResource = fromName(oldDeclaredResource.getName()); - if (updatedResource != null) { - updatedResource.copyUnconfigurableProperties(oldDeclaredResource); - updated = true; + // Copy unconfigurable properties from old instances + boolean updated = false; + for (LockableResource oldDeclaredResource : oldDeclaredResources) { + final LockableResource updatedResource = fromName(oldDeclaredResource.getName()); + if (updatedResource != null) { + updatedResource.copyUnconfigurableProperties(oldDeclaredResource); + updated = true; + } + } + if (updated) { + save(); } } - if (updated) { - save(); - } - return true; } - /** - * @see #checkResourcesAvailability(List, PrintStream, List, List, boolean, - * ResourceSelectStrategy) - */ - public synchronized List checkResourcesAvailability( - List requiredResourcesList, - @Nullable PrintStream logger, - @Nullable List lockedResourcesAboutToBeUnlocked) { - boolean skipIfLocked = false; - ResourceSelectStrategy selectStrategy = ResourceSelectStrategy.SEQUENTIAL; - - return this.checkResourcesAvailability( - requiredResourcesList, logger, lockedResourcesAboutToBeUnlocked, null, skipIfLocked, selectStrategy); - } - - /** - * @see #checkResourcesAvailability(List, PrintStream, List, List, boolean, - * ResourceSelectStrategy) - */ - public synchronized List checkResourcesAvailability( - List requiredResourcesList, - @Nullable PrintStream logger, - @Nullable List lockedResourcesAboutToBeUnlocked, - boolean skipIfLocked) { - ResourceSelectStrategy selectStrategy = ResourceSelectStrategy.SEQUENTIAL; - - return this.checkResourcesAvailability( - requiredResourcesList, logger, lockedResourcesAboutToBeUnlocked, null, skipIfLocked, selectStrategy); - } - - /** - * @see #checkResourcesAvailability(List, PrintStream, List, List, boolean, - * ResourceSelectStrategy) - */ - public synchronized List checkResourcesAvailability( - List requiredResourcesList, - @Nullable PrintStream logger, - @Nullable List lockedResourcesAboutToBeUnlocked, - boolean skipIfLocked, - ResourceSelectStrategy selectStrategy) { - return this.checkResourcesAvailability( - requiredResourcesList, logger, lockedResourcesAboutToBeUnlocked, null, skipIfLocked, selectStrategy); - } - - /** - * @see #checkResourcesAvailability(List, PrintStream, List, List, boolean, - * ResourceSelectStrategy) - */ - public synchronized List checkResourcesAvailability( - List requiredResourcesList, - @Nullable PrintStream logger, - @Nullable List lockedResourcesAboutToBeUnlocked, - @Nullable List reservedResourcesAboutToBeUnreserved) { - boolean skipIfLocked = false; - ResourceSelectStrategy selectStrategy = ResourceSelectStrategy.SEQUENTIAL; - - return this.checkResourcesAvailability( - requiredResourcesList, - logger, - lockedResourcesAboutToBeUnlocked, - reservedResourcesAboutToBeUnreserved, - skipIfLocked, - selectStrategy); + // --------------------------------------------------------------------------- + public List getAvailableResources(final List requiredResourcesList) { + return this.getAvailableResources(requiredResourcesList, null, null); } + // --------------------------------------------------------------------------- /** * Checks if there are enough resources available to satisfy the requirements specified within * requiredResources and returns the necessary available resources. If not enough resources are * available, returns null. */ - public synchronized List checkResourcesAvailability( - List requiredResourcesList, - @Nullable PrintStream logger, - @Nullable List lockedResourcesAboutToBeUnlocked, - @Nullable List reservedResourcesAboutToBeUnreserved, - boolean skipIfLocked, - ResourceSelectStrategy selectStrategy) { - - List requiredResourcesCandidatesList = new ArrayList<>(); + public List getAvailableResources( + final List requiredResourcesList, + final @Nullable PrintStream logger, + final @Nullable ResourceSelectStrategy selectStrategy) { - // Build possible resources for each requirement + LOGGER.finest("getAvailableResources, " + requiredResourcesList); + List candidates = new ArrayList<>(); for (LockableResourcesStruct requiredResources : requiredResourcesList) { - // get possible resources - int requiredAmount = 0; // 0 means all - List candidates = new ArrayList<>(); - if (StringUtils.isBlank(requiredResources.label)) { - candidates.addAll(requiredResources.required); - } else { - candidates.addAll(getResourcesWithLabel(requiredResources.label, null)); + List available = new ArrayList<>(); + // filter by labels + if (!StringUtils.isBlank(requiredResources.label)) { + // get required amount first + int requiredAmount = 0; if (requiredResources.requiredNumber != null) { try { requiredAmount = Integer.parseInt(requiredResources.requiredNumber); @@ -1182,157 +1087,127 @@ public synchronized List checkResourcesAvailability( requiredAmount = 0; } } - } - if (requiredAmount == 0) { - requiredAmount = candidates.size(); - } + available = this.getFreeResourcesWithLabel( + requiredResources.label, requiredAmount, selectStrategy, logger, candidates); + } else if (requiredResources.required != null) { + // resource by name requested - requiredResourcesCandidatesList.add(new LockableResourcesCandidatesStruct(candidates, requiredAmount)); - } + // this is a little hack. The 'requiredResources.required' is a copy, and we need to find + // all of them in LRM + // fromNames() also re-create the resource (ephemeral things) + available = fromNames( + getResourcesNames(requiredResources.required), /*create un-existent resources */ true); - // Process freed resources - int totalSelected = 0; - // These resources are currently reserved, even though candidates - // for freeing (might be reserved inside lock step "bypassing" the - // lockable resources general logic). They may become available - // later and we want to notice that - so they are not selected - // now, but we do not bail out and end the looping either. - int totalReserved = 0; + if (!this.areAllAvailable(available)) { + available = null; + } + } else { + LOGGER.warning("getAvailableResources, Not implemented: " + requiredResources); + } - for (LockableResourcesCandidatesStruct requiredResources : requiredResourcesCandidatesList) { - if (selectStrategy.equals(ResourceSelectStrategy.RANDOM)) { - Collections.shuffle(requiredResources.candidates); + if (available == null || available.isEmpty()) { + LOGGER.finest("No available resources found " + requiredResourcesList); + return null; } - // start with an empty set of selected resources - List selected = new ArrayList<>(); - - // some resources might be already locked, but will be freed. - // Determine if these resources can be reused - // FIXME? Why is this check not outside the for loop? - if (lockedResourcesAboutToBeUnlocked != null || reservedResourcesAboutToBeUnreserved != null) { - for (LockableResource candidate : requiredResources.candidates) { - if (selected.size() >= requiredResources.requiredAmount) { - break; - } - String candidateName = candidate.getName(); - boolean listedUnlock = (lockedResourcesAboutToBeUnlocked != null - && lockedResourcesAboutToBeUnlocked.contains(candidateName)); - boolean listedUnreserve = (reservedResourcesAboutToBeUnreserved != null - && reservedResourcesAboutToBeUnreserved.contains(candidateName)); - boolean isReserved = candidate.isReserved(); - boolean isLocked = candidate.isLocked(); - - if (isReserved) { - if (listedUnreserve) { - if (!isLocked || listedUnlock) { - // Avoid selecting a reserved candidate which *is* also locked - // and not listed for imminent un-locking - selected.add(candidate); - } - } else { - // Caller did not say that this resource will be un-reserved now! - // Still needed, might be `lr.setReservedBy()` from the lock step - // closure by users who deemed that required in their workflow - // and might need to free it manually - maybe after postmortem. - // Note that such un-reservation should go through LRM API, - // as `lrm.unreserve([lr])`, and not just `lr.setReservedBy(null)`, - // (nor `lrm.reset([lr])`) to get into this method among others - // and let the resource be instantly re-used by someone from an - // already waiting queue. Otherwise those already waiting are not - // notified until you lock/unlock that resource again. - if (logger != null) { - logger.println("Candidate resource '" - + candidateName - + "' is reserved by '" - + candidate.getReservedBy() - + "', not treating as available."); - } - totalReserved += 1; - continue; - } - } else { - // If the resource is not reserved (as checked above) - // but listed for releasing in either category, select it - if (listedUnlock || listedUnreserve) { - selected.add(candidate); - } - } - } + final boolean isPreReserved = !Collections.disjoint(candidates, available); + if (isPreReserved) { + // FIXME I think this is failure + // You use filter label1 and it lock resource1 and then in extra you will lock resource1 + // But when I allow this line, many tests will fails, and I am pretty sure it will throws + // exceptions on end-user pipelines + // So when we want to fix, it it might be braking-change + // Therefore keep it here as warning for now + printLogs("Extra filter tries to allocate pre-reserved resources.", logger, Level.WARNING); + available.removeAll(candidates); } - totalSelected += selected.size(); - requiredResources.selected = selected; + candidates.addAll(available); } - // if none of the currently locked resources can be reused, - // this context is not suitable to be continued with - // Note that if arguments lockedResourcesAboutToBeUnlocked==null - // and reservedResourcesAboutToBeUnreserved==null, then - // the loop above was effectively skipped - if (totalSelected == 0 - && totalReserved == 0 - && (lockedResourcesAboutToBeUnlocked != null || reservedResourcesAboutToBeUnreserved != null)) { - return null; + return candidates; + } + + // --------------------------------------------------------------------------- + private boolean areAllAvailable(List resources) { + for (LockableResource resource : resources) { + if (!resource.isFree()) { + return false; + } } + return true; + } - // Find remaining resources - List allSelected = new ArrayList<>(); + // --------------------------------------------------------------------------- + public static void printLogs(final String msg, final Level level, Logger L, final @Nullable PrintStream logger) { + L.log(level, msg); - for (LockableResourcesCandidatesStruct requiredResources : requiredResourcesCandidatesList) { - List candidates = requiredResources.candidates; - List selected = requiredResources.selected; - int requiredAmount = requiredResources.requiredAmount; + if (logger != null) { + if (level == Level.WARNING || level == Level.SEVERE) logger.println(level.getLocalizedName() + ": " + msg); + else logger.println(msg); + } + } - // Try and re-use as many previously selected resources first - List alreadySelectedCandidates = new ArrayList<>(candidates); - alreadySelectedCandidates.retainAll(allSelected); - for (LockableResource rs : alreadySelectedCandidates) { - if (selected.size() >= requiredAmount) { - break; - } - if (!rs.isReserved() && !rs.isLocked()) { - selected.add(rs); - } - } + // --------------------------------------------------------------------------- + private static void printLogs(final String msg, final @Nullable PrintStream logger, final Level level) { + printLogs(msg, level, LOGGER, logger); + } - candidates.removeAll(alreadySelectedCandidates); - for (LockableResource rs : candidates) { - if (selected.size() >= requiredAmount) { - break; - } - // TODO: it shall be used isFree() here, but in that case we need to change the - // logic in parametrized builds and that is much more effort as I want to spend here now - if (!rs.isReserved() && !rs.isLocked()) { - selected.add(rs); - } - } + // --------------------------------------------------------------------------- + @CheckForNull + @Restricted(NoExternalUse.class) + private List getFreeResourcesWithLabel( + @NonNull String label, + long amount, + final @Nullable ResourceSelectStrategy selectStrategy, + final @Nullable PrintStream logger, + final List alreadySelected) { + List found = new ArrayList<>(); - if (selected.size() < requiredAmount) { - // Note: here we are looping over requiredResourcesCandidatesList - // based on original argument requiredResourcesList with its specs - // (maybe several) of required resources and their amounts. - // As soon as we know we can not fulfill the overall requirement - // (not enough of something from that list), we bail out quickly. - if (logger != null && !skipIfLocked) { + List candidates = _getResourcesWithLabel(label, alreadySelected); + candidates.addAll(this.getResourcesWithLabel(label)); - String msg = "Found " + selected.size() + " available resource(s). Waiting for correct amount: " - + requiredAmount + "."; + if (amount <= 0) { + amount = candidates.size(); + } - if (enabledBlockedCount != 0) { - msg += "\nBlocking causes: " + this.getCauses(candidates); - } + if (candidates.size() < amount) { + printLogs( + "Found " + + found.size() + + " possible resource(s). Waiting for correct amount: " + + amount + + "." + + "This may stuck, until you crate enough resources", + logger, + Level.WARNING); + return null; // there are not enough resources + } - logger.println(msg); - } - return null; + if (selectStrategy != null && selectStrategy.equals(ResourceSelectStrategy.RANDOM)) { + Collections.shuffle(candidates); + } + + for (LockableResource r : candidates) { + // TODO: it shall be used isFree() here, but in that case we need to change the + // logic in parametrized builds and that is much more effort as I want to spend here now + if (!r.isReserved() && !r.isLocked()) { + found.add(r); } - allSelected.addAll(selected); + if (amount > 0 && found.size() >= amount) { + return found; + } + } + + String msg = "Found " + found.size() + " available resource(s). Waiting for correct amount: " + amount + "."; + if (enabledBlockedCount != 0) { + msg += "\nBlocking causes: " + getCauses(candidates); } + printLogs(msg, logger, Level.FINE); - return allSelected; + return null; } // --------------------------------------------------------------------------- @@ -1412,32 +1287,40 @@ private String getQueueCause(final LockableResource resource) { * Adds the given context and the required resources to the queue if * this context is not yet queued. */ - public synchronized void queueContext( + public void queueContext( StepContext context, List requiredResources, String resourceDescription, String variableName) { - for (QueuedContextStruct entry : this.queuedContexts) { - if (entry.getContext() == context) { - return; + synchronized (this.syncResources) { + for (QueuedContextStruct entry : this.queuedContexts) { + if (entry.getContext() == context) { + LOGGER.warning("queueContext, duplicated, " + requiredResources); + return; + } } - } - this.queuedContexts.add(new QueuedContextStruct(context, requiredResources, resourceDescription, variableName)); - save(); + this.queuedContexts.add( + new QueuedContextStruct(context, requiredResources, resourceDescription, variableName)); + save(); + } } - public synchronized boolean unqueueContext(StepContext context) { - for (Iterator iter = this.queuedContexts.listIterator(); iter.hasNext(); ) { - if (iter.next().getContext() == context) { - iter.remove(); - save(); - return true; + // --------------------------------------------------------------------------- + public boolean unqueueContext(StepContext context) { + synchronized (this.syncResources) { + for (Iterator iter = this.queuedContexts.listIterator(); iter.hasNext(); ) { + if (iter.next().getContext() == context) { + iter.remove(); + save(); + return true; + } } } return false; } + // --------------------------------------------------------------------------- public static LockableResourcesManager get() { return (LockableResourcesManager) Jenkins.get().getDescriptorOrDie(LockableResourcesManager.class); } @@ -1463,5 +1346,10 @@ public void save() { } } - private static final Logger LOGGER = Logger.getLogger(LockableResourcesManager.class.getName()); + // --------------------------------------------------------------------------- + /** For testing purpose. */ + @Restricted(NoExternalUse.class) + public LockableResource getFirst() { + return this.getResources().get(0); + } } diff --git a/src/main/java/org/jenkins/plugins/lockableresources/queue/LockableResourcesCandidatesStruct.java b/src/main/java/org/jenkins/plugins/lockableresources/queue/LockableResourcesCandidatesStruct.java deleted file mode 100644 index 7be8643de..000000000 --- a/src/main/java/org/jenkins/plugins/lockableresources/queue/LockableResourcesCandidatesStruct.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.jenkins.plugins.lockableresources.queue; - -import java.util.List; -import org.jenkins.plugins.lockableresources.LockableResource; - -public class LockableResourcesCandidatesStruct { - - public List candidates; - public int requiredAmount; - public List selected; - - public LockableResourcesCandidatesStruct(List candidates, int requiredAmount) { - this.candidates = candidates; - this.requiredAmount = requiredAmount; - } - - @Override - public String toString() { - return "LockableResourcesCandidatesStruct [candidates=" - + candidates - + ", requiredAmount=" - + requiredAmount - + ", selected=" - + selected - + "]"; - } -} diff --git a/src/main/java/org/jenkins/plugins/lockableresources/queue/LockableResourcesStruct.java b/src/main/java/org/jenkins/plugins/lockableresources/queue/LockableResourcesStruct.java index aea03f2ed..8b824930f 100644 --- a/src/main/java/org/jenkins/plugins/lockableresources/queue/LockableResourcesStruct.java +++ b/src/main/java/org/jenkins/plugins/lockableresources/queue/LockableResourcesStruct.java @@ -48,17 +48,18 @@ public LockableResourcesStruct(RequiredResourcesProperty property, EnvVars env) queuedAt = new Date().getTime(); required = new ArrayList<>(); - LockableResourcesManager resourcesManager = LockableResourcesManager.get(); + List names = new ArrayList<>(); for (String name : property.getResources()) { String resourceName = env.expand(name); if (resourceName == null) { continue; } - resourcesManager.createResource(resourceName); - LockableResource r = resourcesManager.fromName(resourceName); - this.required.add(r); + names.add(resourceName); } + LockableResourcesManager lrm = LockableResourcesManager.get(); + this.required = lrm.fromNames(names, /*create un-existent resources */ true); + label = env.expand(property.getLabelName()); if (label == null) label = ""; @@ -135,28 +136,23 @@ public SecureGroovyScript getResourceMatchScript() { @Override public String toString() { - return "Required resources: " - + this.required - + ", Required label: " - + this.label - + ", Required label script: " - + (this.resourceMatchScript != null ? this.resourceMatchScript.getScript() : "") - + ", Variable name: " - + this.requiredVar - + ", Number of resources: " - + this.requiredNumber; - } - - /** Returns timestamp when the resource has been added into queue. */ - @Restricted(NoExternalUse.class) // used by jelly - public Date getQueuedTimestamp() { - return new Date(this.queuedAt); - } - - /** Check if the queue takes too long. At the moment "too long" means over 1 hour. */ - @Restricted(NoExternalUse.class) // used by jelly - public boolean takeTooLong() { - return (new Date().getTime() - this.queuedAt) > 3600000L; + String str = ""; + if (this.required != null && !this.required.isEmpty()) { + str += "Required resources: " + this.required; + } + if (this.label != null && !this.label.isEmpty()) { + str += "Required label: " + this.label; + } + if (this.resourceMatchScript != null) { + str += "Required label script: " + this.resourceMatchScript.getScript(); + } + if (this.requiredVar != null) { + str += ", Variable name: " + this.requiredVar; + } + if (this.requiredNumber != null) { + str += ", Number of resources: " + this.requiredNumber; + } + return str; } /** Check if the *resource* is required by this struct / queue */ diff --git a/src/main/java/org/jenkins/plugins/lockableresources/queue/QueuedContextStruct.java b/src/main/java/org/jenkins/plugins/lockableresources/queue/QueuedContextStruct.java index 4e9f116fc..79548bd8e 100644 --- a/src/main/java/org/jenkins/plugins/lockableresources/queue/QueuedContextStruct.java +++ b/src/main/java/org/jenkins/plugins/lockableresources/queue/QueuedContextStruct.java @@ -10,8 +10,11 @@ import edu.umd.cs.findbugs.annotations.CheckForNull; import hudson.model.Run; +import hudson.model.TaskListener; import java.io.IOException; +import java.io.PrintStream; import java.io.Serializable; +import java.util.Date; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -46,11 +49,17 @@ public class QueuedContextStruct implements Serializable { */ private String variableName; + private long queuedAt = 0; + + // cached candidates + public transient List candidates = null; + private static final Logger LOGGER = Logger.getLogger(QueuedContextStruct.class.getName()); /* * Constructor for the QueuedContextStruct class. */ + @Restricted(NoExternalUse.class) public QueuedContextStruct( StepContext context, List lockableResourcesStruct, @@ -60,20 +69,25 @@ public QueuedContextStruct( this.lockableResourcesStruct = lockableResourcesStruct; this.resourceDescription = resourceDescription; this.variableName = variableName; + this.queuedAt = new Date().getTime(); } /* * Gets the pipeline step context. */ + @Restricted(NoExternalUse.class) public StepContext getContext() { return this.context; } /** Return build, where is the resource used. */ @CheckForNull - @Restricted(NoExternalUse.class) // used by jelly + @Restricted(NoExternalUse.class) public Run getBuild() { try { + if (this.getContext() == null) { + return null; + } return this.getContext().get(Run.class); } catch (IOException | InterruptedException e) { // for some reason there is no Run object for this context @@ -82,6 +96,18 @@ public StepContext getContext() { } } + @Restricted(NoExternalUse.class) + public boolean isValid() { + Run run = this.getBuild(); + if (run == null || run.isBuilding() == false) { + // skip this one, for some reason there is no Run object for this context + LOGGER.warning("The queue " + this + " will be removed, because the build does not exists"); + return false; + } + return true; + } + + @Restricted(NoExternalUse.class) /* * Gets the required resources. */ @@ -89,6 +115,7 @@ public List getResources() { return this.lockableResourcesStruct; } + @Restricted(NoExternalUse.class) /* * Gets the resource description for logging messages. */ @@ -96,6 +123,7 @@ public String getResourceDescription() { return this.resourceDescription; } + @Restricted(NoExternalUse.class) /* * Gets the variable name to save the locks taken. */ @@ -103,5 +131,35 @@ public String getVariableName() { return this.variableName; } + /** Get time-ticks, when the item has been added into queue */ + @Restricted(NoExternalUse.class) + public long getAddTime() { + return queuedAt; + } + + @Restricted(NoExternalUse.class) + public String toString() { + return "build: " + + this.getBuild() + + " resources: " + + this.getResourceDescription() + + " added at: " + + this.getAddTime(); + } + + @Restricted(NoExternalUse.class) + public PrintStream getLogger() { + PrintStream logger = null; + try { + TaskListener taskListener = this.getContext().get(TaskListener.class); + if (taskListener != null) { + logger = taskListener.getLogger(); + } + } catch (IOException | InterruptedException e) { + LOGGER.log(Level.FINE, "Could not get logger for next context: " + e, e); + } + return logger; + } + private static final long serialVersionUID = 1L; } diff --git a/src/test/java/org/jenkins/plugins/lockableresources/ConfigurationAsCodeTest.java b/src/test/java/org/jenkins/plugins/lockableresources/ConfigurationAsCodeTest.java index 74c5d3dd2..aec2a1e04 100644 --- a/src/test/java/org/jenkins/plugins/lockableresources/ConfigurationAsCodeTest.java +++ b/src/test/java/org/jenkins/plugins/lockableresources/ConfigurationAsCodeTest.java @@ -31,8 +31,8 @@ public void setUp() { @Test public void should_support_configuration_as_code() { - List declaredResources = - LockableResourcesManager.get().getDeclaredResources(); + LockableResourcesManager LRM = LockableResourcesManager.get(); + List declaredResources = LRM.getDeclaredResources(); assertEquals( "The number of declared resources is wrong. Check your configuration-as-code.yml", 1, @@ -45,10 +45,12 @@ public void should_support_configuration_as_code() { assertEquals("Reserved_A", declaredResource.getReservedBy()); assertEquals("Note A", declaredResource.getNote()); - List resources = LockableResourcesManager.get().getResources(); - assertEquals("The number of resources is wrong. Check your configuration-as-code.yml", 1, resources.size()); + assertEquals( + "The number of resources is wrong. Check your configuration-as-code.yml", + 1, + LRM.getResources().size()); - LockableResource resource = resources.get(0); + LockableResource resource = LRM.getFirst(); assertEquals("Resource_A", resource.getName()); assertEquals("Description_A", resource.getDescription()); assertEquals("Label_A", resource.getLabels()); diff --git a/src/test/java/org/jenkins/plugins/lockableresources/InteroperabilityTest.java b/src/test/java/org/jenkins/plugins/lockableresources/InteroperabilityTest.java index ed2065f5d..49eb3842c 100644 --- a/src/test/java/org/jenkins/plugins/lockableresources/InteroperabilityTest.java +++ b/src/test/java/org/jenkins/plugins/lockableresources/InteroperabilityTest.java @@ -6,6 +6,7 @@ import hudson.model.FreeStyleBuild; import hudson.model.FreeStyleProject; import java.util.concurrent.Semaphore; +import java.util.logging.Logger; import org.jenkins.plugins.lockableresources.util.Constants; import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; import org.jenkinsci.plugins.workflow.job.WorkflowJob; @@ -18,6 +19,7 @@ public class InteroperabilityTest extends LockStepTestBase { + private static final Logger LOGGER = Logger.getLogger(InteroperabilityTest.class.getName()); // --------------------------------------------------------------------------- @Before public void setUp() { @@ -51,11 +53,13 @@ public boolean perform(AbstractBuild build, Launcher launcher, BuildListen FreeStyleBuild f1 = f.scheduleBuild2(0).waitForStart(); WorkflowRun b1 = p.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[resource1] is locked by " + f1.getFullDisplayName() + ", waiting...", b1); + LOGGER.info("wait for: [resource1] is locked by build " + f1.getFullDisplayName()); + j.waitForMessage("[resource1] is locked by build " + f1.getFullDisplayName(), b1); isPaused(b1, 1, 1); semaphore.release(); // Wait for lock after the freestyle finishes + LOGGER.info("wait for2: Lock released on resource [resource1]"); j.waitForMessage("Lock released on resource [resource1]", b1); isPaused(b1, 1, 0); j.assertBuildStatusSuccess(j.waitForCompletion(f1)); diff --git a/src/test/java/org/jenkins/plugins/lockableresources/LockStepHardKillTest.java b/src/test/java/org/jenkins/plugins/lockableresources/LockStepHardKillTest.java index 6d08a1e72..faa34591f 100644 --- a/src/test/java/org/jenkins/plugins/lockableresources/LockStepHardKillTest.java +++ b/src/test/java/org/jenkins/plugins/lockableresources/LockStepHardKillTest.java @@ -58,7 +58,7 @@ public void hardKillNewBuildClearsLock() throws Exception { WorkflowRun b2 = p2.scheduleBuild2(0).waitForStart(); // Make sure that b2 is blocked on b1's lock. - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b2); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b2); isPaused(b2, 1, 1); // Now b2 is still sitting waiting for a lock. Create b3 and launch it to clear the @@ -69,7 +69,7 @@ public void hardKillNewBuildClearsLock() throws Exception { WorkflowRun b3 = p3.scheduleBuild2(0).waitForStart(); // Make sure that b3 is also blocked still on b1's lock. - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b3); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b3); isPaused(b3, 1, 1); // Kill b1 hard. @@ -113,7 +113,7 @@ public void hardKillWithWaitingRuns() throws Exception { for (int i = 0; i < 3; i++) { WorkflowRun rNext = p.scheduleBuild2(0).waitForStart(); if (prevBuild != null) { - j.waitForMessage("[resource1] is locked by " + prevBuild.getFullDisplayName() + ", waiting...", rNext); + j.waitForMessage("[resource1] is locked by build " + prevBuild.getFullDisplayName(), rNext); isPaused(rNext, 1, 1); interruptTermKill(prevBuild); } @@ -152,9 +152,9 @@ public void hardKillWithWaitingRunsOnLabel() throws Exception { j.waitForMessage("Trying to acquire lock on", secondNext); if (firstPrev != null) { - j.waitForMessage("is locked, waiting...", firstNext); + j.waitForMessage(", waiting for execution ...", firstNext); isPaused(firstNext, 1, 1); - j.waitForMessage("is locked, waiting...", secondNext); + j.waitForMessage(", waiting for execution ...", secondNext); isPaused(secondNext, 1, 1); } diff --git a/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest.java b/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest.java index c7b47402f..c7af1d84d 100644 --- a/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest.java +++ b/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest.java @@ -107,12 +107,12 @@ public void lockOrderLabel() throws Exception { WorkflowRun b2 = p.scheduleBuild2(0).waitForStart(); // Ensure that b2 reaches the lock before b3 - j.waitForMessage("[Label: label1, Quantity: 2] is locked, waiting...", b2); + j.waitForMessage("[Label: label1, Quantity: 2] is not free, waiting for execution ...", b2); j.waitForMessage("Found 1 available resource(s). Waiting for correct amount: 2.", b2); isPaused(b2, 1, 1); WorkflowRun b3 = p.scheduleBuild2(0).waitForStart(); // Both 2 and 3 are waiting for locking Label: label1, Quantity: 2 - j.waitForMessage("[Label: label1, Quantity: 2] is locked, waiting...", b3); + j.waitForMessage("[Label: label1, Quantity: 2] is not free, waiting for execution ...", b3); j.waitForMessage("Found 1 available resource(s). Waiting for correct amount: 2.", b3); isPaused(b3, 1, 1); @@ -153,7 +153,7 @@ public void lockOrderLabelQuantity() throws Exception { WorkflowRun b2 = p.scheduleBuild2(0).waitForStart(); // Ensure that b2 reaches the lock before b3 - j.waitForMessage("[Label: label1, Quantity: 2] is locked, waiting...", b2); + j.waitForMessage("[Label: label1, Quantity: 2] is not free, waiting for execution ...", b2); j.waitForMessage("Found 1 available resource(s). Waiting for correct amount: 2.", b2); isPaused(b2, 1, 1); @@ -214,7 +214,7 @@ public void lockOrderLabelQuantityFreedResources() throws Exception { true)); WorkflowRun b2 = p2.scheduleBuild2(0).waitForStart(); // Ensure that b2 reaches the lock before b3 - j.waitForMessage("[Label: label1, Quantity: 2] is locked, waiting...", b2); + j.waitForMessage("[Label: label1, Quantity: 2] is not free, waiting for execution ...", b2); j.waitForMessage("Found 0 available resource(s). Waiting for correct amount: 2.", b2); isPaused(b2, 1, 1); @@ -226,7 +226,7 @@ public void lockOrderLabelQuantityFreedResources() throws Exception { + "echo 'Finish'", true)); WorkflowRun b3 = p3.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[Label: label1, Quantity: 1] is locked, waiting...", b3); + j.waitForMessage("[Label: label1, Quantity: 1] is not free, waiting for execution ...", b3); j.waitForMessage("Found 0 available resource(s). Waiting for correct amount: 1.", b3); isPaused(b3, 1, 1); @@ -265,12 +265,12 @@ public void lockOrder() throws Exception { WorkflowRun b2 = p.scheduleBuild2(0).waitForStart(); // Ensure that b2 reaches the lock before b3 - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b2); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b2); isPaused(b2, 1, 1); WorkflowRun b3 = p.scheduleBuild2(0).waitForStart(); // Both 2 and 3 are waiting for locking resource1 - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b3); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b3); isPaused(b3, 1, 1); // Unlock resource1 @@ -306,12 +306,12 @@ public void lockInverseOrder() throws Exception { WorkflowRun b2 = p.scheduleBuild2(0).waitForStart(); // Ensure that b2 reaches the lock before b3 - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b2); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b2); isPaused(b2, 1, 1); WorkflowRun b3 = p.scheduleBuild2(0).waitForStart(); // Both 2 and 3 are waiting for locking resource1 - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b3); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b3); isPaused(b3, 1, 1); // Unlock resource1 @@ -355,7 +355,7 @@ public void parallelLock() throws Exception { // lock j.waitForMessage("Lock acquired on [resource1]", b1); SemaphoreStep.success("before-a/1", null); - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b1); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b1); isPaused(b1, 2, 1); SemaphoreStep.success("wait-b/1", null); @@ -396,7 +396,7 @@ public void deleteRunningBuildNewBuildClearsLock() throws Exception { p2.setDefinition(new CpsFlowDefinition("lock('resource1') {\n" + " semaphore 'wait-inside'\n" + "}", true)); WorkflowRun b2 = p2.scheduleBuild2(0).waitForStart(); // Make sure that b2 is blocked on b1's lock. - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b2); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b2); isPaused(b2, 1, 1); // Now b2 is still sitting waiting for a lock. Create b3 and launch it to verify order of @@ -404,7 +404,7 @@ public void deleteRunningBuildNewBuildClearsLock() throws Exception { WorkflowJob p3 = j.jenkins.createProject(WorkflowJob.class, "p3"); p3.setDefinition(new CpsFlowDefinition("lock('resource1') {\n" + " semaphore 'wait-inside'\n" + "}", true)); WorkflowRun b3 = p3.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b3); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b3); isPaused(b3, 1, 1); b1.delete(); @@ -429,7 +429,10 @@ public void unlockButtonWithWaitingRuns() throws Exception { LockableResourcesManager.get().createResource("resource1"); WorkflowJob p = j.jenkins.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( - "retry(99) {\n" + " lock('resource1') {\n" + " semaphore('wait-inside')\n" + " }\n" + "}", + // "retry(99) {\n" + " lock('resource1') {\n" + " semaphore('wait-inside')\n" + " }\n" + // + "}" + , true)); JenkinsRule.WebClient wc = j.createWebClient(); @@ -438,8 +441,11 @@ public void unlockButtonWithWaitingRuns() throws Exception { for (int i = 0; i < 3; i++) { WorkflowRun rNext = p.scheduleBuild2(0).waitForStart(); if (prevBuild != null) { - j.waitForMessage("[resource1] is locked by " + prevBuild.getFullDisplayName() + ", waiting...", rNext); + j.waitForMessage("[resource1] is locked by build " + prevBuild.getFullDisplayName(), rNext); isPaused(rNext, 1, 1); + // List resources = new ArrayList<>(); + // resources.add(LockableResourcesManager.get().fromName("resource1")); + // LockableResourcesManager.get().unlock(resources, null); TestHelpers.clickButton(wc, "unlock"); } @@ -479,7 +485,7 @@ public void parallelLockRelease() throws Exception { for (int i = 0; i < 5; i++) { WorkflowRun rNext = job.scheduleBuild2(0).waitForStart(); if (toUnlock != null) { - j.waitForMessage("[resource1] is locked by " + toUnlock.getFullDisplayName() + ", waiting...", rNext); + j.waitForMessage("[resource1] is locked by build " + toUnlock.getFullDisplayName(), rNext); isPaused(rNext, 1, 1); SemaphoreStep.success("wait-inside-1/" + i, null); } @@ -565,14 +571,14 @@ public void lockMultipleResources() throws Exception { p2.setDefinition(new CpsFlowDefinition( "lock('resource1') {\n" + " semaphore 'wait-inside-p2'\n" + "}\n" + "echo 'Finish'", true)); WorkflowRun b2 = p2.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b2); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b2); isPaused(b2, 1, 1); WorkflowJob p3 = j.jenkins.createProject(WorkflowJob.class, "p3"); p3.setDefinition(new CpsFlowDefinition( "lock('resource2') {\n" + " semaphore 'wait-inside-p3'\n" + "}\n" + "echo 'Finish'", true)); WorkflowRun b3 = p3.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[resource2] is locked by " + b1.getFullDisplayName() + ", waiting...", b3); + j.waitForMessage("[resource2] is locked by build " + b1.getFullDisplayName(), b3); isPaused(b3, 1, 1); // Unlock resources @@ -614,14 +620,14 @@ public void lockWithLabelAndResource() throws Exception { p2.setDefinition(new CpsFlowDefinition( "lock('resource1') {\n" + " semaphore 'wait-inside-p2'\n" + "}\n" + "echo 'Finish'", true)); WorkflowRun b2 = p2.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b2); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b2); isPaused(b2, 1, 1); WorkflowJob p3 = j.jenkins.createProject(WorkflowJob.class, "p3"); p3.setDefinition(new CpsFlowDefinition( "lock(label: 'label1') {\n" + " semaphore 'wait-inside-p3'\n" + "}\n" + "echo 'Finish'", true)); WorkflowRun b3 = p3.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[Label: label1] is locked, waiting...", b3); + j.waitForMessage("[Label: label1] is not free, waiting for execution ...", b3); isPaused(b3, 1, 1); // Unlock resources @@ -657,20 +663,23 @@ public void lockWithLabelAndLabeledResource() throws Exception { + "echo 'Finish'", true)); WorkflowRun b1 = p.scheduleBuild2(0).waitForStart(); + + j.waitForMessage("Extra filter tries to allocate pre-reserved resources.", b1); + SemaphoreStep.waitForStart("wait-inside/1", b1); WorkflowJob p2 = j.jenkins.createProject(WorkflowJob.class, "p2"); p2.setDefinition(new CpsFlowDefinition( "lock('resource1') {\n" + " semaphore 'wait-inside-p2'\n" + "}\n" + "echo 'Finish'", true)); WorkflowRun b2 = p2.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b2); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b2); isPaused(b2, 1, 1); WorkflowJob p3 = j.jenkins.createProject(WorkflowJob.class, "p3"); p3.setDefinition(new CpsFlowDefinition( "lock(label: 'label1') {\n" + " semaphore 'wait-inside-p3'\n" + "}\n" + "echo 'Finish'", true)); WorkflowRun b3 = p3.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[Label: label1] is locked, waiting...", b3); + j.waitForMessage("[Label: label1] is not free, waiting for execution ...", b3); isPaused(b3, 1, 1); // Unlock resources @@ -716,6 +725,9 @@ public void lockWithLabelAndLabeledResourceQuantity() throws Exception { WorkflowRun b1 = p.scheduleBuild2(0).waitForStart(); SemaphoreStep.waitForStart("wait-inside/1", b1); + j.waitForMessage("Extra filter tries to allocate pre-reserved resources.", b1); + j.waitForMessage("Resources locked: [resource2, resource4]", b1); + WorkflowJob p2 = j.jenkins.createProject(WorkflowJob.class, "p2"); p2.setDefinition(new CpsFlowDefinition( "lock(label: 'label1', variable: 'var', quantity: 3) {\n" @@ -726,7 +738,7 @@ public void lockWithLabelAndLabeledResourceQuantity() throws Exception { + "echo 'Finish'", true)); WorkflowRun b2 = p2.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[Label: label1, Quantity: 3] is locked, waiting...", b2); + j.waitForMessage("[Label: label1, Quantity: 3] is not free, waiting for execution ...", b2); j.waitForMessage("Found 2 available resource(s). Waiting for correct amount: 3.", b2); isPaused(b2, 1, 1); @@ -855,7 +867,7 @@ public void lockWithLabelFillsVariable() throws Exception { p2.setDefinition(new CpsFlowDefinition( "lock(label: 'label1', variable: 'someVar2') {\n" + " echo \"VAR2 IS $env.someVar2\"\n" + "}", true)); WorkflowRun b2 = p2.scheduleBuild2(0).waitForStart(); - j.waitForMessage("is locked, waiting...", b2); + j.waitForMessage(", waiting for execution ...", b2); isPaused(b2, 1, 1); // Unlock resources @@ -897,7 +909,7 @@ public void parallelLockWithLabelFillsVariable() throws Exception { SemaphoreStep.waitForStart("wait-inside/1", b1); SemaphoreStep.success("wait-outside/1", null); - j.waitForMessage("is locked, waiting...", b1); + j.waitForMessage(", waiting for execution ...", b1); isPaused(b1, 2, 1); // Unlock resources @@ -934,7 +946,7 @@ public void unreserveSetsVariable() throws Exception { "lock(label: 'label1', variable: 'someVar') {\n" + " echo \"VAR IS $env.someVar\"\n" + "}", true)); WorkflowRun b1 = p.scheduleBuild2(0).waitForStart(); - j.waitForMessage("is locked, waiting...", b1); + j.waitForMessage(", waiting for execution ...", b1); lm.unreserve(Collections.singletonList(lm.fromName("resource1"))); j.assertBuildStatusSuccess(j.waitForCompletion(b1)); @@ -1028,6 +1040,7 @@ public void reserveInsideLockHonoured() throws Exception { + "p2: {\n" // + " semaphore 'wait-outside'\n" + " org.jenkins.plugins.lockableresources.LockableResource lr = null\n" + + " sleep 1\n" + " echo \"Locked resource cause 2-1: not locked yet\"\n" + " lock(label: 'label1', variable: 'someVar2') {\n" + " echo \"VAR2 IS $env.someVar2\"\n" @@ -1064,7 +1077,7 @@ public void reserveInsideLockHonoured() throws Exception { + "p3: {\n" + " org.jenkins.plugins.lockableresources.LockableResource lr = null\n" + " echo \"Locked resource cause 3-1: not locked yet\"\n" - + " sleep 1\n" + + " sleep 2\n" + " lock(label: 'label1', variable: 'someVar3') {\n" + " echo \"VAR3 IS $env.someVar3\"\n" + " lr = " @@ -1078,14 +1091,7 @@ public void reserveInsideLockHonoured() throws Exception { + " }\n" + " echo \"Locked resource cause 3-3: ${lr.getLockCause()}\"\n" + " echo \"Locked resource reservedBy 3-3: ${lr.getReservedBy()}\"\n" - + "},\n" - // Add some pressure to try for race conditions: - + "p4: { sleep 2; lock(label: 'label1') { sleep 1 } },\n" - + "p5: { sleep 2; lock(label: 'label1') { sleep 3 } },\n" - + "p6: { sleep 2; lock(label: 'label1') { sleep 2 } },\n" - + "p7: { sleep 2; lock(label: 'label1') { sleep 1 } },\n" - + "p8: { sleep 2; lock(label: 'label1') { sleep 2 } },\n" - + "p9: { sleep 2; lock(label: 'label1') { sleep 1 } }\n" + + "}\n" + "\necho \"Survived the test\"\n" + "}", // timeout wrapper false)); @@ -1184,7 +1190,7 @@ public void reserveInsideLockHonoured() throws Exception { j.assertLogContains("Locked resource cause 3-2", b1); LOGGER.info("GOOD: lock#3 was taken just after we recycled lock#2"); - j.assertLogContains("is locked, waiting...", b1); + j.assertLogContains(", waiting for execution ...", b1); j.assertBuildStatusSuccess(j.waitForCompletion(b1)); @@ -1372,7 +1378,7 @@ public void setReservedByInsideLockHonoured() throws Exception { j.waitForMessage("Locked resource cause 2-2", b1); j.assertLogContains("Locked resource cause 1-5", b1); - j.assertLogContains("is locked, waiting...", b1); + j.assertLogContains(", waiting for execution ...", b1); j.assertBuildStatusSuccess(j.waitForCompletion(b1)); @@ -1401,7 +1407,10 @@ public void skipIfLocked() throws Exception { "lock(resource: 'resource1', skipIfLocked: true) {\n" + " echo 'Running body'\n" + "}", true)); WorkflowRun b1 = p.scheduleBuild2(0).waitForStart(); j.assertBuildStatusSuccess(j.waitForCompletion(b1)); - j.assertLogContains("[resource1] is locked, skipping execution...", b1); + // check: The resource [resource1] is reserved by test at Sep 1, 2023, 8:29 PM, skipping + // execution... + j.assertLogContains("The resource [resource1] is reserved by test", b1); + j.assertLogContains(", skipping execution ...", b1); j.assertLogNotContains("Running body", b1); } diff --git a/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_manualUnreserveUnblocksJob.java b/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_manualUnreserveUnblocksJob.java index 65e89d477..c44f06abb 100644 --- a/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_manualUnreserveUnblocksJob.java +++ b/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_manualUnreserveUnblocksJob.java @@ -43,7 +43,7 @@ public void manualUnreserveUnblocksJob() throws Exception { p.setDefinition(new CpsFlowDefinition("lock('resource1') {\n" + " echo('I am inside')\n" + "}\n", true)); WorkflowRun r = p.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[resource1] is locked, waiting...", r); + j.waitForMessage("[resource1] is not free, waiting for execution ...", r); j.assertLogNotContains("I am inside", r); TestHelpers.clickButton(wc, "unreserve"); j.waitForMessage("I am inside", r); diff --git a/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_reserveInsideLockHonoured.java b/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_reserveInsideLockHonoured.java index ad1b30e25..af892f79e 100644 --- a/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_reserveInsideLockHonoured.java +++ b/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_reserveInsideLockHonoured.java @@ -252,7 +252,7 @@ public void reserveInsideLockHonoured() throws Exception { j.assertLogContains("Locked resource cause 3-2", b1); LOGGER.info("GOOD: lock#3 was taken just after we recycled lock#2"); - j.assertLogContains("is locked, waiting...", b1); + j.assertLogContains(", waiting for execution ...", b1); j.assertBuildStatusSuccess(j.waitForCompletion(b1)); diff --git a/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_setReservedByInsideLockHonoured.java b/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_setReservedByInsideLockHonoured.java index abbd83a7d..565bfb505 100644 --- a/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_setReservedByInsideLockHonoured.java +++ b/src/test/java/org/jenkins/plugins/lockableresources/LockStepTest_setReservedByInsideLockHonoured.java @@ -196,7 +196,7 @@ public void setReservedByInsideLockHonoured() throws Exception { j.waitForMessage("Locked resource cause 2-2", b1); j.assertLogContains("Locked resource cause 1-5", b1); - j.assertLogContains("is locked, waiting...", b1); + j.assertLogContains(", waiting for execution ...", b1); j.assertBuildStatusSuccess(j.waitForCompletion(b1)); diff --git a/src/test/java/org/jenkins/plugins/lockableresources/LockStepWithRestartTest.java b/src/test/java/org/jenkins/plugins/lockableresources/LockStepWithRestartTest.java index 4cfc5ad79..d2f13487b 100644 --- a/src/test/java/org/jenkins/plugins/lockableresources/LockStepWithRestartTest.java +++ b/src/test/java/org/jenkins/plugins/lockableresources/LockStepWithRestartTest.java @@ -25,17 +25,18 @@ public void lockOrderRestart() throws Throwable { LockableResourcesManager.get().createResource("resource1"); WorkflowJob p = j.jenkins.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( - "lock('resource1') {\n" + " semaphore 'wait-inside'\n" + "}\n" + "echo 'Finish'", true)); + "lock('resource1') {\n" + " semaphore 'wait-inside-lockOrderRestart'\n" + "}\n" + "echo 'Finish'", + true)); WorkflowRun b1 = p.scheduleBuild2(0).waitForStart(); - SemaphoreStep.waitForStart("wait-inside/1", b1); + SemaphoreStep.waitForStart("wait-inside-lockOrderRestart/1", b1); WorkflowRun b2 = p.scheduleBuild2(0).waitForStart(); // Ensure that b2 reaches the lock before b3 - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b2); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b2); isPaused(b2, 1, 1); WorkflowRun b3 = p.scheduleBuild2(0).waitForStart(); // Both 2 and 3 are waiting for locking resource1 - j.waitForMessage("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b3); + j.waitForMessage("[resource1] is locked by build " + b1.getFullDisplayName(), b3); isPaused(b3, 1, 1); }); @@ -46,18 +47,18 @@ public void lockOrderRestart() throws Throwable { WorkflowRun b3 = p.getBuildByNumber(3); // Unlock resource1 - SemaphoreStep.success("wait-inside/1", null); + SemaphoreStep.success("wait-inside-lockOrderRestart/1", null); j.waitForMessage("Lock released on resource [resource1]", b1); isPaused(b1, 1, 0); j.waitForMessage("Lock acquired on [resource1]", b2); isPaused(b2, 1, 0); - j.assertLogContains("[resource1] is locked by " + b1.getFullDisplayName() + ", waiting...", b3); + j.assertLogContains("[resource1] is locked by build " + b1.getFullDisplayName(), b3); isPaused(b3, 1, 1); - SemaphoreStep.success("wait-inside/2", null); - SemaphoreStep.waitForStart("wait-inside/3", b3); + SemaphoreStep.success("wait-inside-lockOrderRestart/2", null); + SemaphoreStep.waitForStart("wait-inside-lockOrderRestart/3", b3); j.assertLogContains("Lock acquired on [resource1]", b3); - SemaphoreStep.success("wait-inside/3", null); + SemaphoreStep.success("wait-inside-lockOrderRestart/3", null); j.waitForMessage("Finish", b3); isPaused(b3, 1, 0); }); @@ -69,9 +70,11 @@ public void interoperabilityOnRestart() throws Throwable { LockableResourcesManager.get().createResource("resource1"); WorkflowJob p = j.jenkins.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( - "lock('resource1') {\n" + " semaphore 'wait-inside'\n" + "}\n" + "echo 'Finish'", true)); + "lock('resource1') {\n" + " semaphore 'wait-inside-interoperabilityOnRestart'\n" + "}\n" + + "echo 'Finish'", + true)); WorkflowRun b1 = p.scheduleBuild2(0).waitForStart(); - SemaphoreStep.waitForStart("wait-inside/1", b1); + SemaphoreStep.waitForStart("wait-inside-interoperabilityOnRestart/1", b1); isPaused(b1, 1, 0); FreeStyleProject f = j.createFreeStyleProject("f"); @@ -87,7 +90,7 @@ public void interoperabilityOnRestart() throws Throwable { WorkflowRun b1 = p.getBuildByNumber(1); // Unlock resource1 - SemaphoreStep.success("wait-inside/1", null); + SemaphoreStep.success("wait-inside-interoperabilityOnRestart/1", null); j.waitForMessage("Lock released on resource [resource1]", b1); isPaused(b1, 1, 0); @@ -98,6 +101,10 @@ public void interoperabilityOnRestart() throws Throwable { } j.waitForMessage("acquired lock on [resource1]", fb1); + j.waitForMessage("Finish", b1); + isPaused(b1, 1, 0); + + j.waitUntilNoActivity(); }); } @@ -112,7 +119,7 @@ public void testReserveOverRestart() throws Throwable { p.setDefinition(new CpsFlowDefinition( "lock('resource1') {\n" + " echo 'inside'\n" + "}\n" + "echo 'Finish'", true)); WorkflowRun b1 = p.scheduleBuild2(0).waitForStart(); - j.waitForMessage("[resource1] is locked, waiting...", b1); + j.waitForMessage("The resource [resource1] is reserved by user", b1); isPaused(b1, 1, 1); FreeStyleProject f = j.createFreeStyleProject("f"); diff --git a/src/test/java/org/jenkins/plugins/lockableresources/LockableResourceManagerTest.java b/src/test/java/org/jenkins/plugins/lockableresources/LockableResourceManagerTest.java index d76608e24..073a72846 100644 --- a/src/test/java/org/jenkins/plugins/lockableresources/LockableResourceManagerTest.java +++ b/src/test/java/org/jenkins/plugins/lockableresources/LockableResourceManagerTest.java @@ -28,7 +28,7 @@ public class LockableResourceManagerTest { public void validationFailure() throws Exception { RequiredResourcesProperty.DescriptorImpl d = new RequiredResourcesProperty.DescriptorImpl(); LockableResourcesManager.get().createResource("resource1"); - LockableResource r = LockableResourcesManager.get().getResources().get(0); + LockableResource r = LockableResourcesManager.get().getFirst(); r.setLabels("some-label"); assertEquals( diff --git a/src/test/java/org/jenkins/plugins/lockableresources/PressureTest.java b/src/test/java/org/jenkins/plugins/lockableresources/PressureTest.java new file mode 100644 index 000000000..af9ed96c5 --- /dev/null +++ b/src/test/java/org/jenkins/plugins/lockableresources/PressureTest.java @@ -0,0 +1,207 @@ +package org.jenkins.plugins.lockableresources; + +import hudson.Functions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Logger; +import org.jenkins.plugins.lockableresources.util.Constants; +import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.recipes.WithTimeout; + +public class PressureTest extends LockStepTestBase { + + private static final Logger LOGGER = Logger.getLogger(PressureTest.class.getName()); + + @Rule + public JenkinsRule j = new JenkinsRule(); + + /** + * Pressure test to lock resources via labels, resource name, ephemeral ... It simulates big + * system with many chaotic locks. Hopefully it runs always good, because any analysis here will + * be very hard. + */ + @Test + // it depends on which node you are running + @WithTimeout(900) + public void pressureEnableSave() throws Exception { + // keep in mind, that the windows nodes at jenkins-infra are not very fast + pressure(Functions.isWindows() ? 10 : 20); + } + + @Test + @WithTimeout(900) + public void pressureDisableSave() throws Exception { + System.setProperty(Constants.SYSTEM_PROPERTY_DISABLE_SAVE, "true"); + // keep in mind, that the windows nodes at jenkins-infra are not very fast + pressure(Functions.isWindows() ? 10 : 20); + } + + public void pressure(final int resourcesCount) throws Exception { + // count of parallel jobs + final int jobsCount = (resourcesCount / 2) + 1; + final int nodesCount = (resourcesCount / 10) + 1; + // enable node mirroring to make more chaos + System.setProperty(Constants.SYSTEM_PROPERTY_ENABLE_NODE_MIRROR, "true"); + System.setProperty(Constants.SYSTEM_PROPERTY_PRINT_BLOCKED_RESOURCE, "0"); + LockableResourcesManager lrm = LockableResourcesManager.get(); + + // create resources + LOGGER.info("Create resources with labels"); + for (int i = 1; i <= resourcesCount; i++) { + lrm.createResourceWithLabel("resourceA_" + Integer.toString(i), "label1 label2"); + lrm.createResourceWithLabel("resourceAA_" + Integer.toString(i), "label"); + lrm.createResourceWithLabel("resourceAAA_" + Integer.toString(i), "label1"); + lrm.createResourceWithLabel("resourceAAAA_" + Integer.toString(i), "(=%/!(/)?$/ HH( RU))"); + } + + // define groovy script used by our test jobs + String pipeCode = ""; + + pipeCode += "lock('initpipe') {echo 'initialized'};\n"; + + pipeCode += "def stages = [:];\n"; + pipeCode += "for(int i = 1; i < " + + resourcesCount + + "; i++) {\n" + + " final int index = i;\n" + + " String stageName = 'stage_' + index;\n" + + " stages[stageName] = {\n" + // + " echo 'my stage: ' + stageName;\n" + // + " echo 'test: label1 && label2 at ' + index;\n" + + " lock(label: 'label1 && label2', variable: 'someVar', quantity : 1) {\n" + // + " echo \"*** VAR-1 IS $env.someVar\"\n" + + " }\n" + // + " echo 'test: label2 at ' + index;\n" + + " lock(label: 'label1', variable: 'someVar', quantity : 5) {\n" + // + " echo \"*** VAR-3 IS $env.someVar\"\n" + + " }\n" + // + " echo 'test: resource_ephemeral_' + stageName;\n" + + " lock('resource_ephemeral_' + stageName) {\n" + // + " echo \"*** locked resource_ephemeral_\" + stageName\n" + + " }\n" + // + " echo 'test: resourceA_' + index;\n" + + " lock('resourceA_' + index) {\n" + // + " echo \"*** locked resourceA_\" + index\n" + + " }\n" + // recursive lock + // do not activate it, this break down the execution time + // and we need to speed up actions + // + " lock(label: 'label2', quantity : 1) {\n" + // + " lock(label: 'label2', quantity : 1, inversePrecedence : true) {\n" + // + " lock(label: 'label2', quantity : 4, skipIfLocked: true, + // resourceSelectStrategy: 'random') {\n" + // + " lock('resource_ephemeral_' + stageName) {\n" + // + " lock('resourceA_' + index) {\n" + // + " echo \"inside recursive lock \" + stageName\n" + // + " }\n" + // + " }\n" + // + " }\n" + // + " }\n" + // + " }\n" + + " }\n" + + "}\n"; + + pipeCode += "parallel stages;"; + + // reserve 'initpipe' resource to be sure that parallel builds and stages are paused at the same + // time. + LOGGER.info("Lock execution by 'initpipe'"); + lrm.createResourceWithLabel("initpipe", "sync step"); + lrm.reserve(Collections.singletonList(lrm.fromName("initpipe")), "test"); + + // create first project and start the build + LOGGER.info("Create project"); + WorkflowJob p = j.jenkins.createProject(WorkflowJob.class, "main-project"); + p.setDefinition(new CpsFlowDefinition(pipeCode, true)); + WorkflowRun b1 = p.scheduleBuild2(0).waitForStart(); + j.waitForMessage(", waiting for execution ...", b1); + + List otherBuilds = new ArrayList<>(); + + // create extra jobs to make more chaos + LOGGER.info("Create extra projects"); + for (int i = 1; i <= jobsCount; i++) { + WorkflowJob p2 = j.jenkins.createProject(WorkflowJob.class, "extra-project_" + i); + p2.setDefinition(new CpsFlowDefinition(pipeCode, true)); + WorkflowRun b2 = p2.scheduleBuild2(0).waitForStart(); + otherBuilds.add(b2); + j.waitForMessage(", waiting for execution ...", b2); + } + + // create more resources until the first job has been started + LOGGER.info("Create more resources"); + for (int i = 1; i <= resourcesCount; i++) { + lrm.createResourceWithLabel("resourceB_" + Integer.toString(i), "label1"); + } + + // create jenkins nodes. All shall be mirrored to resources + LOGGER.info("Create jenkins nodes"); + for (int i = 1; i <= nodesCount; i++) { + j.createSlave("AgentAAA_" + i, "label label1 label2", null); + lrm.createResourceWithLabel("resourceC_" + Integer.toString(i), "label1"); + j.createSlave("AGENT_BBB_" + i, null, null); + } + + // unreserve it now, and the fun may starts. Because all the parallel jobs and stages will be + // "free" + LOGGER.info("Start the chaos"); + lrm.unreserve(Collections.singletonList(lrm.fromName("initpipe"))); + + // create more resources until the first job has been started + LOGGER.info("Additional resources"); + for (int i = 1; i <= resourcesCount; i++) { + lrm.createResourceWithLabel("resourceD_" + Integer.toString(i), "label1"); + } + + // create more jenkins nodes to make more chaos + LOGGER.info("Create slaves"); + for (int i = 1; i <= nodesCount; i++) { + j.createSlave("AgentCCC_" + i, "label label1 label2", null); + j.createSlave("AGENT_DDD_" + i, null, null); + } + + // simulate chaos by user actions + LOGGER.info("User action 'reserve'"); + for (int i = 1; i <= resourcesCount; i++) { + lrm.reserve(Collections.singletonList(lrm.fromName("resourceA_" + i)), "test"); + } + + LOGGER.info("User action 'reserve' slaves"); + for (int i = 1; i <= nodesCount; i++) { + lrm.reserve(Collections.singletonList(lrm.fromName("AgentCCC_" + i)), "test"); + lrm.reserve(Collections.singletonList(lrm.fromName("AGENT_DDD_" + i)), "test"); + } + + LOGGER.info("User action 'reassign', 'steal', 'unreserve'"); + for (int i = 1; i <= resourcesCount; i++) { + lrm.reassign(Collections.singletonList(lrm.fromName("resourceA_" + i)), "second user"); + lrm.steal(Collections.singletonList(lrm.fromName("resourceAA_" + i)), "second user"); + lrm.unreserve(Collections.singletonList(lrm.fromName("resourceA_" + i))); + lrm.unreserve(Collections.singletonList(lrm.fromName("resourceAA_" + i))); + } + + LOGGER.info("User action 'unreserve' slaves"); + for (int i = 1; i <= nodesCount; i++) { + lrm.unreserve(Collections.singletonList(lrm.fromName("AgentCCC_" + i))); + lrm.unreserve(Collections.singletonList(lrm.fromName("AGENT_DDD_" + i))); + } + + // wait until the first build has been stopped + LOGGER.info("Wait for build b1"); + j.assertBuildStatusSuccess(j.waitForCompletion(b1)); + LOGGER.info("build b1: done"); + + // wait until all parallel jobs has been stopped + for (WorkflowRun b2 : otherBuilds) { + LOGGER.info("Wait for build " + b2.getAbsoluteUrl()); + j.assertBuildStatusSuccess(j.waitForCompletion(b2)); + LOGGER.info("build " + b2.getUrl() + ": done"); + } + } +}