From 35f5cd1ee6c14f36d5321adb5525299a506da65f Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Thu, 14 Jan 2021 15:16:58 +0100 Subject: [PATCH 1/2] Add Publish Verb to Control Panel The Publish verb takes a "trigger" and "targets" parameter just as the ootb TriggerAutoPublishSyncedItems processor. - trigger can be an ID or path - targets is a comma seperated list of publish target names (ea "web" or "web,preview") This verb can be used to trigger the Unicorn Publish Manual Queue incase you desire to delay its execution for DevOps reasons. Make sure to disable the TriggerAutoPublishSyncedItems processor if this is desired. --- doc/PowerShell Remote Scripting/Unicorn.psm1 | 97 +++++++++++++++++-- doc/PowerShell Remote Scripting/sample.ps1 | 9 ++ .../UnicornControlPanelRequest/PublishVerb.cs | 97 +++++++++++++++++++ .../Unicorn.AutoPublish.config | 5 +- .../Standard Config Files/Unicorn.UI.config | 1 + src/Unicorn/Unicorn.csproj | 1 + 6 files changed, 200 insertions(+), 10 deletions(-) create mode 100644 src/Unicorn/ControlPanel/Pipelines/UnicornControlPanelRequest/PublishVerb.cs diff --git a/doc/PowerShell Remote Scripting/Unicorn.psm1 b/doc/PowerShell Remote Scripting/Unicorn.psm1 index 0cdcca0..0fd091f 100644 --- a/doc/PowerShell Remote Scripting/Unicorn.psm1 +++ b/doc/PowerShell Remote Scripting/Unicorn.psm1 @@ -10,6 +10,7 @@ $global:unicornWarnings = @{ } [int]$Global:totalProjectCount Function Sync-Unicorn { + [cmdletbinding()] Param( [Parameter(Mandatory = $True)] [string]$ControlPanelUrl, @@ -19,7 +20,7 @@ Function Sync-Unicorn { [string[]]$Configurations, - [string]$Verb = 'Sync', + [string]$Verb = "Sync", [switch]$SkipTransparentConfigs, @@ -42,10 +43,88 @@ Function Sync-Unicorn { $skipValue = 1 } - $url = "{0}?verb={1}&configuration={2}&skipTransparentConfigs={3}" -f $ControlPanelUrl, $Verb, $parsedConfigurations, $skipValue + $url = "{0}?verb={1}&configuration={2}&skipTransparentConfigs={3}" -f $ControlPanelUrl, $Verb, $parsedConfigurations, $skipValue + $params = @{ + ControlPanelUrl = $ControlPanelUrl + VerbUrl = $url + SharedSecret = $SharedSecret + SleepTime = $SleepTime + } + if ($DebugSecurity) { + $params["DebugSecurity"] = $true + } + if ($StreamLogs) { + $params["StreamLogs"] = $true + } + + Invoke-UnicornControlPanel @params +} + +Function Publish-Unicorn { + [cmdletbinding()] + Param( + [Parameter(Mandatory = $True)] + [string]$ControlPanelUrl, + + [Parameter(Mandatory = $True)] + [string]$SharedSecret, + + [string]$Verb = "Publish", + + # Item ID or path to an item used as Publish Trigger + [string]$TriggerItem = "/sitecore/templates/common/folder", + + # List of Publish target database names + [string[]]$Targets = @("web"), + + [switch]$DebugSecurity, + + # defines, if logs shall be streamed to output + [switch]$StreamLogs, + [int]$SleepTime = 30 + ) + + $parsedTargets = $Targets -join "," + + $url = "{0}?verb={1}&trigger={2}&targets={3}" -f $ControlPanelUrl, $Verb, $TriggerItem, $parsedTargets + + $params = @{ + ControlPanelUrl = $ControlPanelUrl + VerbUrl = $url + SharedSecret = $SharedSecret + SleepTime = $SleepTime + } + if ($DebugSecurity) { + $params["DebugSecurity"] = $true + } + if ($StreamLogs) { + $params["StreamLogs"] = $true + } + + Invoke-UnicornControlPanel @params +} + +Function Invoke-UnicornControlPanel { + Param( + [Parameter(Mandatory = $True)] + [string]$ControlPanelUrl, + + [Parameter(Mandatory = $True)] + [string]$VerbUrl, + + [Parameter(Mandatory = $True)] + [string]$SharedSecret, + + [switch]$DebugSecurity, + + # defines, if logs shall be streamed to output + [switch]$StreamLogs, + [int]$SleepTime = 30 + ) + if ($DebugSecurity) { - Write-Host "Sync-Unicorn: Preparing authorization for $url" + Write-Host "Sync-Unicorn: Preparing authorization for $VerbUrl" } # GET AN AUTH CHALLENGE @@ -58,7 +137,7 @@ Function Sync-Unicorn { # CREATE A SIGNATURE WITH THE SHARED SECRET AND CHALLENGE $signatureService = New-Object MicroCHAP.SignatureService -ArgumentList $SharedSecret - $signature = $signatureService.CreateSignature($challenge, $url, $null) + $signature = $signatureService.CreateSignature($challenge, $VerbUrl, $null) if ($DebugSecurity) { Write-Host "Sync-Unicorn: MAC '$($signature.SignatureSource)'" @@ -74,19 +153,19 @@ Function Sync-Unicorn { # USING THE SIGNATURE, EXECUTE UNICORN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - $result = Invoke-StreamingWebRequest -Uri $url -Mac $signature.SignatureHash -Nonce $challenge -RequestVerb $Verb + $result = Invoke-StreamingWebRequest -Uri $VerbUrl -Mac $signature.SignatureHash -Nonce $challenge -RequestVerb $Verb while ($result.Trim().ToLowerInvariant() -eq "Sync in progress".ToLowerInvariant()) { Write-Host "Sync is still running, sleeping for $SleepTime seconds" Start-Sleep $SleepTime # renew challenge and signature $challenge = Get-Challenge -ControlPanelUrl $ControlPanelUrl - $signature = $signatureService.CreateSignature($challenge, $url, $null) - $result = Invoke-StreamingWebRequest -Uri $url -Mac $signature.SignatureHash -Nonce $challenge -RequestVerb $Verb + $signature = $signatureService.CreateSignature($challenge, $VerbUrl, $null) + $result = Invoke-StreamingWebRequest -Uri $VerbUrl -Mac $signature.SignatureHash -Nonce $challenge -RequestVerb $Verb } if ($result.TrimEnd().EndsWith('****ERROR OCCURRED****')) { - throw "Unicorn $Verb to $url returned an error. See the preceding log for details." + throw "Unicorn Control Panel invoke of $VerbUrl returned an error. See the preceding log for details." } # Uncomment this if you want the console results to be returned by the function @@ -232,4 +311,4 @@ Function Invoke-StreamingWebRequest($Uri, $MAC, $Nonce, $RequestVerb) { return $resultingData } -Export-ModuleMember -Function Sync-Unicorn +Export-ModuleMember -Function Sync-Unicorn,Publish-Unicorn diff --git a/doc/PowerShell Remote Scripting/sample.ps1 b/doc/PowerShell Remote Scripting/sample.ps1 index b97d41c..3bccafb 100644 --- a/doc/PowerShell Remote Scripting/sample.ps1 +++ b/doc/PowerShell Remote Scripting/sample.ps1 @@ -18,6 +18,15 @@ Sync-Unicorn -ControlPanelUrl 'https://localhost/unicorn.aspx' -SharedSecret 'yo # SYNC ALL CONFIGURATIONS WITHOUT KEEPING CONNECTION OPEN ALL THE TIME (specify configurations if you need only some of them). This is introduced to fix edge case, described https://github.com/SitecoreUnicorn/Unicorn/issues/387 Sync-Unicorn -ControlPanelUrl 'https://localhost/unicorn.aspx' -SharedSecret 'your-sharedsecret-here' -Verb 'SyncSilent' +# PUBLISH UNICORN MANUAL QUEUE +Publish-Unicorn -ControlPanelUrl 'https://localhost/unicorn.aspx' -SharedSecret 'your-sharedsecret-here' + +# PUBLISH UNICORN MANUAL QUEUE, CUSTOM TARGETS +Publish-Unicorn -ControlPanelUrl 'https://localhost/unicorn.aspx' -SharedSecret 'your-sharedsecret-here' -Targets @('web','preview') + +# PUBLISH UNICORN MANUAL QUEUE, SPECIFIC TRIGGER ITEM +Publish-Unicorn -ControlPanelUrl 'https://localhost/unicorn.aspx' -SharedSecret 'your-sharedsecret-here' -TriggerItem 'item-path-or-id' + # Note: you may pass -Verb 'Reserialize' for remote reserialize. Usually not needed though. # Note: If you are having authorization issues, add -DebugSecurity to your cmdlet invocation; this will display the raw signatures being used to compare to the server. \ No newline at end of file diff --git a/src/Unicorn/ControlPanel/Pipelines/UnicornControlPanelRequest/PublishVerb.cs b/src/Unicorn/ControlPanel/Pipelines/UnicornControlPanelRequest/PublishVerb.cs new file mode 100644 index 0000000..8b6e7ef --- /dev/null +++ b/src/Unicorn/ControlPanel/Pipelines/UnicornControlPanelRequest/PublishVerb.cs @@ -0,0 +1,97 @@ +using System; +using System.Linq; +using System.Web; + +using Kamsar.WebConsole; + +using Sitecore.Configuration; +using Sitecore.Data; +using Sitecore.Data.Items; +using Sitecore.Diagnostics; + +using Unicorn.ControlPanel.Headings; +using Unicorn.ControlPanel.Responses; +using Unicorn.Logging; +using Unicorn.Publishing; + +namespace Unicorn.ControlPanel.Pipelines.UnicornControlPanelRequest +{ + public class PublishVerb : UnicornControlPanelRequestPipelineProcessor + { + public PublishVerb() : base("Publish") + { + } + + protected override IResponse CreateResponse(UnicornControlPanelRequestPipelineArgs args) + { + return new WebConsoleResponse("Publish Unicorn Queue", args.SecurityState.IsAutomatedTool, new HeadingService(), progress => Process(progress, new WebConsoleLogger(progress, args.Context.Request.QueryString["log"]))); + } + + protected virtual void Process(IProgressStatus progress, ILogger logger) + { + try + { + if (ManualPublishQueueHandler.HasItemsToPublish) + { + Item trigger = GetTriggerItem(); + Database[] targets = GetTargets(); + + Log.Info("Unicorn: initiated synchronous publishing of synced items.", this); + if (ManualPublishQueueHandler.PublishQueuedItems(trigger, targets, logger)) + { + Log.Info("Unicorn: publishing of synced items is complete.", this); + } + } + else + { + logger.Warn("[Unicorn Publish] There were no items to publish."); + } + } + catch (Exception ex) + { + logger.Error(ex); + } + } + + protected virtual Item GetTriggerItem() + { + Item result; + string triggerIdOrPath = HttpContext.Current.Request.QueryString["trigger"]; + if (!string.IsNullOrWhiteSpace(triggerIdOrPath)) + { + result = Factory.GetDatabase("master").GetItem(triggerIdOrPath); + if (result == null || result.Empty) + { + throw new ArgumentException($"No item found for '{triggerIdOrPath}'.", "trigger"); + } + } + else + { + throw new ArgumentNullException("trigger"); + } + + return result; + } + + protected virtual Database[] GetTargets() + { + Database[] result; + string targets = HttpContext.Current.Request.QueryString["targets"]; + if (!string.IsNullOrWhiteSpace(targets)) + { + string[] targetNames = targets.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); + result = targetNames.Select(Factory.GetDatabase).ToArray(); + if (result.Length <= 0) + { + throw new ArgumentException("At least 1 valid target should be specified.", "targets"); + } + } + else + { + throw new ArgumentNullException("targets"); + } + + return result; + } + } +} diff --git a/src/Unicorn/Standard Config Files/Unicorn.AutoPublish.config b/src/Unicorn/Standard Config Files/Unicorn.AutoPublish.config index bef4e0c..bd4627a 100644 --- a/src/Unicorn/Standard Config Files/Unicorn.AutoPublish.config +++ b/src/Unicorn/Standard Config Files/Unicorn.AutoPublish.config @@ -16,7 +16,10 @@ - + /sitecore/templates/Common/Folder diff --git a/src/Unicorn/Standard Config Files/Unicorn.UI.config b/src/Unicorn/Standard Config Files/Unicorn.UI.config index f8a00eb..0056ec9 100644 --- a/src/Unicorn/Standard Config Files/Unicorn.UI.config +++ b/src/Unicorn/Standard Config Files/Unicorn.UI.config @@ -75,6 +75,7 @@ + diff --git a/src/Unicorn/Unicorn.csproj b/src/Unicorn/Unicorn.csproj index 8088588..9678d33 100644 --- a/src/Unicorn/Unicorn.csproj +++ b/src/Unicorn/Unicorn.csproj @@ -91,6 +91,7 @@ + From 192e8fba321f836c6d373ce67fe41e029e69b613 Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Tue, 9 Feb 2021 14:43:18 +0100 Subject: [PATCH 2/2] Added persistance options to manual publish queue to support post app-reset publishing --- .../Pipelines/Initialize/ManualQueueLoad.cs | 18 +++ .../UnicornSyncEnd/ManualQueuePersistence.cs | 16 ++ .../Publishing/ManualPublishQueueHandler.cs | 150 ++++++++++++++---- .../Unicorn.AutoPublish.config | 11 ++ src/Unicorn/Unicorn.csproj | 2 + 5 files changed, 163 insertions(+), 34 deletions(-) create mode 100644 src/Unicorn/Pipelines/Initialize/ManualQueueLoad.cs create mode 100644 src/Unicorn/Pipelines/UnicornSyncEnd/ManualQueuePersistence.cs diff --git a/src/Unicorn/Pipelines/Initialize/ManualQueueLoad.cs b/src/Unicorn/Pipelines/Initialize/ManualQueueLoad.cs new file mode 100644 index 0000000..659fdd5 --- /dev/null +++ b/src/Unicorn/Pipelines/Initialize/ManualQueueLoad.cs @@ -0,0 +1,18 @@ +using Sitecore.Diagnostics; +using Sitecore.Pipelines; + +using Unicorn.Publishing; + +namespace Unicorn.Pipelines.Initialize +{ + public class ManualQueueLoad + { + public void Process(PipelineArgs args) + { + if (ManualPublishQueueHandler.LoadFromPersistentStore()) + { + Log.Info("Loaded persisted unicorn queue.", this); + } + } + } +} diff --git a/src/Unicorn/Pipelines/UnicornSyncEnd/ManualQueuePersistence.cs b/src/Unicorn/Pipelines/UnicornSyncEnd/ManualQueuePersistence.cs new file mode 100644 index 0000000..97a9df6 --- /dev/null +++ b/src/Unicorn/Pipelines/UnicornSyncEnd/ManualQueuePersistence.cs @@ -0,0 +1,16 @@ +using Unicorn.Publishing; + +namespace Unicorn.Pipelines.UnicornSyncEnd +{ + /// + /// Persists the Manual Publish Queue generated by Unicorn for its synched items. + /// This is required due to the app pool recycle that will be caused by swapping between sync and publish. + /// + public class ManualQueuePersistence : IUnicornSyncEndProcessor + { + public void Process(UnicornSyncEndPipelineArgs args) + { + ManualPublishQueueHandler.Persist(); + } + } +} diff --git a/src/Unicorn/Publishing/ManualPublishQueueHandler.cs b/src/Unicorn/Publishing/ManualPublishQueueHandler.cs index e1bb2c7..1c65e96 100644 --- a/src/Unicorn/Publishing/ManualPublishQueueHandler.cs +++ b/src/Unicorn/Publishing/ManualPublishQueueHandler.cs @@ -1,10 +1,15 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; -using Kamsar.WebConsole; +using System.Text; + +using Sitecore; using Sitecore.Configuration; using Sitecore.Data; using Sitecore.Data.Items; +using Sitecore.IO; using Sitecore.Publishing; using Sitecore.Publishing.Pipelines.Publish; using Unicorn.Logging; @@ -15,51 +20,133 @@ namespace Unicorn.Publishing /// Maintains a manual publish queue that arbitrary items can be added to /// See https://www.velir.com/blog/2013/11/22/how-create-custom-publish-queue-sitecore among other sources /// + [SuppressMessage("ReSharper", "InconsistentlySynchronizedField", Justification = "ManuallyAddedCandidates is concurrent by design.")] public class ManualPublishQueueHandler : PublishProcessor { private static readonly ConcurrentQueue ManuallyAddedCandidates = new ConcurrentQueue(); + + private static readonly object PersistenceLock = new object(); + protected static bool UsePublishManager = Settings.GetBoolSetting("Unicorn.UsePublishManager", true); + protected static bool UsePublishingService = Settings.GetBoolSetting("Unicorn.UsePublishingService", false); + protected static int PublishingServiceMaxItemsToQueue = Settings.GetIntSetting("Unicorn.PublishingServiceMaxItemsToQueue", 50); + protected static string PublishQueuePersistenceFile = Settings.GetSetting("Unicorn.PublishQueuePersistenceFile", "~/App_Data/Unicorn/publishqueue.tmp"); + + public static bool HasItemsToPublish => ManuallyAddedCandidates.Count > 0; + public static void AddItemToPublish(Guid itemId) { - ManuallyAddedCandidates.Enqueue(new ID(itemId)); + lock (PersistenceLock) + { + ManuallyAddedCandidates.Enqueue(new ID(itemId)); + } } - public static bool HasItemsToPublish => ManuallyAddedCandidates.Count > 0; - public static bool PublishQueuedItems(Item triggerItem, Database[] targets, ILogger logger = null) { - if (ManuallyAddedCandidates.Count == 0) return false; - var suffix = ManuallyAddedCandidates.Count == 1 ? string.Empty : "s"; - var compareRevisions = false; + bool result; + lock (PersistenceLock) + { + result = PublishQueuedItemsInternal(triggerItem, targets, logger); + } + + return result; + } + + public static void Persist() + { + lock (PersistenceLock) + { + StringBuilder contentBuilder = new StringBuilder(); + foreach (ID id in ManuallyAddedCandidates.ToArray()) + { + contentBuilder.AppendLine(id.Guid.ToString("N")); + } + + FileUtil.WriteToFile(PublishQueuePersistenceFile, contentBuilder.ToString()); + } + } + + public static bool LoadFromPersistentStore() + { + bool result = false; + lock (PersistenceLock) + { + if (FileUtil.FileExists(PublishQueuePersistenceFile)) + { + string content = FileUtil.ReadFromFile(PublishQueuePersistenceFile); + string[] persistedIds = content.Split( + new[] { Environment.NewLine }, + StringSplitOptions.RemoveEmptyEntries); + foreach (string id in persistedIds) + { + if (Guid.TryParse(id, out Guid guidId)) + { + ManuallyAddedCandidates.Enqueue(new ID(guidId)); + result = true; + } + } + } + } + + return result; + } + + public override void Process(PublishContext context) + { + IEnumerable candidates = ManuallyAddedCandidates + .ToArray() + .Select(id => new PublishingCandidate(id, context.PublishOptions)); + + context.Queue.Add(candidates); + } + + protected static bool PublishQueuedItemsInternal(Item triggerItem, Database[] targets, ILogger logger = null) + { + if (ManuallyAddedCandidates.Count == 0) + { + return false; + } + + string suffix = ManuallyAddedCandidates.Count == 1 ? string.Empty : "s"; + const bool CompareRevisions = false; if (!UsePublishingService) { - foreach (var database in targets) + foreach (Database database in targets) { logger?.Debug($"> Publishing {ManuallyAddedCandidates.Count} synced item{suffix} in queue to {database.Name}"); - var publishOptions = new PublishOptions(triggerItem.Database, database, PublishMode.SingleItem, triggerItem.Language, DateTime.UtcNow) { RootItem = triggerItem, CompareRevisions = compareRevisions, RepublishAll = true }; + PublishOptions publishOptions = + new PublishOptions( + triggerItem.Database, + database, + PublishMode.SingleItem, + triggerItem.Language, + DateTime.UtcNow) + { + RootItem = triggerItem, CompareRevisions = CompareRevisions, RepublishAll = true + }; if (UsePublishManager) { // this works much faster then `new Publisher(publishOptions, triggerItem.Database.Languages).PublishWithResult();` - var handle = PublishManager.Publish(new PublishOptions[] { publishOptions }); - var publishingSucces = PublishManager.WaitFor(handle); + Handle handle = PublishManager.Publish(new[] { publishOptions }); + bool publishingSuccess = PublishManager.WaitFor(handle); - if (publishingSucces) + if (publishingSuccess) { logger?.Debug($"> Published synced item{suffix} to {database.Name}. Statistics is not retrievable when Publish Manager is used (see setting Unicorn.UsePublishManager comments)."); } else { - logger?.Error($"> Error happened during publishing. Check Sitecore logs for details."); + logger?.Error("> Error happened during publishing. Check Sitecore logs for details."); } - } else { - var result = new Publisher(publishOptions, triggerItem.Database.Languages).PublishWithResult(); + PublishResult result = new Publisher(publishOptions, triggerItem.Database.Languages).PublishWithResult(); logger?.Debug($"> Published synced item{suffix} to {database.Name} (New: {result.Statistics.Created}, Updated: {result.Statistics.Updated}, Deleted: {result.Statistics.Deleted} Skipped: {result.Statistics.Skipped})"); } @@ -67,10 +154,10 @@ public static bool PublishQueuedItems(Item triggerItem, Database[] targets, ILog } else { - var counter = 0; - var triggerItemDatabase = triggerItem.Database; - var deepModePublish = false; - var publishRelatedItems = false; + int counter = 0; + Database triggerItemDatabase = triggerItem.Database; + const bool DeepModePublish = false; + const bool PublishRelatedItems = false; logger?.Debug($"> Queueing {ManuallyAddedCandidates.Count} synced item{suffix} in publishing service."); @@ -79,14 +166,13 @@ public static bool PublishQueuedItems(Item triggerItem, Database[] targets, ILog // using publishing service to manually queue items in publishing service while (ManuallyAddedCandidates.Count > 0) { - ID itemId; - ManuallyAddedCandidates.TryDequeue(out itemId); + ManuallyAddedCandidates.TryDequeue(out ID itemId); - var publishCandidateItem = triggerItemDatabase.GetItem(itemId); + Item publishCandidateItem = triggerItemDatabase.GetItem(itemId); if (publishCandidateItem != null) { counter++; - PublishManager.PublishItem(publishCandidateItem, targets, triggerItemDatabase.Languages, deepModePublish, compareRevisions, publishRelatedItems); + PublishManager.PublishItem(publishCandidateItem, targets, triggerItemDatabase.Languages, DeepModePublish, CompareRevisions, PublishRelatedItems); } } @@ -103,20 +189,16 @@ public static bool PublishQueuedItems(Item triggerItem, Database[] targets, ILog // clear the queue after we publish while (ManuallyAddedCandidates.Count > 0) { - ID fake; - ManuallyAddedCandidates.TryDequeue(out fake); + ManuallyAddedCandidates.TryDequeue(out _); } - return true; - } - - public override void Process(PublishContext context) - { - var candidates = ManuallyAddedCandidates - .ToArray() - .Select(id => new PublishingCandidate(id, context.PublishOptions)); + // delete queue persistence after we publish + if (FileUtil.FileExists(PublishQueuePersistenceFile)) + { + FileUtil.Delete(PublishQueuePersistenceFile); + } - context.Queue.Add(candidates); + return true; } } } diff --git a/src/Unicorn/Standard Config Files/Unicorn.AutoPublish.config b/src/Unicorn/Standard Config Files/Unicorn.AutoPublish.config index bd4627a..fb08b71 100644 --- a/src/Unicorn/Standard Config Files/Unicorn.AutoPublish.config +++ b/src/Unicorn/Standard Config Files/Unicorn.AutoPublish.config @@ -10,6 +10,13 @@ + + @@ -28,6 +35,10 @@ web + diff --git a/src/Unicorn/Unicorn.csproj b/src/Unicorn/Unicorn.csproj index 9678d33..d4666ae 100644 --- a/src/Unicorn/Unicorn.csproj +++ b/src/Unicorn/Unicorn.csproj @@ -159,12 +159,14 @@ + +