#52 Azure cache invalidate via pooling (#59)

* #52 azure cache invalidate via pooling [draft]

* #52 blob storage pooling architecture refactoring

* #52 default azure poling interval increased

* #52 typos fix, renaming pooling interval property

VirtoCommerce.Storefront/Domain/ContentBlobProviders/AzureBlobContentOptions.cs

@@ -1,4 +1,5 @@
-using Microsoft.WindowsAzure.Storage.Blob;
+using System;
+using Microsoft.WindowsAzure.Storage.Blob;
 
 namespace VirtoCommerce.Storefront.Domain
 {
@@ -7,7 +8,7 @@ public class AzureBlobContentOptions
         public string Container { get; set; }
         public string ConnectionString { get; set; }
         public bool PollForChanges { get; set; } = false;
-        public int PollingChangesInterval { get; set; } = 5000;
+        public TimeSpan ChangesPoolingInterval { get; set; } = TimeSpan.FromSeconds(15);
         public BlobRequestOptions BlobRequestOptions { get; set; } = new BlobRequestOptions();
     }
 }

VirtoCommerce.Storefront/Domain/ContentBlobProviders/AzureBlobContentProvider.cs

@@ -8,12 +8,12 @@
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
-using VirtoCommerce.Storefront.Common;
 using VirtoCommerce.Storefront.Extensions;
 using VirtoCommerce.Storefront.Model.Common;
 using VirtoCommerce.Storefront.Model.Common.Caching;
 using VirtoCommerce.Storefront.Model.Common.Exceptions;
 using VirtoCommerce.Storefront.Model.StaticContent;
+using VirtoCommerce.Storefront.Infrastructure;
 
 namespace VirtoCommerce.Storefront.Domain
 {
@@ -24,19 +24,21 @@ public class AzureBlobContentProvider : IContentBlobProvider
         private readonly CloudBlobContainer _container;
         private readonly IMemoryCache _memoryCache;
         private readonly AzureBlobContentOptions _options;
+        private readonly IBlobChangesWatcher _watcher;
 
-        public AzureBlobContentProvider(IOptions<AzureBlobContentOptions> options, IMemoryCache memoryCache)
+        public AzureBlobContentProvider(IOptions<AzureBlobContentOptions> options, IMemoryCache memoryCache, IBlobChangesWatcher watcher)
         {
             _options = options.Value;
-            _memoryCache = memoryCache;       
+            _memoryCache = memoryCache;
 
             if (!CloudStorageAccount.TryParse(_options.ConnectionString, out _cloudStorageAccount))
             {
                 throw new StorefrontException("Failed to get valid connection string");
             }
             _cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
-            _container = _cloudBlobClient.GetContainerReference(_options.Container);       
-        }       
+            _container = _cloudBlobClient.GetContainerReference(_options.Container);
+            _watcher = watcher;
+        }
 
         #region IContentBlobProvider Members
         /// <summary>
@@ -56,7 +58,7 @@ public async virtual Task<Stream> OpenReadAsync(string path)
                 throw new ArgumentNullException(nameof(path));
             }
             path = NormalizePath(path);
-          
+
             return await _container.GetBlobReference(path).OpenReadAsync();
         }
 
@@ -93,26 +95,26 @@ public async virtual Task<bool> PathExistsAsync(string path)
         {
             path = NormalizePath(path);
             var cacheKey = CacheKey.With(GetType(), "PathExistsAsync", path);
-            return await _memoryCache.GetOrCreateExclusiveAsync(cacheKey,  async (cacheEntry) =>
-            {
-                cacheEntry.AddExpirationToken(ContentBlobCacheRegion.CreateChangeToken());
+            return await _memoryCache.GetOrCreateExclusiveAsync(cacheKey, async (cacheEntry) =>
+           {
+               cacheEntry.AddExpirationToken(ContentBlobCacheRegion.CreateChangeToken());
 
                 // If requested path is a directory we should always return true because Azure blob storage does not support checking if directories exist
                 var result = string.IsNullOrEmpty(Path.GetExtension(path));
-                if (!result)
-                {
-                    var url = GetAbsoluteUrl(path);
-                    try
-                    {
-                        result = await (await _cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(url))).ExistsAsync();
-                    }
-                    catch (Exception)
-                    {
+               if (!result)
+               {
+                   var url = GetAbsoluteUrl(path);
+                   try
+                   {
+                       result = await (await _cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(url))).ExistsAsync();
+                   }
+                   catch (Exception)
+                   {
                         //Azure blob storage client does not provide method to check blob url exist without throwing exception
                     }
-                }
-                return result;
-            });
+               }
+               return result;
+           });
         }
 
 
@@ -127,7 +129,7 @@ public virtual IEnumerable<string> Search(string path, string searchPattern, boo
         {
             return Task.Factory.StartNew(() => SearchAsync(path, searchPattern, recursive), CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult();
         }
-       
+
         public virtual async Task<IEnumerable<string>> SearchAsync(string path, string searchPattern, bool recursive)
         {
             var retVal = new List<string>();
@@ -171,20 +173,17 @@ public virtual async Task<IEnumerable<string>> SearchAsync(string path, string s
 
         public virtual IChangeToken Watch(string path)
         {
-            //TODO
-            //See https://docs.microsoft.com/en-us/azure/azure-functions/functions-bindings-storage-blob
-            return new CancellationChangeToken(new CancellationToken());
+            return _watcher.CreateBlobChangeToken(NormalizePath(path));
         }
         #endregion
 
-
         protected virtual CloudBlobDirectory GetCloudBlobDirectory(string path)
         {
             var isPathToFile = !string.IsNullOrEmpty(Path.GetExtension(path));
-            if(isPathToFile)
+            if (isPathToFile)
             {
                 path = NormalizePath(Path.GetDirectoryName(path));
-            }         
+            }
             return _container.GetDirectoryReference(path);
         }
 
@@ -204,6 +203,6 @@ protected virtual string GetAbsoluteUrl(string path)
             var builder = new UriBuilder(_cloudBlobClient.BaseUri);
             builder.Path += string.Join("/", _options.Container, path).Replace("//", "/");
             return builder.Uri.ToString();
-        }     
+        }
     }
 }

VirtoCommerce.Storefront/Infrastructure/BlobChangeToken.cs

@@ -0,0 +1,129 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Primitives;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
+using VirtoCommerce.Storefront.Domain;
+using VirtoCommerce.Storefront.Model.Common;
+
+namespace VirtoCommerce.Storefront.Infrastructure
+{
+    /// <summary>
+    /// Based on PollingFileChangeToken 
+    /// </summary>
+    public class BlobChangeToken : IChangeToken
+    {
+        private static ConcurrentDictionary<string, DateTime> _previousChangeTimeUtcTokenLookup = new ConcurrentDictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
+
+        public string BlobName { get; set; }
+        private bool _hasChanged;
+        private readonly CloudBlobContainer _container;
+        private readonly AzureBlobContentOptions _options;
+        private DateTime _lastModifiedUtc;
+        private DateTime _prevModifiedUtc;
+        private static DateTime _lastCheckedTimeUtcStatic;
+        private static object _lock = new object();
+
+        public BlobChangeToken(string blobName, CloudBlobContainer container, AzureBlobContentOptions options)
+        {
+            BlobName = blobName;
+
+            _container = container;
+            _options = options;
+
+            _lastModifiedUtc = _prevModifiedUtc = DateTime.UtcNow;
+        }
+
+        public bool HasChanged
+        {
+            get
+            {
+                //get last modified dt
+                _lastModifiedUtc = _previousChangeTimeUtcTokenLookup.GetOrAdd(BlobName, _lastModifiedUtc);
+
+                var hasChanged = _lastModifiedUtc > _prevModifiedUtc;
+                if (hasChanged)
+                {
+                    _prevModifiedUtc = _lastModifiedUtc;
+                    _hasChanged = true;
+                }
+
+                //check pooling interval
+                var currentTime = DateTime.UtcNow;
+                if (currentTime - _lastCheckedTimeUtcStatic < _options.ChangesPoolingInterval)
+                {
+                    return _hasChanged;
+                }
+
+                bool lockTaken = Monitor.TryEnter(_lock);
+                try
+                {
+                    if (lockTaken)
+                    {
+                        Task.Run(() => EvaluateBlobsModifiedDate());
+                        _lastCheckedTimeUtcStatic = currentTime;
+                    }
+                }
+                finally
+                {
+                    if (lockTaken)
+                        Monitor.Exit(_lock);
+                }
+
+                return _hasChanged;
+            }
+        }
+
+        private void EvaluateBlobsModifiedDate(CancellationToken cancellationToken = default(CancellationToken))
+        {
+            var files = ListBlobs().GetAwaiter().GetResult();
+            foreach (var file in files)
+            {
+                if (cancellationToken.IsCancellationRequested)
+                    break;
+
+                var lastModifiedUtc = file.Properties.LastModified.HasValue ? file.Properties.LastModified.Value.UtcDateTime : DateTime.MinValue;
+
+                if (!_previousChangeTimeUtcTokenLookup.TryGetValue(file.Name, out DateTime dt))
+                {
+                    _previousChangeTimeUtcTokenLookup.GetOrAdd(file.Name, lastModifiedUtc);
+                }
+                else
+                {
+                    _previousChangeTimeUtcTokenLookup[file.Name] = lastModifiedUtc;
+                }
+            }
+        }
+
+        private async Task<IEnumerable<CloudBlob>> ListBlobs()
+        {
+            var context = new OperationContext();
+            var blobItems = new List<IListBlobItem>();
+            BlobContinuationToken token = null;
+            var operationContext = new OperationContext();
+            do
+            {
+                var resultSegment = await _container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.Metadata, null, token, _options.BlobRequestOptions, operationContext);
+                token = resultSegment.ContinuationToken;
+                blobItems.AddRange(resultSegment.Results);
+            } while (token != null);
+
+            var result = blobItems.OfType<CloudBlob>().ToList();
+            return result;
+        }
+
+        /// <summary>
+        /// Don't know what to do with this one, so false
+        /// </summary>
+        public bool ActiveChangeCallbacks => false;
+
+        /// <summary>
+        /// Don't know  what to do with this either
+        /// </summary>
+        public IDisposable RegisterChangeCallback(Action<object> callback, object state) => EmptyDisposable.Instance;
+    }
+}

VirtoCommerce.Storefront/Infrastructure/BlobChangesWatcher.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Primitives;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
+using VirtoCommerce.Storefront.Domain;
+
+namespace VirtoCommerce.Storefront.Infrastructure
+{
+    public class BlobChangesWatcher : IBlobChangesWatcher
+    {
+        private readonly AzureBlobContentOptions _options;
+        private readonly CloudBlobContainer _container;
+
+        public BlobChangesWatcher(IOptions<AzureBlobContentOptions> options)
+        {
+            _options = options.Value;
+
+            if (CloudStorageAccount.TryParse(_options.ConnectionString, out CloudStorageAccount cloudStorageAccount))
+            {
+                var cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
+                _container = cloudBlobClient.GetContainerReference(_options.Container);
+            }
+        }
+
+        public IChangeToken CreateBlobChangeToken(string key)
+        {
+            if (!_options.PollForChanges || _container == null)
+            {
+                return new CancellationChangeToken(new CancellationToken());
+            }
+
+            return new BlobChangeToken(key, _container, _options);
+        }
+    }
+}

VirtoCommerce.Storefront/Infrastructure/IBlobChangesWatcher.cs

@@ -0,0 +1,9 @@
+using Microsoft.Extensions.Primitives;
+
+namespace VirtoCommerce.Storefront.Infrastructure
+{
+    public interface IBlobChangesWatcher
+    {
+        IChangeToken CreateBlobChangeToken(string path);
+    }
+}

VirtoCommerce.Storefront/Startup.cs

@@ -92,6 +92,7 @@ public void ConfigureServices(IServiceCollection services)
             services.AddSingleton<IRecommendationProviderFactory, RecommendationProviderFactory>(provider => new RecommendationProviderFactory(provider.GetService<AssociationRecommendationsProvider>(), provider.GetService<CognitiveRecommendationsProvider>()));
             services.AddTransient<IQuoteRequestBuilder, QuoteRequestBuilder>();
             services.AddTransient<ICartBuilder, CartBuilder>();
+            services.AddSingleton<IBlobChangesWatcher, BlobChangesWatcher>();
 
             //Register events framework dependencies
             services.AddSingleton(new InProcessBus());
@@ -113,10 +114,15 @@ public void ConfigureServices(IServiceCollection services)
             var contentConnectionString = BlobConnectionString.Parse(Configuration.GetConnectionString("ContentConnectionString"));
             if (contentConnectionString.Provider.EqualsInvariant("AzureBlobStorage"))
             {
+                var azureBlobOptions = new AzureBlobContentOptions();
+                Configuration.GetSection("VirtoCommerce:AzureBlobStorage").Bind(azureBlobOptions);
+
                 services.AddAzureBlobContent(options =>
                 {
                     options.Container = contentConnectionString.RootPath;
                     options.ConnectionString = contentConnectionString.ConnectionString;
+                    options.PollForChanges = azureBlobOptions.PollForChanges;
+                    options.ChangesPoolingInterval = azureBlobOptions.ChangesPoolingInterval;
                 });
             }
             else

VirtoCommerce.Storefront/appsettings.json

@@ -1,23 +1,27 @@
 {
-  "ConnectionStrings": {
-    "ContentConnectionString": "provider=LocalStorage;rootPath=~/cms-content"
-  },
-  "VirtoCommerce": {
-    "DefaultStore": "Electronics",
-    "Endpoint": {
-      "Url": "http://localhost/admin",
-      "AppId": "27e0d789f12641049bd0e939185b4fd2",
-      "SecretKey": "34f0a3c12c9dbb59b63b5fece955b7b2b9a3b20f84370cba1524dd5c53503a2e2cb733536ecf7ea1e77319a47084a3a2c9d94d36069a432ecc73b72aeba6ea78",
-      "RequestTimeout": "0:0:30"
+    "ConnectionStrings": {
+        "ContentConnectionString": "provider=LocalStorage;rootPath=~/cms-content"
     },
-    "ChangesPoolingInterval": "0:0:15",
-    "LiquidThemeEngine": {
-      "RethrowLiquidRenderErrors": false
-    },
-    "RequireHttps": {
-      "Enabled": false,
-      "StatusCode": "308",
-      "Port": "443"
+    "VirtoCommerce": {
+        "DefaultStore": "Electronics",
+        "Endpoint": {
+            "Url": "http://localhost/admin",
+            "AppId": "27e0d789f12641049bd0e939185b4fd2",
+            "SecretKey": "34f0a3c12c9dbb59b63b5fece955b7b2b9a3b20f84370cba1524dd5c53503a2e2cb733536ecf7ea1e77319a47084a3a2c9d94d36069a432ecc73b72aeba6ea78",
+            "RequestTimeout": "0:0:30"
+        },
+        "ChangesPoolingInterval": "0:0:15",
+        "LiquidThemeEngine": {
+            "RethrowLiquidRenderErrors": false
+        },
+        "RequireHttps": {
+            "Enabled": false,
+            "StatusCode": "308",
+            "Port": "443"
+        },
+        "AzureBlobStorage": {
+            "PollForChanges": true,
+            "ChangesPoolingInterval": "0:0:15"
+        }
     }
-  } 
 }