Merge remote-tracking branch 'origin/main'

MyApp.ServiceInterface/BackgroundMqServices.cs

@@ -64,6 +64,8 @@ public async Task Any(DbWrites request)
             await Db.DeleteAsync<PostJob>(x => x.PostId == request.DeletePost);
             await Db.DeleteAsync<Vote>(x => x.PostId == request.DeletePost);
             await Db.DeleteByIdAsync<Post>(request.DeletePost);
+            var maxPostId = await Db.ScalarAsync<int>("SELECT MAX(Id) FROM Post");
+            AppConfig.Instance.SetInitialPostId(Math.Max(100_000_000, maxPostId));
         }
 
         if (request.CreatePostJobs is { Count: > 0 })
@@ -108,7 +110,7 @@ public async Task Any(DbWrites request)
                     };
                     await Db.InsertAsync(rankJob);
                     modelWorkers.Enqueue(rankJob);
-                    MessageProducer.Publish(new SearchTasks { AddPostId = postJob.PostId });
+                    MessageProducer.Publish(new SearchTasks { AddPostToIndex = postJob.PostId });
                 }
             }
         }
@@ -121,15 +123,18 @@ public async Task Any(DbWrites request)
                 }, 
                 x => x.PostId == request.FailJob.Id);
             var postJob = await Db.SingleByIdAsync<PostJob>(request.FailJob.Id);
-            if (postJob.RetryCount > 3)
+            if (postJob != null)
             {
-                await Db.UpdateOnlyAsync(() =>
-                        new PostJob { CompletedDate = DateTime.UtcNow },
-                    x => x.PostId == request.FailJob.Id);
-            }
-            else
-            {
-                modelWorkers.Enqueue(postJob);
+                if (postJob.RetryCount > 3)
+                {
+                    await Db.UpdateOnlyAsync(() =>
+                            new PostJob { CompletedDate = DateTime.UtcNow },
+                        x => x.PostId == request.FailJob.Id);
+                }
+                else
+                {
+                    modelWorkers.Enqueue(postJob);
+                }
             }
         }
 

MyApp.ServiceInterface/Data/DbExtensions.cs

@@ -1,4 +1,5 @@
-using MyApp.ServiceModel;
+using System.Data;
+using MyApp.ServiceModel;
 using ServiceStack;
 using ServiceStack.OrmLite;
 using ServiceStack.Text;
@@ -40,10 +41,59 @@ public static SqlExpression<Post> OrderByView(this SqlExpression<Post> q, string
         view = view?.ToLower();
         if (view is "popular" or "most-views")
             q.OrderByDescending(x => x.ViewCount);
-        else if (view is "interesting" or "most-votes")
+        else if (view is null or "" or "interesting" or "most-votes")
             q.OrderByDescending(x => x.Score);
         else
             q.OrderByDescending(x => x.Id);
         return q;
     }
+
+    public static SqlExpression<PostFts> OrderByView(this SqlExpression<PostFts> q, string? view)
+    {
+        view = view?.ToLower();
+        if (view == "newest")
+            q.OrderByDescending("ModifiedDate");
+        else if (view == "oldest")
+            q.OrderBy("ModifiedDate");
+        else
+            q.OrderBy("Rank");
+        return q;
+    }
+
+    public static Post ToPost(this PostFts x) => new Post
+    {
+        Id = x.RefId.LeftPart('-').ToInt(),
+        PostTypeId = x.RefId.Contains('-') ? 2 : 1,
+        Summary = x.Body.StripHtml().SubstringWithEllipsis(0, 200),
+        CreationDate = x.ModifiedDate,
+        LastEditDate = x.ModifiedDate,
+    };
+
+    public static async Task<List<Post>> PopulatePostsAsync(this IDbConnection db, List<PostFts> posts) =>
+        await db.PopulatePostsAsync(posts.Select(ToPost).ToList());
+
+    public static async Task<List<Post>> PopulatePostsAsync(this IDbConnection db, List<Post> posts)
+    {
+        var postIds = posts.Select(x => x.Id).ToSet();
+        var fullPosts = await db.SelectAsync(db.From<Post>().Where(x => postIds.Contains(x.Id)));
+        var fullPostsMap = fullPosts.ToDictionary(x => x.Id);
+
+        foreach (var post in posts)
+        {
+            if (fullPostsMap.TryGetValue(post.Id, out var fullPost))
+            {
+                post.Title = fullPost.Title;
+                post.Slug = fullPost.Slug;
+                if (post.PostTypeId == 1)
+                {
+                    post.Tags = fullPost.Tags;
+                    post.Score = fullPost.Score;
+                    post.ViewCount = fullPost.ViewCount;
+                    post.CreationDate = fullPost.CreationDate;
+                    post.LastEditDate = fullPost.LastEditDate;
+                }
+            }
+        }
+        return posts;
+    }
 }

MyApp.ServiceInterface/Data/RendererCache.cs

@@ -70,6 +70,17 @@ public async Task SetHomeTabHtmlAsync(string? tab, string html)
         await File.WriteAllTextAsync(filePath, html);
     }
 
+    public void DeleteHomeTabHtml(string? tab)
+    {
+        try
+        {
+            var filePath = GetHtmlTabFilePath(tab);
+            var fileInfo = new FileInfo(filePath);
+            fileInfo.Delete();
+        }
+        catch {}
+    }
+
     public async Task<string?> GetHomeTabHtmlAsync(string? tab)
     {
         if (DisableCache)

MyApp.ServiceInterface/JobServices.cs

@@ -16,12 +16,24 @@ public object Any(ViewModelQueues request)
         };
     }
 
-    public object Any(GetNextJobs request)
+    public async Task<object> Any(GetNextJobs request)
     {
         var to = new GetNextJobsResponse();
         var job = workerQueues.Dequeue(request.Models, TimeSpan.FromSeconds(30));
         if (job == null)
-            return to;
+        {
+            var dbHasIncompleteJobs = await Db.SelectAsync(Db.From<PostJob>()
+                .Where(x => (x.CompletedDate == null || x.StartedDate < DateTime.UtcNow.AddMinutes(-5)) && x.Model != "rank"));
+            if (dbHasIncompleteJobs.Count > 0)
+            {
+                await Any(new RestoreModelQueues { RestoreFailedJobs = true });
+                job = workerQueues.Dequeue(request.Models, TimeSpan.FromSeconds(30));
+            }
+            if (job == null)
+            {
+                return to;
+            }
+        }
 
         MessageProducer.Publish(new DbWrites {
             StartJob = new() { Id = job.Id, Worker = request.Worker, WorkerIp = Request!.RemoteIp }
@@ -53,14 +65,17 @@ public async Task<object> Any(RestoreModelQueues request)
 
         foreach (var lostJob in lostJobs)
         {
+            if (pendingJobIds.Contains(lostJob.Id)) continue;
             workerQueues.Enqueue(lostJob);
         }
         foreach (var missingJob in missingJobs)
         {
+            if (pendingJobIds.Contains(missingJob.Id)) continue;
             workerQueues.Enqueue(missingJob);
         }
         foreach (var failedJob in failedJobs)
         {
+            if (pendingJobIds.Contains(failedJob.Id)) continue;
             workerQueues.Enqueue(failedJob);
         }
         var failedSuffix = restoreFailedJobs ? ", restored!" : "";

MyApp.ServiceInterface/SearchServices.cs

@@ -11,16 +11,17 @@ public class SearchServices(ILogger<SearchServices> log, QuestionsProvider quest
 {
     public async Task Any(SearchTasks request)
     {
-        if (request.AddPostId != null)
+        if (request.AddPostToIndex != null)
         {
-            var id = request.AddPostId.Value;
+            var id = request.AddPostToIndex.Value;
             var questionFiles = await questions.GetQuestionAsync(id);
 
             log.LogInformation("Adding Post '{PostId}' Question and {AnswerCount} to Search Index...", 
                 id,
                 questionFiles.GetAnswerFiles().Count());
             using var db = HostContext.AppHost.GetDbConnection(Databases.Search);
             var nextId = await db.ScalarAsync<int>("SELECT MAX(rowid) FROM PostFts");
+            nextId += 1;
             var existingIds = new HashSet<int>();
             var minDate = new DateTime(2008,08,1);
 
@@ -35,10 +36,13 @@ public async Task Any(SearchTasks request)
                     if (fileType == "json")
                     {
                         var post = await ToPostAsync(file);
+                        if (post.Id == default)
+                            throw new ArgumentNullException(nameof(post.Id));
                         if (!existingIds.Add(post.Id)) continue;
                         log.LogDebug("Adding Question {FilePath}", file.VirtualPath);
                         var modifiedDate = post.LastEditDate ?? (post.CreationDate > minDate ? post.CreationDate : minDate);
-                        db.ExecuteNonQuery($@"INSERT INTO {nameof(PostFts)} (
+                        await db.ExecuteNonQueryAsync($"DELETE FROM {nameof(PostFts)} WHERE rowid = {post.Id}");
+                        await db.ExecuteNonQueryAsync($@"INSERT INTO {nameof(PostFts)} (
                             rowid,
                             {nameof(PostFts.RefId)},
                             {nameof(PostFts.UserName)},
@@ -61,16 +65,18 @@ public async Task Any(SearchTasks request)
                         var userName = fileType.Substring(2); 
                         log.LogDebug("Adding Human Answer {FilePath}", file.VirtualPath);
                         var modifiedDate = post.LastEditDate ?? (post.CreationDate > minDate ? post.CreationDate : minDate);
-                        db.ExecuteNonQuery($@"INSERT INTO {nameof(PostFts)} (
+                        var refId = $"{id}-{userName}";
+                        await db.ExecuteNonQueryAsync($"DELETE FROM {nameof(PostFts)} where {nameof(PostFts.RefId)} = {QuotedValue(refId)}");
+                        await db.ExecuteNonQueryAsync($@"INSERT INTO {nameof(PostFts)} (
                             rowid,
                             {nameof(PostFts.RefId)},
                             {nameof(PostFts.UserName)},
                             {nameof(PostFts.Body)},
                             {nameof(PostFts.ModifiedDate)}
                         ) VALUES (
                             {post.Id},
-                            '{id}-{userName}',
-                            '{userName}',
+                            {QuotedValue(refId)},
+                            {QuotedValue(userName)},
                             {QuotedValue(post.Body)},
                             {QuotedValue(modifiedDate.ToString("yyyy-MM-dd HH:mm:ss"))}
                         )");
@@ -88,6 +94,8 @@ public async Task Any(SearchTasks request)
                             ? DateTimeOffset.FromUnixTimeSeconds(created).DateTime
                             : file.LastModified;
                         log.LogDebug("Adding Model Answer {FilePath} {UserName}", file.VirtualPath, userName);
+                        var refId = $"{id}-{userName}";
+                        await db.ExecuteNonQueryAsync($"DELETE FROM {nameof(PostFts)} where {nameof(PostFts.RefId)} = {QuotedValue(refId)}");
                         db.ExecuteNonQuery($@"INSERT INTO {nameof(PostFts)} (
                             rowid,
                             {nameof(PostFts.RefId)},
@@ -96,8 +104,8 @@ public async Task Any(SearchTasks request)
                             {nameof(PostFts.ModifiedDate)}
                         ) VALUES (
                             {nextId++},
-                            '{id}-{userName}',
-                            '{userName}',
+                            {QuotedValue(refId)},
+                            {QuotedValue(userName)},
                             {QuotedValue(body)},
                             {QuotedValue(modifiedDate.ToString("yyyy-MM-dd HH:mm:ss"))}
                         )");
@@ -121,5 +129,4 @@ async Task<Post> ToPostAsync(IVirtualFile file)
         var post = json.FromJson<Post>();
         return post;
     }
-
-}
+}

MyApp.ServiceModel/Stats.cs

@@ -131,5 +131,5 @@ public class DbWrites
 [Restrict(InternalOnly = true)]
 public class SearchTasks
 {
-    public int? AddPostId { get; set; }
+    public int? AddPostToIndex { get; set; }
 }

MyApp/Components/Pages/Home.razor

@@ -2,6 +2,7 @@
 @inject RendererCache RendererCache
 @inject IMessageProducer MessageProducer
 @inject IDbConnectionFactory DbFactory
+@inject IWebHostEnvironment Env
 
 <PageTitle>pvq.app</PageTitle>
 
@@ -20,18 +21,31 @@
 </div>
 
 @code {
+    [CascadingParameter]
+    public HttpContext? HttpContext { get; set; }
+
     string? Html;
 
     [SupplyParameterFromQuery] 
     string? Tab { get; set; }
 
     List<Post> Posts { get; set; } = [];
 
+    [SupplyParameterFromQuery] int? Force { get; set; }
+
     protected override async Task OnInitializedAsync()
     {
-        Html = await RendererCache.GetHomeTabHtmlAsync(Tab);
-        if (Html != null)
-            return;
+        var noCache = Force != null && (Env.IsDevelopment() || HttpContext?.User.IsAdminOrModerator() == true);
+        if (noCache)
+        {
+            RendererCache.DeleteHomeTabHtml(Tab);
+        }
+        else
+        {
+            Html = await RendererCache.GetHomeTabHtmlAsync(Tab);
+            if (Html != null)
+                return;
+        }
 
         using var db = await DbFactory.OpenAsync();
         var q = db.From<Post>();

MyApp/Components/Pages/Posts/Post.razor

@@ -229,6 +229,4 @@ else
     }
 
     protected override void OnInitialized() => load();
-
-    protected override void OnParametersSet() => load();
 }

MyApp/Components/Pages/Questions/Index.razor

@@ -122,57 +122,16 @@
                     q.Where("Body match {0}", searchPhrase);
                 }
 
-                if (Tab == "newest")
-                {
-                    q.OrderByDescending("ModifiedDate");
-                }
-                else if (Tab == "oldest")
-                {
-                    q.OrderBy("ModifiedDate");
-                }
-                else
-                {
-                    q.OrderBy("Rank");
-                }
+                q.OrderByView(Tab);
 
                 List<PostFts> postsFts = await dbSearch.SelectAsync(q
                     .Select("RefId, substring(Body,0,400) as Body, ModifiedDate")
                     .Skip(skip)
                     .Take(take));
+                total = dbSearch.Count(q);
 
-                posts = postsFts.Select(x => new Post
-                {
-                    Id = x.RefId.LeftPart('-').ToInt(),
-                    PostTypeId = x.RefId.Contains('-') ? 2 : 1,
-                    Summary = x.Body.StripHtml().SubstringWithEllipsis(0,200),
-                    CreationDate = x.ModifiedDate,
-                    LastEditDate = x.ModifiedDate,
-                }).ToList();
-
-                var postIds = posts.Select(x => x.Id).ToSet();
-
                 using var db = await DbFactory.OpenAsync();
-                var fullPosts = await db.SelectAsync(db.From<Post>().Where(x => postIds.Contains(x.Id)));
-                var fullPostsMap = fullPosts.ToDictionary(x => x.Id);
-
-                foreach (var post in posts)
-                {
-                    if (fullPostsMap.TryGetValue(post.Id, out var fullPost))
-                    {
-                        post.Title = fullPost.Title;
-                        post.Slug = fullPost.Slug;
-                        if (post.PostTypeId == 1)
-                        {
-                            post.Tags = fullPost.Tags;
-                            post.Score = fullPost.Score;
-                            post.ViewCount = fullPost.ViewCount;
-                            post.CreationDate = fullPost.CreationDate;
-                            post.LastEditDate = fullPost.LastEditDate;
-                        }
-                    }
-                }
-
-                total = dbSearch.Count(q);
+                posts = await db.PopulatePostsAsync(postsFts);
             }
             else
             {