Bump Robots Exclusion Tools to 0.9.1, Sitemap Tools to 0.7.2 (#147)

TurnerSoftware · Aug 10, 2022 · f6f16b3 · f6f16b3
1 parent 9c570c1
commit f6f16b3
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 5 deletions.
diff --git a/src/InfinityCrawler/Crawler.cs b/src/InfinityCrawler/Crawler.cs
@@ -80,10 +80,15 @@ public async Task<CrawlResult> Crawl(Uri siteUri, CrawlSettings settings)
 
 		private void UpdateCrawlDelay(RobotsFile robotsFile, string userAgent, RequestProcessorOptions requestProcessorOptions)
 		{
+			var minimumCrawlDelayInMilliseconds = 0;
+
 			//Apply Robots.txt crawl-delay (if defined)
-			var userAgentEntry = robotsFile.GetEntryForUserAgent(userAgent);
-			var minimumCrawlDelay = userAgentEntry?.CrawlDelay ?? 0;
-			var taskDelay = Math.Max(minimumCrawlDelay * 1000, requestProcessorOptions.DelayBetweenRequestStart.TotalMilliseconds);
+			if (robotsFile.TryGetEntryForUserAgent(userAgent, out var accessEntry))
+			{
+				minimumCrawlDelayInMilliseconds = accessEntry.CrawlDelay ?? 0 * 1000;
+			}
+
+			var taskDelay = Math.Max(minimumCrawlDelayInMilliseconds, requestProcessorOptions.DelayBetweenRequestStart.TotalMilliseconds);
 			requestProcessorOptions.DelayBetweenRequestStart = new TimeSpan(0, 0, 0, 0, (int)taskDelay);
 		}
 	}

diff --git a/src/InfinityCrawler/InfinityCrawler.csproj b/src/InfinityCrawler/InfinityCrawler.csproj
@@ -10,8 +10,8 @@
 
 	<ItemGroup>
 		<PackageReference Include="HtmlAgilityPack" Version="1.11.43" />
-		<PackageReference Include="TurnerSoftware.RobotsExclusionTools" Version="0.8.1" />
-		<PackageReference Include="TurnerSoftware.SitemapTools" Version="0.7.1" />
+		<PackageReference Include="TurnerSoftware.RobotsExclusionTools" Version="0.9.1" />
+		<PackageReference Include="TurnerSoftware.SitemapTools" Version="0.7.2" />
 	</ItemGroup>
 
 	<ItemGroup Condition="$(TargetFramework) == 'netstandard2.0'">