#19 Added basic support for filtering on different domains

.github/workflows/dependabot-build.yml

@@ -22,10 +22,10 @@ jobs:
     - name: Checkout sources
       uses: actions/checkout@v4
 
-    - name: Install .NET 7
+    - name: Install .NET 8.0
       uses: actions/setup-dotnet@v3
       with:
-        dotnet-version: 7.0.x
+        dotnet-version: 8.0
         dotnet-quality: ga
 
     - name: Restore NuGet packages
@@ -43,10 +43,10 @@ jobs:
     - name: Checkout sources
       uses: actions/checkout@v4
 
-    - name: Install .NET 7
+    - name: Install .NET 8.0
       uses: actions/setup-dotnet@v3
       with:
-        dotnet-version: 7.0.x
+        dotnet-version: 8.0
         dotnet-quality: ga
 
     - name: Restore NuGet packages

Directory.Build.props

@@ -1,6 +1,6 @@
 <Project>
   <PropertyGroup>
-    <VersionPrefix>3.0.0</VersionPrefix>
+    <VersionPrefix>3.1.0</VersionPrefix>
     <LangVersion>latest</LangVersion>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <Deterministic>true</Deterministic>
@@ -12,6 +12,7 @@
       Dependabot updates. https://github.com/dependabot/dependabot-core/issues/1303
     -->
     <RestorePackagesWithLockFile>false</RestorePackagesWithLockFile>
+    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <ItemGroup>
@@ -27,14 +28,6 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.7.0">
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-      <PrivateAssets>all</PrivateAssets>
-    </PackageReference>
-    <PackageReference Include="Microsoft.CodeAnalysis.CSharp.CodeStyle" Version="4.7.0">
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-      <PrivateAssets>all</PrivateAssets>
-    </PackageReference>
     <PackageReference Include="Karls.Analyzers" Version="0.3.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>

benchmark/RobotsTxt.Benchmarks/Benchmarks.cs

@@ -1,6 +1,4 @@
-using System;
-using System.Reflection;
-using System.Threading.Tasks;
+using System.Reflection;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Diagnosers;

benchmark/RobotsTxt.Benchmarks/SimpleHttpContext.cs

@@ -1,9 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
 using System.Security.Claims;
-using System.Threading;
-using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
 

samples/sample-provider/CoolRobotsTxtProvider.cs

@@ -1,7 +1,4 @@
-using System;
 using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
 
 namespace RobotsTxt.Samples;
 

src/RobotsTxt/IRobotsTxtProvider.cs

@@ -1,6 +1,3 @@
-using System.Threading;
-using System.Threading.Tasks;
-
 namespace RobotsTxt;
 
 public interface IRobotsTxtProvider {

src/RobotsTxt/IServiceCollectionExtensions.cs

@@ -1,6 +1,7 @@
-using System;
+using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
+using Microsoft.Extensions.Hosting;
 
 namespace RobotsTxt;
 
@@ -11,6 +12,45 @@ public static void AddStaticRobotsTxt(this IServiceCollection services, Func<Rob
 
         services.AddSingleton(options);
 
-        services.TryAddSingleton<IRobotsTxtProvider, StaticRobotsTxtProvider>();
+        var previousRegistration = services.FirstOrDefault(s => s.ServiceType == typeof(IRobotsTxtProvider));
+        if(!options.Hostnames.Any()) {
+            if(previousRegistration?.Lifetime != ServiceLifetime.Scoped) {
+                services.TryAddSingleton<IRobotsTxtProvider, StaticRobotsTxtProvider>();
+            }
+        } else {
+            if(previousRegistration?.Lifetime != ServiceLifetime.Scoped) {
+                services.Remove(previousRegistration);
+            }
+
+            services.TryAddScoped(CreateRobotsTxtProviderForMultipleHosts);
+        }
+    }
+
+    private static readonly RobotsTxtOptions _denyAllOptions = new RobotsTxtOptionsBuilder().DenyAll().Build();
+
+    internal static IRobotsTxtProvider CreateRobotsTxtProviderForMultipleHosts(IServiceProvider services) {
+        var httpContextAccessor = services.GetRequiredService<IHttpContextAccessor>();
+        var httpContext = httpContextAccessor.HttpContext;
+
+        if(httpContext == null) {
+            throw new InvalidOperationException("Could not get HttpContext from IHttpContextAccessor.");
+        }
+
+        var host = httpContext.Request.Host.Value;
+
+        var options = services.GetRequiredService<IEnumerable<RobotsTxtOptions>>().ToArray();
+        var robotsOptions = options
+            .Where(option => option.Hostnames.Any(hostname => hostname.Equals(host, StringComparison.OrdinalIgnoreCase)));
+
+        if(!robotsOptions.Any()) {
+            robotsOptions = options.Where(option => !option.Hostnames.Any());
+
+            if(!robotsOptions.Any()) {
+                robotsOptions = new[] { _denyAllOptions };
+            }
+        }
+
+        var hostEnvironemnt = services.GetRequiredService<IHostEnvironment>();
+        return new StaticRobotsTxtProvider(robotsOptions, hostEnvironemnt);
     }
 }

src/RobotsTxt/RobotsTxtMiddleware.cs

@@ -1,5 +1,4 @@
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http;
 
 namespace RobotsTxt;
 

src/RobotsTxt/RobotsTxtOptions.cs

@@ -1,5 +1,3 @@
-using System;
-using System.Collections.Generic;
 using System.Globalization;
 using System.Text;
 
@@ -9,9 +7,11 @@ public class RobotsTxtOptions {
     public RobotsTxtOptions() {
         Sections = new List<RobotsTxtSection>();
         SitemapUrls = new List<string>();
+        Hostnames = new List<string>();
     }
 
     public string Environment { get; internal set; }
+    public List<string> Hostnames { get; internal set; }
     public List<RobotsTxtSection> Sections { get; }
     public List<string> SitemapUrls { get; }
     public TimeSpan MaxAge { get; internal set; } = TimeSpan.FromDays(1);

src/RobotsTxt/RobotsTxtOptionsBuilder.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace RobotsTxt;
 
 public class RobotsTxtOptionsBuilder {
@@ -17,6 +15,11 @@ public RobotsTxtOptionsBuilder ForEnvironment(string environment) {
         return this;
     }
 
+    public RobotsTxtOptionsBuilder ForHostnames(params string[] hostnames) {
+        _options.Hostnames = hostnames.ToList();
+        return this;
+    }
+
     public RobotsTxtOptionsBuilder WithMageAge(TimeSpan maxAge) {
         _options.MaxAge = maxAge;
         return this;

src/RobotsTxt/RobotsTxtResult.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace RobotsTxt;
 
 public sealed class RobotsTxtResult {

src/RobotsTxt/StaticRobotsTxtProvider.cs

@@ -1,9 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
 using Microsoft.Extensions.Hosting;
 
 namespace RobotsTxt;

tests/Directory.Build.props

@@ -2,9 +2,8 @@
   <PropertyGroup>
     <ParentDirectoryBuildPropsPath>$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)..\'))</ParentDirectoryBuildPropsPath>
     <TargetFrameworks>net6.0;net7.0;net8.0</TargetFrameworks>
-	<IsPublishable>false</IsPublishable>
-	<IsTestProject>true</IsTestProject>
-    <ImplicitUsings>enable</ImplicitUsings>
+    <IsPublishable>false</IsPublishable>
+    <IsTestProject>true</IsTestProject>
     <Nullable>enable</Nullable>
     <LangVersion>preview</LangVersion>
   </PropertyGroup>

tests/RobotsTxt.Tests/ServiceCollectionExtensionTests.cs

@@ -0,0 +1,100 @@
+using System.Text;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+
+namespace RobotsTxt.Tests;
+
+public class ServiceCollectionExtensionTests {
+    [Fact]
+    public void AddStaticRobotsTxt_WhenRegisteringWithoutHostname_RegistersProviderAsSingleton() {
+        var services = new ServiceCollection();
+
+        services.AddStaticRobotsTxt(builder =>
+            builder
+                .DenyAll()
+        );
+
+        var serviceRegistration = services.FirstOrDefault(s => s.ServiceType == typeof(IRobotsTxtProvider));
+
+        serviceRegistration.ShouldNotBeNull();
+        serviceRegistration.Lifetime.ShouldBe(ServiceLifetime.Singleton);
+    }
+
+    [Fact]
+    public void AddStaticRobotsTxt_WhenRegisteringMultipleOptionsWithDifferentHostNameOptions_RegistersProviderAsScoped() {
+        var services = new ServiceCollection();
+
+        services.AddStaticRobotsTxt(builder =>
+            builder
+                .DenyAll()
+        );
+
+        services.AddStaticRobotsTxt(builder =>
+            builder
+                .ForHostnames("example.com")
+                .DenyAll()
+        );
+
+        services.AddStaticRobotsTxt(builder =>
+            builder
+                .DenyAll()
+        );
+
+        var serviceRegistration = services.FirstOrDefault(s => s.ServiceType == typeof(IRobotsTxtProvider));
+
+        serviceRegistration.ShouldNotBeNull();
+        serviceRegistration.Lifetime.ShouldBe(ServiceLifetime.Scoped);
+    }
+
+    [Theory]
+    [InlineData("example.com", "https://example.com/sitemap.xml")]
+    [InlineData("www.example.com", "https://example.com/sitemap.xml")]
+    [InlineData("example-com-test-site.com", "https://example.com/sitemap.xml")]
+    [InlineData("cool-horses-with-glasses.com", "https://cool-horses-with-glasses.com/sitemap.xml")]
+    [InlineData("squirrels-with-hats.com", "https://sample-website.com/sitemap.xml")]
+
+    public async Task CreateRobotsTxtProviderForMultipleHosts_WhenCalledWithDifferentHostnames_ReturnsExpectedResultAsync(string hostname, string sitemapUrl) {
+        var services = new ServiceCollection();
+
+        services.AddStaticRobotsTxt(builder =>
+            builder
+                .ForHostnames("example.com", "www.example.com", "example-com-test-site.com")
+                .AddSitemap("https://example.com/sitemap.xml")
+                .DenyAll()
+        );
+
+        services.AddStaticRobotsTxt(builder =>
+            builder
+                .ForHostnames("cool-horses-with-glasses.com")
+                .AddSitemap("https://cool-horses-with-glasses.com/sitemap.xml")
+                .DenyAll()
+        );
+
+        services.AddStaticRobotsTxt(builder =>
+            builder
+                .AddSitemap("https://sample-website.com/sitemap.xml")
+                .DenyAll()
+        );
+
+        var hostEnvironemnt = A.Dummy<IHostEnvironment>();
+        services.AddSingleton(hostEnvironemnt);
+
+        var httpContext = new DefaultHttpContext();
+        httpContext.Request.Host = new HostString(hostname);
+
+        services.AddSingleton<IHttpContextAccessor>(new HttpContextAccessor { HttpContext = httpContext });
+
+        var serviceProvider = services.BuildServiceProvider();
+
+        var provider = IServiceCollectionExtensions.CreateRobotsTxtProviderForMultipleHosts(serviceProvider);
+
+        var sitemap = await provider.GetResultAsync(CancellationToken.None);
+
+        sitemap.ShouldNotBeNull();
+
+        var sitemapContent = Encoding.UTF8.GetString(sitemap.Content.Span);
+
+        sitemapContent.ShouldContain(sitemapUrl);
+    }
+}