Skip to content

Commit

Permalink
Added environment filter and a small sample site
Browse files Browse the repository at this point in the history
  • Loading branch information
karl-sjogren committed Oct 1, 2023
1 parent 87fdc27 commit 99aa5af
Show file tree
Hide file tree
Showing 85 changed files with 74,653 additions and 168 deletions.
28 changes: 28 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,34 @@ User-agent: *
Disallow: /
```

Since version 3.0 you can call AddStaticRobotsTxt multiple times and specify different
settings for different environments. If there is a matching envrionment it will be used,
otherwise it will fall back to any configuration without an environment specified.

```csharp
public void ConfigureServices(IServiceCollection services) {
services.AddStaticRobotsTxt(builder =>
builder
.ForEnvironment("Production")
.AddSection(section =>
section
.AddComment("Allow Googlebot")
.AddUserAgent("Googlebot")
.Allow("/")
)
);

services.AddStaticRobotsTxt(builder =>
builder
.DenyAll()
);
}

public void Configure(IApplicationBuilder app) {
app.UseRobotsTxt();
}
```

## IRobotsTxtProvider

`IRobotsTxtProvider` allows for dynamicly configuring the Robots.txt output depending
Expand Down
19 changes: 18 additions & 1 deletion RobotsTxt.sln
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,14 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "benchmark", "benchmark", "{
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobotsTxt.Benchmarks", "benchmark\RobotsTxt.Benchmarks\RobotsTxt.Benchmarks.csproj", "{9B93B6F8-B310-4BC2-85BD-8F67818E958F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobotsTxt.Samples", "samples\RobotsTxt.Samples.csproj", "{921048AE-506E-4738-A671-3FBCCD0394F9}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobotsTxt.Samples", "samples\sample-provider\RobotsTxt.Samples.csproj", "{921048AE-506E-4738-A671-3FBCCD0394F9}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobotsTxt.Samples.Tests", "tests\RobotsTxt.Samples.Tests\RobotsTxt.Samples.Tests.csproj", "{1F85C7B2-CE49-4865-AD67-B62A262F4769}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{8A69AE72-33A2-48A5-B9FA-A88464680942}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "sample-site", "samples\sample-site\sample-site.csproj", "{9526C44F-94AC-47D8-B014-DEC976CF607C}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -92,11 +96,24 @@ Global
{1F85C7B2-CE49-4865-AD67-B62A262F4769}.Release|x64.Build.0 = Release|Any CPU
{1F85C7B2-CE49-4865-AD67-B62A262F4769}.Release|x86.ActiveCfg = Release|Any CPU
{1F85C7B2-CE49-4865-AD67-B62A262F4769}.Release|x86.Build.0 = Release|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Debug|x64.ActiveCfg = Debug|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Debug|x64.Build.0 = Debug|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Debug|x86.ActiveCfg = Debug|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Debug|x86.Build.0 = Debug|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Release|Any CPU.Build.0 = Release|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Release|x64.ActiveCfg = Release|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Release|x64.Build.0 = Release|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Release|x86.ActiveCfg = Release|Any CPU
{9526C44F-94AC-47D8-B014-DEC976CF607C}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{610B25CA-70BD-4433-89D3-FEF56E583477} = {F0FB1E81-3C38-4B4F-819E-27D258FF2899}
{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE} = {962DBC53-02D4-46CD-842D-AFC95CEBA59B}
{9B93B6F8-B310-4BC2-85BD-8F67818E958F} = {F0B615D7-87D1-4E56-A52A-F5B5E5FBF157}
{1F85C7B2-CE49-4865-AD67-B62A262F4769} = {962DBC53-02D4-46CD-842D-AFC95CEBA59B}
{9526C44F-94AC-47D8-B014-DEC976CF607C} = {8A69AE72-33A2-48A5-B9FA-A88464680942}
EndGlobalSection
EndGlobal
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\src\RobotsTxt\RobotsTxt.csproj" />
<ProjectReference Include="..\..\src\RobotsTxt\RobotsTxt.csproj" />
</ItemGroup>

<ItemGroup>
Expand Down
26 changes: 26 additions & 0 deletions samples/sample-site/Controllers/HomeController.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
using System.Diagnostics;
using Microsoft.AspNetCore.Mvc;
using sample_site.Models;

namespace sample_site.Controllers;

public class HomeController : Controller {
private readonly ILogger<HomeController> _logger;

public HomeController(ILogger<HomeController> logger) {
_logger = logger;
}

public IActionResult Index() {
return View();
}

public IActionResult Privacy() {
return View();
}

[ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)]
public IActionResult Error() {
return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier });
}
}
7 changes: 7 additions & 0 deletions samples/sample-site/Models/ErrorViewModel.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace sample_site.Models;

public class ErrorViewModel {
public string? RequestId { get; set; }

public bool ShowRequestId => !string.IsNullOrEmpty(RequestId);
}
47 changes: 47 additions & 0 deletions samples/sample-site/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
using RobotsTxt;

var builder = WebApplication.CreateBuilder(args);

// Add services to the container.
builder.Services.AddControllersWithViews();

builder.Services.AddStaticRobotsTxt(builder =>
builder
.ForEnvironment("Production")
.AddSection(section =>
section
.AddComment("Allow Googlebot")
.AddUserAgent("Googlebot")
.Allow("/")
)
);

builder.Services.AddStaticRobotsTxt(builder =>
builder
.DenyAll()
);

var app = builder.Build();

// Configure the HTTP request pipeline.
if(!app.Environment.IsDevelopment()) {
app.UseExceptionHandler("/Home/Error");
// The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
app.UseHsts();
}

app.UseHttpsRedirection();

app.UseRobotsTxt();

app.UseStaticFiles();

app.UseRouting();

app.UseAuthorization();

app.MapControllerRoute(
name: "default",
pattern: "{controller=Home}/{action=Index}/{id?}");

app.Run();
8 changes: 8 additions & 0 deletions samples/sample-site/Views/Home/Index.cshtml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
@{
ViewData["Title"] = "Home Page";
}

<div class="text-center">
<h1 class="display-4">Welcome</h1>
<p>Learn about <a href="https://learn.microsoft.com/aspnet/core">building Web apps with ASP.NET Core</a>.</p>
</div>
6 changes: 6 additions & 0 deletions samples/sample-site/Views/Home/Privacy.cshtml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
@{
ViewData["Title"] = "Privacy Policy";
}
<h1>@ViewData["Title"]</h1>

<p>Use this page to detail your site's privacy policy.</p>
25 changes: 25 additions & 0 deletions samples/sample-site/Views/Shared/Error.cshtml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
@model ErrorViewModel
@{
ViewData["Title"] = "Error";
}

<h1 class="text-danger">Error.</h1>
<h2 class="text-danger">An error occurred while processing your request.</h2>

@if (Model.ShowRequestId)
{
<p>
<strong>Request ID:</strong> <code>@Model.RequestId</code>
</p>
}

<h3>Development Mode</h3>
<p>
Swapping to <strong>Development</strong> environment will display more detailed information about the error that occurred.
</p>
<p>
<strong>The Development environment shouldn't be enabled for deployed applications.</strong>
It can result in displaying sensitive information from exceptions to end users.
For local debugging, enable the <strong>Development</strong> environment by setting the <strong>ASPNETCORE_ENVIRONMENT</strong> environment variable to <strong>Development</strong>
and restarting the app.
</p>
49 changes: 49 additions & 0 deletions samples/sample-site/Views/Shared/_Layout.cshtml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>@ViewData["Title"] - sample_site</title>
<link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
<link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
<link rel="stylesheet" href="~/sample_site.styles.css" asp-append-version="true" />
</head>
<body>
<header>
<nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light bg-white border-bottom box-shadow mb-3">
<div class="container-fluid">
<a class="navbar-brand" asp-area="" asp-controller="Home" asp-action="Index">sample_site</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target=".navbar-collapse" aria-controls="navbarSupportedContent"
aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="navbar-collapse collapse d-sm-inline-flex justify-content-between">
<ul class="navbar-nav flex-grow-1">
<li class="nav-item">
<a class="nav-link text-dark" asp-area="" asp-controller="Home" asp-action="Index">Home</a>
</li>
<li class="nav-item">
<a class="nav-link text-dark" asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>
</li>
</ul>
</div>
</div>
</nav>
</header>
<div class="container">
<main role="main" class="pb-3">
@RenderBody()
</main>
</div>

<footer class="border-top footer text-muted">
<div class="container">
&copy; 2023 - sample_site - <a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>
</div>
</footer>
<script src="~/lib/jquery/dist/jquery.min.js"></script>
<script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
<script src="~/js/site.js" asp-append-version="true"></script>
@await RenderSectionAsync("Scripts", required: false)
</body>
</html>
48 changes: 48 additions & 0 deletions samples/sample-site/Views/Shared/_Layout.cshtml.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/* Please see documentation at https://learn.microsoft.com/aspnet/core/client-side/bundling-and-minification
for details on configuring this project to bundle and minify static web assets. */

a.navbar-brand {
white-space: normal;
text-align: center;
word-break: break-all;
}

a {
color: #0077cc;
}

.btn-primary {
color: #fff;
background-color: #1b6ec2;
border-color: #1861ac;
}

.nav-pills .nav-link.active, .nav-pills .show > .nav-link {
color: #fff;
background-color: #1b6ec2;
border-color: #1861ac;
}

.border-top {
border-top: 1px solid #e5e5e5;
}
.border-bottom {
border-bottom: 1px solid #e5e5e5;
}

.box-shadow {
box-shadow: 0 .25rem .75rem rgba(0, 0, 0, .05);
}

button.accept-policy {
font-size: 1rem;
line-height: inherit;
}

.footer {
position: absolute;
bottom: 0;
width: 100%;
white-space: nowrap;
line-height: 60px;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
<script src="~/lib/jquery-validation/dist/jquery.validate.min.js"></script>
<script src="~/lib/jquery-validation-unobtrusive/jquery.validate.unobtrusive.min.js"></script>
3 changes: 3 additions & 0 deletions samples/sample-site/Views/_ViewImports.cshtml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
@using sample_site
@using sample_site.Models
@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers
3 changes: 3 additions & 0 deletions samples/sample-site/Views/_ViewStart.cshtml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
@{
Layout = "_Layout";
}
8 changes: 8 additions & 0 deletions samples/sample-site/appsettings.Development.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
}
}
9 changes: 9 additions & 0 deletions samples/sample-site/appsettings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"AllowedHosts": "*"
}
14 changes: 14 additions & 0 deletions samples/sample-site/sample-site.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk.Web">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RootNamespace>sample_site</RootNamespace>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\RobotsTxt\RobotsTxt.csproj" />
</ItemGroup>

</Project>
22 changes: 22 additions & 0 deletions samples/sample-site/wwwroot/css/site.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
html {
font-size: 14px;
}

@media (min-width: 768px) {
html {
font-size: 16px;
}
}

.btn:focus, .btn:active:focus, .btn-link.nav-link:focus, .form-control:focus, .form-check-input:focus {
box-shadow: 0 0 0 0.1rem white, 0 0 0 0.25rem #258cfb;
}

html {
position: relative;
min-height: 100%;
}

body {
margin-bottom: 60px;
}
Binary file added samples/sample-site/wwwroot/favicon.ico
Binary file not shown.
4 changes: 4 additions & 0 deletions samples/sample-site/wwwroot/js/site.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// Please see documentation at https://learn.microsoft.com/aspnet/core/client-side/bundling-and-minification
// for details on configuring this project to bundle and minify static web assets.

// Write your JavaScript code.
Loading

0 comments on commit 99aa5af

Please sign in to comment.