Merge pull request #35 from ernado-x/33-bug-nullref-in-savetodirectory

X.Web.Sitemap.sln

@@ -11,7 +11,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "X.Web.Sitemap.Tests", "test
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "X.Web.Sitemap", "src\X.Web.Sitemap\X.Web.Sitemap.csproj", "{704FA5E2-2694-44C9-826E-85C2CEC96D5D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "X.Web.Sitemap.Examples", "src\X.Web.Sitemap.Examples\X.Web.Sitemap.Examples.csproj", "{EA29E3A8-D073-4517-BE60-B39AA3D089AF}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "X.Web.Sitemap.Example", "src\X.Web.Sitemap.Example\X.Web.Sitemap.Example.csproj", "{97B9B296-63C0-4816-AD53-E069E6BDEF66}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -27,17 +27,17 @@ Global
 		{704FA5E2-2694-44C9-826E-85C2CEC96D5D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{704FA5E2-2694-44C9-826E-85C2CEC96D5D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{704FA5E2-2694-44C9-826E-85C2CEC96D5D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EA29E3A8-D073-4517-BE60-B39AA3D089AF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EA29E3A8-D073-4517-BE60-B39AA3D089AF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EA29E3A8-D073-4517-BE60-B39AA3D089AF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EA29E3A8-D073-4517-BE60-B39AA3D089AF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{97B9B296-63C0-4816-AD53-E069E6BDEF66}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{97B9B296-63C0-4816-AD53-E069E6BDEF66}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{97B9B296-63C0-4816-AD53-E069E6BDEF66}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{97B9B296-63C0-4816-AD53-E069E6BDEF66}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{5AA327E0-C63F-4567-9C09-23707EB5E4C4} = {5662CFB2-6193-4FB8-BBA3-B5822FDB583F}
 		{704FA5E2-2694-44C9-826E-85C2CEC96D5D} = {DD3DEEE0-ABF3-4DFB-A5A9-14AA3FB1DBA2}
-		{EA29E3A8-D073-4517-BE60-B39AA3D089AF} = {DD3DEEE0-ABF3-4DFB-A5A9-14AA3FB1DBA2}
+		{97B9B296-63C0-4816-AD53-E069E6BDEF66} = {DD3DEEE0-ABF3-4DFB-A5A9-14AA3FB1DBA2}
 	EndGlobalSection
 EndGlobal

src/X.Web.Sitemap.Example/IExample.cs

@@ -0,0 +1,6 @@
+namespace X.Web.Sitemap.Example;
+
+public interface IExample
+{
+     void Run();
+}

src/X.Web.Sitemap.Example/Program.cs

@@ -0,0 +1,10 @@
+using X.Web.Sitemap.Example;
+
+Console.WriteLine("OK");
+
+IExample example1 = new SitemapGenerationWithSitemapIndexExample();
+example1.Run();
+
+
+IExample example2 = new SimpleSitemapGenerationExample();
+example2.Run();

src/X.Web.Sitemap.Example/SimpleSitemapGenerationExample.cs

@@ -0,0 +1,20 @@
+namespace X.Web.Sitemap.Example;
+
+public class SimpleSitemapGenerationExample : IExample
+{
+    public void Run()
+    {
+        // Pick a place where you would like to write the sitemap files in that folder will get overwritten by new ones
+        var directory = Path.Combine(Path.GetTempPath(), "XWebsiteExample");
+
+        var urlGenerator = new UrlGenerator();
+
+        // Get list of website urls
+        var allUrls = urlGenerator.GetUrls("mywebsite.com");
+
+        var sitemap = new Sitemap(allUrls);
+
+        sitemap.SaveToDirectory(directory);
+    }
+
+}

src/X.Web.Sitemap.Example/SitemapGenerationWithSitemapIndexExample.cs

@@ -0,0 +1,50 @@
+namespace X.Web.Sitemap.Example;
+
+/// <summary>
+/// This is an example showing how you might take a large list of URLs of different kinds of resources and build
+/// both a bunch of sitemaps (depending on how many URls you have) as well as a sitemap index file to go with it
+/// </summary>
+public class SitemapGenerationWithSitemapIndexExample : IExample
+{
+    public void Run()
+    { 
+        // Pick a place where you would like to write the sitemap files in that folder will get overwritten by new ones
+        var targetSitemapDirectory = Path.Combine(Path.GetTempPath(), "XWebsiteExample");
+
+        // Pick a place where sitemaps will be accessible from internet
+        var sitemapRootUrl = "https://www.mywebsite.com/sitemaps/";
+
+        var sitemapGenerator = new SitemapGenerator();
+        var sitemapIndexGenerator = new SitemapIndexGenerator();
+        var urlGenerator = new UrlGenerator();
+
+        // Get list of website urls
+        var allUrls = urlGenerator.GetUrls("mywebsite.com"); 
+
+
+        // generate one or more sitemaps (depending on the number of URLs) in the designated location.
+        var fileInfoForGeneratedSitemaps = sitemapGenerator.GenerateSitemaps(allUrls, targetSitemapDirectory);
+
+        var sitemapInfos = new List<SitemapInfo>();
+        var dateSitemapWasUpdated = DateTime.UtcNow.Date;
+
+        foreach (var fileInfo in fileInfoForGeneratedSitemaps)
+        {
+            // It's up to you to figure out what the URI is to the sitemap you wrote to the file sytsem.
+            // In this case we are assuming that the directory above has files exposed
+            // via the /sitemaps/ subfolder of www.mywebsite.com
+
+            var uriToSitemap = new Uri($"{sitemapRootUrl}{fileInfo.Name}");
+
+            sitemapInfos.Add(new SitemapInfo(uriToSitemap, dateSitemapWasUpdated));
+        }
+
+        // Now generate the sitemap index file which has a reference to all of the sitemaps that were generated. 
+        sitemapIndexGenerator.GenerateSitemapIndex(sitemapInfos, targetSitemapDirectory, "sitemap-index.xml");
+
+        // After this runs you'll want to make sure your robots.txt has a reference to the sitemap index (at the bottom of robots.txt) like this: 
+        // "Sitemap: https://www.mywebsite.com/sitemaps/sitemap-index.xml"
+        // You could do this manually (since this may never change) or if you are ultra-fancy, you could dynamically update your robots.txt with the names of the sitemap index
+        // file(s) you generated
+    }
+}

src/X.Web.Sitemap.Example/UrlGenerator.cs

@@ -0,0 +1,67 @@
+namespace X.Web.Sitemap.Example;
+
+public class UrlGenerator
+{
+    public List<Url> GetUrls(string domain)
+    {
+        var productPageUrlStrings = GetHighPriorityProductPageUrls(domain);
+
+        //--build a list of X.Web.Sitemap.Url objects and determine what is the appropriate ChangeFrequency, TimeStamp (aka "LastMod" or date that the resource last had changes),
+        //  and the a priority for the page. If you can build in some logic to prioritize your pages then you are more sophisticated than most! :)
+        var allUrls = productPageUrlStrings.Select(url => new Url
+        {
+            //--assign the location of the HTTP request -- e.g.: https://www.somesite.com/some-resource
+            Location = url,
+            //--let's instruct crawlers to crawl these pages monthly since the content doesn't change that much
+            ChangeFrequency = ChangeFrequency.Monthly,
+            //--in this case we don't know when the page was last modified so we wouldn't really set this. Only assigning here to demonstrate that the property exists.
+            //  if your system is smart enough to know when a page was last modified then that is the best case scenario
+            TimeStamp = DateTime.UtcNow,
+            //--set this to between 0 and 1. This should only be used as a relative ranking of other pages in your site so that search engines know which result to prioritize
+            //  in SERPS if multiple pages look pertinent from your site. Since product pages are really important to us, we'll make them a .9
+            Priority = .9
+        }).ToList();
+
+        var miscellaneousLowPriorityUrlStrings = GetMiscellaneousLowPriorityUrls(domain);
+
+        var miscellaneousLowPriorityUrls = miscellaneousLowPriorityUrlStrings.Select(url => new Url
+        {
+            Location = url,
+            //--let's instruct crawlers to crawl these pages yearly since the content almost never changes
+            ChangeFrequency = ChangeFrequency.Yearly,
+            //--let's pretend this content was changed a year ago
+            TimeStamp = DateTime.UtcNow.AddYears(-1),
+            //--these pages are super low priority
+            Priority = .1
+        }).ToList();
+
+        //--combine the urls into one big list. These could of course bet kept seperate and two different sitemap index files could be generated if we wanted
+        allUrls.AddRange(miscellaneousLowPriorityUrls);
+
+        return allUrls;
+    }
+
+    private IReadOnlyCollection<string> GetMiscellaneousLowPriorityUrls(string domain)
+    {
+        var result = new List<string>();
+
+        for (int i = 0; i < 40000; i++)
+        {
+            result.Add($"https://{domain}/page/{i}.html");
+        }
+
+        return result;
+    }
+
+    private IReadOnlyCollection<string> GetHighPriorityProductPageUrls(string domain)
+    {
+        var result = new List<string>();
+
+        for (int i = 0; i < 10000; i++)
+        {
+            result.Add($"https://{domain}/priority-page/{i}.html");
+        }
+
+        return result;
+    }
+}

src/X.Web.Sitemap.Example/X.Web.Sitemap.Example.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\X.Web.Sitemap\X.Web.Sitemap.csproj" />
+    </ItemGroup>
+
+</Project>

src/X.Web.Sitemap/FileSystemWrapper.cs

@@ -1,5 +1,4 @@
-using System;
-using System.IO;
+using System.IO;
 using System.Threading.Tasks;
 
 namespace X.Web.Sitemap;

src/X.Web.Sitemap/ISitemap.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using JetBrains.Annotations;
 
@@ -11,7 +12,8 @@ public interface ISitemap : IList<Url>
 
     Task<bool> SaveAsync(string path);
 
-    bool SaveToDirectory(string directory);
+    [Obsolete("This method will be removed in future version. Use SitemapGenerator instead")]
+    bool SaveToDirectory(string targetSitemapDirectory);
 
     string ToXml();
 }

src/X.Web.Sitemap/ISitemapGenerator.cs

@@ -27,7 +27,31 @@ public interface ISitemapGenerator
     /// files with names like products-001.xml, products-002.xml, etc.
     /// </param>
     List<FileInfo> GenerateSitemaps(
-        List<Url> urls, 
-        DirectoryInfo targetDirectory, 
+        IEnumerable<Url> urls,
+        DirectoryInfo targetDirectory,
+        string sitemapBaseFileNameWithoutExtension = "sitemap");
+
+    /// <summary>
+    /// Creates one or more sitemaps based on the number of Urls passed in. As of 2016, the maximum number of
+    /// urls per sitemap is 50,000 and the maximum file size is 50MB. See https://www.sitemaps.org/protocol.html
+    /// for current standards. Filenames will be sitemap-001.xml, sitemap-002.xml, etc.
+    /// Returns a list of FileInfo objects for each sitemap that was created (e.g. for subsequent use in generating
+    /// a sitemap index file)
+    /// </summary>
+    /// <param name="urls">
+    /// Urls to include in the sitemap(s). If the number of Urls exceeds 50,000 or the file size exceeds 50MB,
+    /// then multiple files
+    /// will be generated and multiple SitemapInfo objects will be returned.
+    /// </param>
+    /// <param name="targetDirectory">
+    /// The directory where the sitemap(s) will be saved.
+    /// </param>
+    /// <param name="sitemapBaseFileNameWithoutExtension">
+    /// The base file name of the sitemap. For example, if you pick 'products' then it will generate
+    /// files with names like products-001.xml, products-002.xml, etc.
+    /// </param>
+    List<FileInfo> GenerateSitemaps(
+        IEnumerable<Url> urls,
+        string targetDirectory,
         string sitemapBaseFileNameWithoutExtension = "sitemap");
 }