Create RestClient once rather than on every request

Author: ed-graham

LinkCrawler/LinkCrawler.Tests/LinkCrawlerTests.cs

@@ -49,7 +49,7 @@ public void CrawlForLinksInResponse_ResponseModelWithMarkup_ValidUrlFoundInMarku
             var mockResponseModel = new Mock<IResponseModel>();
             mockResponseModel.Setup(x => x.Markup).Returns(markup);
 
-            LinkCrawler.CrawlForLinksInResponse(mockResponseModel.Object);
+            LinkCrawler.CrawlLinksInResponse(mockResponseModel.Object);
             Assert.That(LinkCrawler.UrlList.Where(l=>l.Address == url).Count() > 0);
         }
     }

LinkCrawler/LinkCrawler/LinkCrawler.cs

@@ -17,11 +17,12 @@ public class LinkCrawler
         public string BaseUrl { get; set; }
         public bool CheckImages { get; set; }
         public bool FollowRedirects { get; set; }
-        public RestRequest RestRequest { get; set; }
+        private RestRequest GetRequest { get; set; }
+        private RestClient Client{ get; set; }
         public IEnumerable<IOutput> Outputs { get; set; }
         public IValidUrlParser ValidUrlParser { get; set; }
         public bool OnlyReportBrokenLinksToOutput { get; set; }
-        public static List<LinkModel> UrlList;
+        public List<LinkModel> UrlList;
         private ISettings _settings;
         private Stopwatch timer;
 
@@ -33,7 +34,10 @@ public LinkCrawler(IEnumerable<IOutput> outputs, IValidUrlParser validUrlParser,
             CheckImages = settings.CheckImages;
             FollowRedirects = settings.FollowRedirects;
             UrlList = new List<LinkModel>();
-            RestRequest = new RestRequest(Method.GET).SetHeader("Accept", "*/*");
+            GetRequest = new RestRequest(Method.GET).SetHeader("Accept", "*/*");
+            Client = new RestClient() { FollowRedirects = false }; // we don't want RestSharp following the redirects, otherwise we won't see them
+            // https://stackoverflow.com/questions/8823349/how-do-i-use-the-cookie-container-with-restsharp-and-asp-net-sessions - set cookies up according to this link?
+
             OnlyReportBrokenLinksToOutput = settings.OnlyReportBrokenLinksToOutput;
             _settings = settings;
             this.timer = new Stopwatch();
@@ -49,9 +53,9 @@ public void Start()
         public void SendRequest(string crawlUrl, string referrerUrl = "")
         {
             var requestModel = new RequestModel(crawlUrl, referrerUrl, BaseUrl);
-            var restClient = new RestClient(new Uri(crawlUrl)) { FollowRedirects = false }; // we don't want RestSharp following the redirects, otherwise we won't see them
+            Client.BaseUrl = new Uri(crawlUrl);
 
-            restClient.ExecuteAsync(RestRequest, response =>
+            Client.ExecuteAsync(GetRequest, response =>
             {
                 if (response == null)
                     return;
@@ -67,25 +71,34 @@ public void ProcessResponse(IResponseModel responseModel)
 
             // follow 3xx redirects
             if (FollowRedirects && responseModel.IsRedirect)
-            {
-                string redirectUrl;
-                if (responseModel.Location.StartsWith("/"))
-                    redirectUrl = responseModel.RequestedUrl.GetUrlBase() + responseModel.Location;
-                else
-                    redirectUrl = responseModel.Location;
-                SendRequest(redirectUrl, responseModel.RequestedUrl);
-            }
+                FollowRedirect(responseModel);
 
             // follow internal links in response
             if (responseModel.ShouldCrawl)
-                CrawlForLinksInResponse(responseModel);
+                CrawlLinksInResponse(responseModel);
+        }
+
+        private void FollowRedirect(IResponseModel responseModel)
+        {
+            string redirectUrl;
+            if (responseModel.Location.StartsWith("/"))
+                redirectUrl = responseModel.RequestedUrl.GetUrlBase() + responseModel.Location; // add base URL to relative links
+            else
+                redirectUrl = responseModel.Location;
+
+            SendRequest(redirectUrl, responseModel.RequestedUrl);
         }
 
-        public void CrawlForLinksInResponse(IResponseModel responseModel)
+        public void CrawlLinksInResponse(IResponseModel responseModel)
         {
             var linksFoundInMarkup = MarkupHelpers.GetValidUrlListFromMarkup(responseModel.Markup, ValidUrlParser, CheckImages);
 
-            foreach (var url in linksFoundInMarkup)
+            SendRequestsToLinks(linksFoundInMarkup, responseModel.RequestedUrl);
+        }
+
+        private void SendRequestsToLinks(List<string> urls, string referrerUrl)
+        {
+            foreach (string url in urls)
             {
                 lock (UrlList)
                 {
@@ -94,7 +107,7 @@ public void CrawlForLinksInResponse(IResponseModel responseModel)
 
                     UrlList.Add(new LinkModel(url));
                 }
-                SendRequest(url, responseModel.RequestedUrl);
+                SendRequest(url, referrerUrl);
             }
         }
 

LinkCrawler/LinkCrawler/Models/RequestModel.cs

@@ -7,7 +7,7 @@ public class RequestModel
         public string Url;
         public string ReferrerUrl;
         public bool IsInternalUrl { get; set; }
-        public RestClient Client;
+        private readonly RestClient Client;
 
         public RequestModel(string url, string referrerUrl, string baseUrl)
         {

LinkCrawler/LinkCrawler/Models/ResponseModel.cs

@@ -13,14 +13,15 @@ public class ResponseModel : IResponseModel
         public string RequestedUrl { get; }
         public string ReferrerUrl { get; }
         public string Location { get; }
+        public IList<RestResponseCookie> Cookies { get; }
 
         public HttpStatusCode StatusCode { get; }
         public int StatusCodeNumber { get { return (int)StatusCode; } }
         public bool IsSuccess { get; }
         public bool IsInteresting { get; }
         public bool IsRedirect { get; }
         public bool ShouldCrawl { get; }
-        public string ErrorMessage { get; }
+        private string ErrorMessage { get; }
 
         public ResponseModel(IRestResponse restResponse, RequestModel requestModel, ISettings settings)
         {
@@ -29,6 +30,8 @@ public ResponseModel(IRestResponse restResponse, RequestModel requestModel, ISet
             RequestedUrl = requestModel.Url;
             Location = restResponse.GetHeaderByName("Location"); // returns null if no Location header present in the response
             ErrorMessage = restResponse.ErrorMessage;
+            Cookies = restResponse.Cookies;
+
             IsSuccess = settings.IsSuccess(StatusCode);
             IsInteresting = settings.IsInteresting(StatusCode);
             IsRedirect = settings.IsRedirect(StatusCode);

LinkCrawler/LinkCrawler/Program.cs

@@ -11,7 +11,6 @@ class Program
     {
         static void Main(string[] args)
         {
-            
             using (var container = Container.For<StructureMapRegistry>())
             {
                 var linkCrawler = container.GetInstance<LinkCrawler>();