doFetch: Try a failed HEAD again

linkcheck/linkcheck.go

@@ -225,45 +225,53 @@ func (c *crawler) doFetch(ctx context.Context, pageurl string) (links, ids []str
 		method = http.MethodHead
 	}
 	var doc html.Node
-	err = requests.
-		URL(pageurl).
-		Method(method).
-		Accept("text/html,application/xhtml+xml,application/xml,*/*").
-		UserAgent(c.userAgent).
-		Client(c.Client).
-		CheckStatus(http.StatusOK).
-		CheckContentType(
-			"text/html",
-			"application/xhtml+xml",
-			"text/xml",
-			"text/plain",
-		).
-		CheckPeek(512, func(b []byte) error {
-			if ct := http.DetectContentType(b); !strings.Contains(ct, "html") {
-				return fmt.Errorf("content-type is %s", ct)
-			}
-			return nil
-		}).
-		AddValidator(func(res *http.Response) error {
-			// If we've been 30X redirected, pageurl will not be response URL
-			pageurl = res.Request.URL.String()
-			return nil
-		}).
-		Handle(requests.ToHTML(&doc)).
-		Fetch(ctx)
-
-	if err != nil {
+	for {
+		err = requests.
+			URL(pageurl).
+			Method(method).
+			Accept("text/html,application/xhtml+xml,application/xml,*/*").
+			UserAgent(c.userAgent).
+			Client(c.Client).
+			CheckStatus(http.StatusOK).
+			CheckContentType(
+				"text/html",
+				"application/xhtml+xml",
+				"text/xml",
+				"text/plain",
+			).
+			CheckPeek(512, func(b []byte) error {
+				if ct := http.DetectContentType(b); !strings.Contains(ct, "html") {
+					return fmt.Errorf("content-type is %s", ct)
+				}
+				return nil
+			}).
+			AddValidator(func(res *http.Response) error {
+				// If we've been 30X redirected, pageurl will not be response URL
+				pageurl = res.Request.URL.String()
+				return nil
+			}).
+			Handle(requests.ToHTML(&doc)).
+			Fetch(ctx)
+		if method == http.MethodGet || err == nil {
+			break
+		}
+		method = http.MethodGet
+	}
+	switch {
+	case err == nil:
+		break
+	case
 		// report 401, 404, 410; ignore temporary status errors
-		if requests.HasStatusErr(err,
+		requests.HasStatusErr(err,
 			http.StatusUnauthorized,
 			http.StatusNotFound,
-			http.StatusGone) {
-			return nil, nil, err
-		}
+			http.StatusGone),
 		// Report DNS errors
-		if d := new(net.DNSError); errors.As(err, &d) {
-			return nil, nil, err
-		}
+		errors.As(err, new(*net.DNSError)):
+
+		return nil, nil, err
+
+	default:
 		// Ignore other errors
 		c.l.Debug("doFetch ignore", "url", pageurl, "err", err)
 		return nil, nil, nil