clearer scope check (#615)

split isInScope into a protected sync getScope() used for link
extraction (no need for async as we know seed is already set) and which
returns url / isOOS count.
and a simpler, public async isInScope() which just returns a bool, but
also ensures the seed exists.

src/crawler.ts

@@ -611,7 +611,7 @@ export class Crawler {
     }
   }
 
-  async isInScope(
+  protected getScope(
     {
       seedId,
       url,
@@ -620,13 +620,25 @@ export class Crawler {
     }: { seedId: number; url: string; depth: number; extraHops: number },
     logDetails = {},
   ) {
+    return this.seeds[seedId].isIncluded(url, depth, extraHops, logDetails);
+  }
+
+  async isInScope(
+    {
+      seedId,
+      url,
+      depth,
+      extraHops,
+    }: { seedId: number; url: string; depth: number; extraHops: number },
+    logDetails = {},
+  ): Promise<boolean> {
     const seed = await this.crawlState.getSeedAt(
       this.seeds,
       this.numOriginalSeeds,
       seedId,
     );
 
-    return seed.isIncluded(url, depth, extraHops, logDetails);
+    return !!seed.isIncluded(url, depth, extraHops, logDetails);
   }
 
   async setupPage({
@@ -2014,7 +2026,7 @@ self.__bx_behaviors.selectMainBehavior();
       const newExtraHops = extraHops + 1;
 
       for (const possibleUrl of urls) {
-        const res = await this.isInScope(
+        const res = this.getScope(
           { url: possibleUrl, extraHops: newExtraHops, depth, seedId },
           logDetails,
         );
@@ -2023,11 +2035,6 @@ self.__bx_behaviors.selectMainBehavior();
           continue;
         }
 
-        if (res === true) {
-          logger.warn("Invalid scope response: true", logDetails, "links");
-          continue;
-        }
-
         const { url, isOOS } = res;
 
         if (url) {

src/util/seeds.ts

@@ -213,7 +213,12 @@ export class ScopedSeed {
     return depth >= this.maxDepth;
   }
 
-  isIncluded(url: string, depth: number, extraHops = 0, logDetails = {}) {
+  isIncluded(
+    url: string,
+    depth: number,
+    extraHops = 0,
+    logDetails = {},
+  ): { url: string; isOOS: boolean } | false {
     if (depth > this.maxDepth) {
       return false;
     }
@@ -231,7 +236,7 @@ export class ScopedSeed {
     url = urlParsed.href;
 
     if (url === this.url) {
-      return true;
+      return { url, isOOS: false };
     }
 
     // skip already crawled