list links from specified page

deps.ts

@@ -31,6 +31,8 @@ export { Sha1 } from "https://deno.land/[email protected]/hash/sha1.ts";
 
 export { parse } from "https://deno.land/[email protected]/encoding/csv.ts";
 
+export { DOMParser } from "https://deno.land/x/deno_dom/deno-dom-wasm.ts";
+
 export const zipWrapper = {
   decompress: _decompress,
 };

dim.ts

@@ -52,6 +52,14 @@ await new Command()
         "-A, --asyncInstall",
         "Execute asyncronous install.",
       )
+      .option(
+        "-P, --pageInstall <pageInstall:string>",
+        "Execute install from links in specified page.",
+      )
+      .option(
+        "-e, --expression <expression:string>",
+        "Filter PageInstall result by regular expression.",
+      )
       .description(
         "Install the data.\n" +
           "Specify the url of data. If you dont't specify argument, install all data which is not installed dependency.",

libs/action_helper/installer.ts

@@ -1,4 +1,4 @@
-import { Colors, Confirm, Input, ky, Number, Sha1 } from "../../deps.ts";
+import { Colors, Confirm, DOMParser, Input, ky, Number, Sha1 } from "../../deps.ts";
 import { DEFAULT_DIM_LOCK_FILE_PATH, ENCODINGS } from "../consts.ts";
 import { Downloader } from "../downloader.ts";
 import { ConsoleAnimation } from "../console_animation.ts";
@@ -310,3 +310,43 @@ export const interactiveInstall = async (catalogs: Catalog[]): Promise<string> =
 
   return fullPath;
 };
+
+export const installFromPage = async (
+  pageInstallUrl: string,
+  expression: string | undefined,
+  postProcesses: string[] | undefined,
+  headers: Record<string, string> = {},
+  name?: string,
+) => {
+  const getResult = await fetch(pageInstallUrl);
+  const html = await getResult.text();
+  const document = new DOMParser().parseFromString(html, "text/html")!;
+  const linklist = document.getElementsByTagName("a")!;
+  let idx = 1;
+  for (const link of linklist) {
+    const re = new RegExp(expression as string, "g");
+    const href = new URL(
+      link.getAttribute("href") as string,
+      pageInstallUrl,
+    ).toString();
+    if (re.test(href)) {
+      const dataName = `${name}_${idx}`;
+      await installFromURL(
+        href,
+        dataName,
+        postProcesses,
+        headers,
+      ).then((fullPath) => {
+        console.log(Colors.green(`Installed to ${fullPath}`));
+        idx += 1;
+      }).catch((error) => {
+        console.error(
+          Colors.red("Failed to pageInstall."),
+          Colors.red("target:" + href),
+          Colors.red(error.message),
+        );
+      });
+    }
+  }
+  return idx - 1;
+};

libs/actions.ts

@@ -4,7 +4,13 @@ import { DimFileAccessor, DimLockFileAccessor } from "./accessor.ts";
 import { ky, Sha1 } from "../deps.ts";
 import { CkanApiClient } from "./ckan_api_client.ts";
 import { createDataFilesDir, initDimFile, initDimLockFile } from "./action_helper/initializer.ts";
-import { installFromDimFile, installFromURL, interactiveInstall, parseHeader } from "./action_helper/installer.ts";
+import {
+  installFromDimFile,
+  installFromPage,
+  installFromURL,
+  interactiveInstall,
+  parseHeader,
+} from "./action_helper/installer.ts";
 import { OpenAIClient } from "./openai_client.ts";
 import { ConsoleAnimation } from "./console_animation.ts";
 
@@ -26,23 +32,42 @@ export class InstallAction {
       file?: string;
       force?: boolean;
       asyncInstall?: boolean;
+      pageInstall?: string;
+      expression?: string;
     },
     url: string | undefined,
   ) {
     if (url && options.file) {
       console.log(
-        Colors.red("Cannot use -f option and URL at the same time."),
+        Colors.red("Can not use -f option and URL at the same time."),
       );
       Deno.exit(1);
     }
+    if (url && options.pageInstall) {
+      console.log(Colors.red("Cannot use -P option and URL at the same time."));
+      Deno.exit(1);
+    }
+    if (options.file && options.pageInstall) {
+      console.log(
+        Colors.red("Can not use -f option and -P option at the same time."),
+      );
+      Deno.exit(1);
+    }
+    if (options.pageInstall && !options.expression) {
+      console.log(Colors.red("Can not use -P option without -e option."));
+      Deno.exit(1);
+    }
 
     const parsedHeaders: Record<string, string> = parseHeader(options.headers);
+
     if (url !== undefined) {
       if (options.name === undefined) {
         console.log(Colors.red("The -n option is not specified."));
         Deno.exit(1);
       }
-      const targetContent = new DimFileAccessor().getContents().find((c) => c.name === options.name);
+      const targetContent = new DimFileAccessor()
+        .getContents()
+        .find((c) => c.name === options.name);
       if (targetContent !== undefined && !options.force) {
         console.log(Colors.red("The name already exists."));
         console.log(
@@ -57,18 +82,33 @@ export class InstallAction {
         options.name,
         options.postProcesses,
         parsedHeaders,
-      ).catch(
-        (error) => {
-          console.error(
-            Colors.red("Failed to install."),
-            Colors.red(error.message),
-          );
-          Deno.exit(1);
-        },
-      );
-      console.log(
-        Colors.green(`Installed to ${fullPath}`),
-      );
+      ).catch((error) => {
+        console.error(
+          Colors.red("Failed to install."),
+          Colors.red(error.message),
+        );
+        Deno.exit(1);
+      });
+      console.log(Colors.green(`Installed to ${fullPath}`));
+    } else if (options.pageInstall !== undefined) {
+      if (options.name === undefined) {
+        console.log(Colors.red("The -n option is not specified."));
+        Deno.exit(1);
+      }
+      const installed = await installFromPage(
+        options.pageInstall,
+        options.expression,
+        options.postProcesses,
+        parsedHeaders,
+        options.name,
+      ).catch((error) => {
+        console.error(
+          Colors.red("Failed to pageInstall."),
+          Colors.red(error.message),
+        );
+        Deno.exit(1);
+      });
+      console.log(Colors.green(`Completed page install ${installed} files.`));
     } else {
       const lockContentList = await installFromDimFile(
         options.file || DEFAULT_DIM_FILE_PATH,
@@ -81,9 +121,7 @@ export class InstallAction {
 
       if (lockContentList !== undefined) {
         if (lockContentList.length != 0) {
-          console.log(
-            Colors.green(`Successfully installed.`),
-          );
+          console.log(Colors.green(`Successfully installed.`));
         } else {
           console.log("All contents have already been installed.");
           console.log(