Refactor code and simplify file processing

package.json

@@ -13,6 +13,7 @@
     "pretest": "npm run build && node ./dist/src/bin/index.js ./__mocks__/content",
     "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
     "build": "tsc --p tsconfig.lib.json",
+    "dev": "tsc --p tsconfig.lib.json --watch",
     "changeset": "changeset",
     "prepublishOnly": "npm run build",
     "release": "changeset publish"
@@ -55,6 +56,7 @@
     "@changesets/changelog-github": "^0.4.8",
     "@changesets/cli": "^2.26.1",
     "@types/jest": "^29.5.1",
+    "@types/node": "^20.8.7",
     "@typescript-eslint/eslint-plugin": "^5.59.5",
     "@typescript-eslint/parser": "^5.59.5",
     "eslint": "^8.40.0",

src/lib/DbQueryManager.ts

@@ -0,0 +1,116 @@
+import { Knex } from "knex";
+import { MddbFile, MddbTag, MddbLink } from "./schema.js";
+import { FilesQuery } from "./types/DbQueryTypes.js";
+
+export class DbQueryManager {
+  private db: Knex;
+
+  constructor(db: Knex) {
+    this.db = db;
+  }
+
+  async getFileById(id: string): Promise<MddbFile | null> {
+    const file = await this.db.from("files").where("_id", id).first();
+    return new MddbFile(file);
+  }
+
+  async getFileByUrl(url: string): Promise<MddbFile | null> {
+    const file = await this.db
+      .from("files")
+      .where("url_path", encodeURI(url))
+      .first();
+    return new MddbFile(file);
+  }
+
+  async getFiles(query?: FilesQuery): Promise<MddbFile[]> {
+    const { filetypes, tags, extensions, folder, frontmatter } = query || {};
+
+    const files = await this.db
+      // TODO join only if tags are specified ?
+      .leftJoin("file_tags", "files._id", "file_tags.file")
+      .where((builder) => {
+        // TODO temporary solution before we have a proper way to filter files by and assign file types
+        if (folder) {
+          builder.whereLike("url_path", `${folder}/%`);
+        }
+
+        if (tags) {
+          builder.whereIn("tag", tags);
+        }
+
+        if (extensions) {
+          builder.whereIn("extension", extensions);
+        }
+
+        if (filetypes) {
+          builder.whereIn("filetype", filetypes);
+        }
+
+        if (frontmatter) {
+          Object.entries(frontmatter).forEach(([key, value]) => {
+            if (typeof value === "string" || typeof value === "number") {
+              builder.whereRaw(`json_extract(metadata, '$.${key}') = ?`, [
+                value,
+              ]);
+            } else if (typeof value === "boolean") {
+              if (value) {
+                builder.whereRaw(`json_extract(metadata, '$.${key}') = ?`, [
+                  true,
+                ]);
+              } else {
+                builder.where(function () {
+                  this.whereRaw(`json_extract(metadata, '$.${key}') = ?`, [
+                    false,
+                  ]).orWhereRaw(`json_extract(metadata, '$.${key}') IS NULL`);
+                });
+              }
+            }
+
+            // To check if the provided value exists in an array inside the JSON
+            else {
+              builder.whereRaw(`json_extract(metadata, '$.${key}') LIKE ?`, [
+                `%${value}%`,
+              ]);
+            }
+          });
+        }
+      })
+      .select("files.*")
+      .from("files")
+      .groupBy("_id");
+
+    return files.map((file) => new MddbFile(file));
+  }
+
+  async getTags(): Promise<MddbTag[]> {
+    const tags = await this.db("tags").select();
+    return tags.map((tag) => new MddbTag(tag));
+  }
+
+  async getLinks(query?: {
+    fileId: string;
+    linkType?: "normal" | "embed";
+    direction?: "forward" | "backward";
+  }): Promise<MddbLink[]> {
+    const { fileId, direction = "forward", linkType } = query || {};
+    const joinKey = direction === "forward" ? "from" : "to";
+    const where = {
+      [joinKey]: fileId,
+    };
+    if (linkType) {
+      where["link_type"] = linkType;
+    }
+    const dbLinks = await this.db
+      .select("links.*")
+      .from("links")
+      .rightJoin("files", `links.${joinKey}`, "=", "files._id")
+      .where(where);
+
+    const links = dbLinks.map((link) => new MddbLink(link));
+    return links;
+  }
+
+  _destroyDb() {
+    this.db.destroy();
+  }
+}

src/lib/indexFolderToObjects.ts

@@ -0,0 +1,79 @@
+import { getUniqueValues, recursiveWalkDir } from "../utils/index.js";
+import type { WikiLink } from "../utils/index.js";
+import { extractFileSchemeFromObject } from "../utils/extractFileSchemeFromObject.js";
+import { readLocalMarkdownFileToObject } from "./readLocalMarkdownFileToObject.js";
+import type { File, FileTag, Link } from "./types/schemaTypes.js";
+
+export function indexFolderToObjects(
+  folderPath: string,
+  pathToUrlResolver: (filePath: string) => string,
+  ignorePatterns?: RegExp[]
+) {
+  const filePathsToIndex = recursiveWalkDir(folderPath);
+  const files: File[] = [];
+  const tags: string[] = [];
+  const fileTags: FileTag[] = [];
+  const links: Link[] = [];
+  const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) =>
+    shouldIncludeFile(filePath, ignorePatterns)
+  );
+
+  for (const filePath of filteredFilePathsToIndex) {
+    const fileObject = readLocalMarkdownFileToObject(
+      folderPath,
+      filePath,
+      filePathsToIndex,
+      pathToUrlResolver
+    );
+
+    const file = extractFileSchemeFromObject(fileObject);
+    files.push(file);
+
+    tags.push(...fileObject.tags);
+
+    const fileTagsToInsert = fileObject.tags.map((tag) => ({
+      tag: tag,
+      file: fileObject._id,
+    }));
+    fileTags.push(...fileTagsToInsert);
+
+    const linksToInsert: Link[] = processWikiLinks(
+      fileObject.links,
+      fileObject._id,
+      filePathsToIndex
+    );
+    links.push(...linksToInsert);
+  }
+
+  const uniqueTags = getUniqueValues(tags);
+  const TagsToInsert = uniqueTags.map((tag) => ({ name: tag }));
+  return {
+    files: files,
+    tags: TagsToInsert,
+    fileTags: fileTags,
+    links: links,
+  };
+}
+
+function processWikiLinks(
+  links: WikiLink[],
+  fileId: string,
+  filePathsToIndex: string[]
+): Link[] {
+  return links
+    .map((link) => ({
+      from: fileId,
+      to: filePathsToIndex.find((file) => file === link.linkSrc)!,
+      link_type: link.linkType,
+    }))
+    .filter((link) => link.to !== undefined);
+}
+
+function shouldIncludeFile(
+  filePath: string,
+  ignorePatterns?: RegExp[]
+): boolean {
+  return !(
+    ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath))
+  );
+}

src/lib/markdowndb.spec.ts

@@ -1,7 +1,8 @@
 // import knex from "knex";
 import { MarkdownDB } from "./markdowndb";
-import { File, MddbFile, Table } from "./schema";
+import {  MddbFile } from "./schema";
 import { recursiveWalkDir } from "../utils";
+import { File, Table } from "./types/schemaTypes";
 
 /**
  * @jest-environment node
@@ -173,14 +174,14 @@
     test("can find file by url path", async () => {
       const dbFile = await mddb.getFileByUrl("blog/blog2");
       expect(dbFile).not.toBeNull();
       expect(dbFile!.url_path).toBe("blog/blog2");
     });
 
     test("can find file by id", async () => {
       const dbFile = await mddb.getFileByUrl("blog/blog2");
       const dbFileById = await mddb.getFileById(dbFile!._id);
       expect(dbFileById).not.toBeNull();
       expect(dbFileById!.url_path).toBe("blog/blog2");
     });
   });
 
@@ -208,10 +209,10 @@
       const toFile = await mddb.getFileByUrl("blog0");
 
       const forwardLinks = await mddb.getLinks({
         fileId: fromFile!._id,
       });
       expect(forwardLinks.length).toBe(1);
       expect(forwardLinks[0].to).toBe(toFile!._id);
     });
 
     test("can get all backward links of a file", async () => {
@@ -220,13 +221,13 @@
       const fromFile2 = await mddb.getFileByUrl("blog/blog1");
 
       const backwardLinks = await mddb.getLinks({
         fileId: toFile!._id,
         direction: "backward",
       });
       const backwardLinksFileIds = backwardLinks.map((l) => l.from);
       expect(backwardLinksFileIds).toHaveLength(2);
       expect(backwardLinksFileIds).toContain(fromFile1!._id);
       expect(backwardLinksFileIds).toContain(fromFile2!._id);
     });
   });
 
@@ -268,7 +269,7 @@
         },
       ];
       // TODO fix types
       expect(() => MddbFile.batchInsert(mddb as any, files)).toThrow();
     });
   });
 });