Refactor code and simplify file processing

package.json

@@ -13,6 +13,7 @@
     "pretest": "npm run build && node ./dist/src/bin/index.js ./__mocks__/content",
     "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
     "build": "tsc --p tsconfig.lib.json",
+    "dev": "tsc --p tsconfig.lib.json --watch",
     "changeset": "changeset",
     "prepublishOnly": "npm run build",
     "release": "changeset publish"
@@ -55,6 +56,7 @@
     "@changesets/changelog-github": "^0.4.8",
     "@changesets/cli": "^2.26.1",
     "@types/jest": "^29.5.1",
+    "@types/node": "^20.8.7",
     "@typescript-eslint/eslint-plugin": "^5.59.5",
     "@typescript-eslint/parser": "^5.59.5",
     "eslint": "^8.40.0",

src/bin/index.js → src/bin/index.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 
 import { MarkdownDB } from "../lib/markdowndb.js";
+// import MddbFolderWatcher from "./watch.js";
 
 // TODO get these from markdowndb.config.js or something
 const dbPath = "markdown.db";
@@ -18,11 +19,12 @@
   },
 });
 
+// Ignore top-level await errors
+//@ts-ignore
 await client.init();
 
+//@ts-ignore
 await client.indexFolder({
   folderPath: contentPath,
   ignorePatterns: ignorePatterns,
 });
-
-process.exit();

src/lib/DbQueryManager.ts

@@ -0,0 +1,116 @@
+import { Knex } from "knex";
+import { MddbFile, MddbTag, MddbLink } from "./schema.js";
+import { FilesQuery } from "./types/DbQueryTypes.js";
+
+export class DbQueryManager {
+  private db: Knex;
+
+  constructor(db: Knex) {
+    this.db = db;
+  }
+
+  async getFileById(id: string): Promise<MddbFile | null> {
+    const file = await this.db.from("files").where("_id", id).first();
+    return new MddbFile(file);
+  }
+
+  async getFileByUrl(url: string): Promise<MddbFile | null> {
+    const file = await this.db
+      .from("files")
+      .where("url_path", encodeURI(url))
+      .first();
+    return new MddbFile(file);
+  }
+
+  async getFiles(query?: FilesQuery): Promise<MddbFile[]> {
+    const { filetypes, tags, extensions, folder, frontmatter } = query || {};
+
+    const files = await this.db
+      // TODO join only if tags are specified ?
+      .leftJoin("file_tags", "files._id", "file_tags.file")
+      .where((builder) => {
+        // TODO temporary solution before we have a proper way to filter files by and assign file types
+        if (folder) {
+          builder.whereLike("url_path", `${folder}/%`);
+        }
+
+        if (tags) {
+          builder.whereIn("tag", tags);
+        }
+
+        if (extensions) {
+          builder.whereIn("extension", extensions);
+        }
+
+        if (filetypes) {
+          builder.whereIn("filetype", filetypes);
+        }
+
+        if (frontmatter) {
+          Object.entries(frontmatter).forEach(([key, value]) => {
+            if (typeof value === "string" || typeof value === "number") {
+              builder.whereRaw(`json_extract(metadata, '$.${key}') = ?`, [
+                value,
+              ]);
+            } else if (typeof value === "boolean") {
+              if (value) {
+                builder.whereRaw(`json_extract(metadata, '$.${key}') = ?`, [
+                  true,
+                ]);
+              } else {
+                builder.where(function () {
+                  this.whereRaw(`json_extract(metadata, '$.${key}') = ?`, [
+                    false,
+                  ]).orWhereRaw(`json_extract(metadata, '$.${key}') IS NULL`);
+                });
+              }
+            }
+
+            // To check if the provided value exists in an array inside the JSON
+            else {
+              builder.whereRaw(`json_extract(metadata, '$.${key}') LIKE ?`, [
+                `%${value}%`,
+              ]);
+            }
+          });
+        }
+      })
+      .select("files.*")
+      .from("files")
+      .groupBy("_id");
+
+    return files.map((file) => new MddbFile(file));
+  }
+
+  async getTags(): Promise<MddbTag[]> {
+    const tags = await this.db("tags").select();
+    return tags.map((tag) => new MddbTag(tag));
+  }
+
+  async getLinks(query?: {
+    fileId: string;
+    linkType?: "normal" | "embed";
+    direction?: "forward" | "backward";
+  }): Promise<MddbLink[]> {
+    const { fileId, direction = "forward", linkType } = query || {};
+    const joinKey = direction === "forward" ? "from" : "to";
+    const where = {
+      [joinKey]: fileId,
+    };
+    if (linkType) {
+      where["link_type"] = linkType;
+    }
+    const dbLinks = await this.db
+      .select("links.*")
+      .from("links")
+      .rightJoin("files", `links.${joinKey}`, "=", "files._id")
+      .where(where);
+
+    const links = dbLinks.map((link) => new MddbLink(link));
+    return links;
+  }
+
+  _destroyDb() {
+    this.db.destroy();
+  }
+}

src/lib/indexFolderToObjects.ts

@@ -0,0 +1,63 @@
+import { WikiLink, getUniqueValues, recursiveWalkDir } from "../utils/index.js";
+import { markdownToObject } from "./markdownToObject.js";
+import { File, FileTag, Link, Tag } from "./types/schemaTypes.js";
+
+export function indexFolderToObjects(
+  folderPath: string,
+  ignorePatterns?: RegExp[],
+  pathToUrlResolver?: (filePath: string) => string
+) {
+  const filePathsToIndex = recursiveWalkDir(folderPath);
+  const files: File[] = [];
+  const tags: Tag[] = [];
+  const fileTags: FileTag[] = [];
+  const links: Link[] = [];
+  const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) => {
+    return !(
+      ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath))
+    );
+  });
+
+  for (const filePath of filteredFilePathsToIndex) {
+    const fileObject = markdownToObject(folderPath, filePath, filePathsToIndex);
+
+    files.push(fileObject.file);
+    tags.push(...fileObject.tags);
+
+    const fileTagsToInsert = fileObject.tags.map((tag) => ({
+      tag: tag.name,
+      file: fileObject.file._id,
+    }));
+    const uniqueFileTags = getUniqueValues(fileTagsToInsert);
+    fileTags.push(...uniqueFileTags);
+
+    const linksToInsert: Link[] = processWikiLinks(
+      fileObject.links,
+      fileObject.file._id,
+      filePathsToIndex
+    );
+    links.push(...linksToInsert);
+  }
+
+  const uniqueTags = getUniqueValues(tags);
+  return {
+    files: files,
+    tags: uniqueTags,
+    fileTags: fileTags,
+    links: links,
+  };
+}
+
+function processWikiLinks(
+  links: WikiLink[],
+  fileId: string,
+  filePathsToIndex: string[]
+): Link[] {
+  return links
+    .map((link) => ({
+      from: fileId,
+      to: filePathsToIndex.find((file) => file === link.linkSrc)!,
+      link_type: link.linkType,
+    }))
+    .filter((link) => link.to !== undefined);
+}

src/lib/markdownToObject.ts

@@ -0,0 +1,82 @@
+import fs from "fs";
+import path from "path";
+import { parseFile } from "../utils/index.js";
+import { FileObject } from "./types/FileObject.js";
+import { MddbFile } from "./schema.js";
+import {
+  defaultFilePathToUrl,
+} from "./markdowndb.js";
+import { generateFileIdFromPath } from "../utils/index.js";
+import { getFileExtensionFromPath } from "../utils/index.js";
+
+export function markdownToObject(
+  folderPath: string,
+  filePath: string,
+  filePathsToIndex: string[]
+): FileObject {
+  const id = generateFileIdFromPath(filePath);
+  const extension = getFileExtensionFromPath(filePath);
+  const defaultFileJson = {
+    file: {
+      _id: id,
+      file_path: filePath,
+      extension,
+      url_path: null,
+      filetype: null,
+      metadata: null,
+    },
+    tags: [],
+    links: [],
+  };
+
+  // if not md or mdx return this
+  const isExtensionNotSupported =
+    !MddbFile.supportedExtensions.includes(extension);
+  if (isExtensionNotSupported) {
+    return defaultFileJson;
+  }
+
+  // url_path
+  const pathRelativeToFolder = path.relative(folderPath, filePath);
+  // const urlPath = defaultFilePathToUrl(pathRelativeToFolder);
+  const urlPath = defaultFilePathToUrl(pathRelativeToFolder);
+
+  // metadata, tags, links
+  let source: string, metadata, links;
+  try {
+    source = fs.readFileSync(filePath, {
+      encoding: "utf8",
+      flag: "r",
+    });
+
+    ({ metadata, links } = parseFile(source, {
+      permalinks: filePathsToIndex,
+    }));
+  } catch (e) {
+    console.error(
+      `Failed to parse file ${filePath}. Waiting for file changes.`
+    );
+
+    return defaultFileJson;
+  }
+
+  // get tags in the file
+  const tags = (metadata?.tags || []).map((tag: string) => {
+    return { name: tag };
+  });
+
+  let fileObject: FileObject = {
+    file: {
+      _id: id,
+      extension: extension,
+      url_path: urlPath,
+      file_path: filePath,
+      filetype: metadata?.type || null,
+      metadata: metadata,
+    },
+    tags: [...tags],
+    links: links,
+  };
+
+  return fileObject;
+}

src/lib/markdowndb.spec.ts

@@ -1,7 +1,8 @@
 // import knex from "knex";
 import { MarkdownDB } from "./markdowndb";
-import { File, MddbFile, Table } from "./schema";
+import {  MddbFile } from "./schema";
 import { recursiveWalkDir } from "../utils";
+import { File, Table } from "./types/schemaTypes";
 
 /**
  * @jest-environment node
@@ -173,14 +174,14 @@
     test("can find file by url path", async () => {
       const dbFile = await mddb.getFileByUrl("blog/blog2");
       expect(dbFile).not.toBeNull();
       expect(dbFile!.url_path).toBe("blog/blog2");
     });
 
     test("can find file by id", async () => {
       const dbFile = await mddb.getFileByUrl("blog/blog2");
       const dbFileById = await mddb.getFileById(dbFile!._id);
       expect(dbFileById).not.toBeNull();
       expect(dbFileById!.url_path).toBe("blog/blog2");
     });
   });
 
@@ -208,10 +209,10 @@
       const toFile = await mddb.getFileByUrl("blog0");
 
       const forwardLinks = await mddb.getLinks({
         fileId: fromFile!._id,
       });
       expect(forwardLinks.length).toBe(1);
       expect(forwardLinks[0].to).toBe(toFile!._id);
     });
 
     test("can get all backward links of a file", async () => {
@@ -220,13 +221,13 @@
       const fromFile2 = await mddb.getFileByUrl("blog/blog1");
 
       const backwardLinks = await mddb.getLinks({
         fileId: toFile!._id,
         direction: "backward",
       });
       const backwardLinksFileIds = backwardLinks.map((l) => l.from);
       expect(backwardLinksFileIds).toHaveLength(2);
       expect(backwardLinksFileIds).toContain(fromFile1!._id);
       expect(backwardLinksFileIds).toContain(fromFile2!._id);
     });
   });