From 81f85f6c382722031f24f75e8ac822bcf91a3fa5 Mon Sep 17 00:00:00 2001 From: mohamed yahia Date: Mon, 11 Dec 2023 20:51:57 +0200 Subject: [PATCH] [#54 Computed Fields] (#86) * Computed Fields * Make configuration optional * Add changeset for computed fields * Update changeset --- .changeset/flat-bikes-own.md | 6 ++++++ README.md | 37 +++++++++++++++++++++++++++++++++--- src/lib/CustomConfig.ts | 6 ++++++ src/lib/indexFolder.ts | 6 +++++- src/lib/markdowndb.ts | 4 ++++ src/lib/parseFile.ts | 1 + src/lib/process.ts | 10 ++++++++-- src/tests/process.spec.ts | 1 + 8 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 .changeset/flat-bikes-own.md create mode 100644 src/lib/CustomConfig.ts diff --git a/.changeset/flat-bikes-own.md b/.changeset/flat-bikes-own.md new file mode 100644 index 0000000..f791297 --- /dev/null +++ b/.changeset/flat-bikes-own.md @@ -0,0 +1,6 @@ +--- +"mddb": minor +--- + +[#54 Computed Fields] +- Introduce a configuration parameter to support the inclusion of a custom function for computing fields. diff --git a/README.md b/README.md index b6798c8..490d630 100644 --- a/README.md +++ b/README.md @@ -115,18 +115,49 @@ const blogs = await mddb.getFiles({ }); ``` -# Configuring `markdowndb.config.js` +## Computed Fields + +This feature helps you define functions that compute additional fields you want to include. + +### Step 1: Define the Computed Field Function + +Next, define a function that computes the additional field you want to include. In this example, we have a function named `addTitle` that extracts the title from the first heading in the AST (Abstract Syntax Tree) of a Markdown file. + +```javascript +const addTitle = (fileInfo, ast) => { + // Find the first header node in the AST + const headerNode = ast.children.find((node) => node.type === "heading"); + + // Extract the text content from the header node + const title = headerNode + ? headerNode.children.map((child) => child.value).join("") + : ""; + + // Add the title property to the fileInfo + fileInfo.title = title; +}; +``` + +### Step 2: Indexing the Folder with Computed Fields + +Now, use the `client.indexFolder` method to scan and index the folder containing your Markdown files. Pass the `addTitle` function in the `computedFields` option array to include the computed title in the database. + +```javascript +client.indexFolder(folderPath: "PATH_TO_FOLDER", customConfig: { computedFields: [addTitle] }); +``` + +## Configuring `markdowndb.config.js` - Implement computed fields to dynamically calculate values based on specified logic or dependencies. - Specify the patterns for including or excluding files in MarkdownDB. -## Example Configuration +### Example Configuration Here's an example `markdowndb.config.js` with custom configurations: ```javascript export default { - customFields: [ + computedFields: [ (fileInfo, ast) => { // Your custom logic here }, diff --git a/src/lib/CustomConfig.ts b/src/lib/CustomConfig.ts new file mode 100644 index 0000000..a92ae92 --- /dev/null +++ b/src/lib/CustomConfig.ts @@ -0,0 +1,6 @@ +import { FileInfo } from "./process.js"; +import { Root } from "remark-parse/lib/index.js"; + +export interface CustomConfig { + computedFields?: ((fileInfo: FileInfo, ast: Root) => any)[]; +} diff --git a/src/lib/indexFolder.ts b/src/lib/indexFolder.ts index 6b28241..7426b13 100644 --- a/src/lib/indexFolder.ts +++ b/src/lib/indexFolder.ts @@ -1,9 +1,11 @@ +import { CustomConfig } from "./CustomConfig.js"; import { FileInfo, processFile } from "./process.js"; import { recursiveWalkDir } from "./recursiveWalkDir.js"; export function indexFolder( folderPath: string, pathToUrlResolver: (filePath: string) => string, + config: CustomConfig, ignorePatterns?: RegExp[] ) { const filePathsToIndex = recursiveWalkDir(folderPath); @@ -11,12 +13,14 @@ export function indexFolder( shouldIncludeFile(filePath, ignorePatterns) ); const files: FileInfo[] = []; + const computedFields = config.computedFields || []; for (const filePath of filteredFilePathsToIndex) { const fileObject = processFile( folderPath, filePath, pathToUrlResolver, - filePathsToIndex + filePathsToIndex, + computedFields ); files.push(fileObject); } diff --git a/src/lib/markdowndb.ts b/src/lib/markdowndb.ts index a759467..d49d698 100644 --- a/src/lib/markdowndb.ts +++ b/src/lib/markdowndb.ts @@ -12,6 +12,7 @@ import { getUniqueValues, } from "./databaseUtils.js"; import fs from "fs"; +import { CustomConfig } from "./CustomConfig.js"; const defaultFilePathToUrl = (filePath: string) => { let url = filePath @@ -72,16 +73,19 @@ export class MarkdownDB { // TODO support glob patterns ignorePatterns = [], pathToUrlResolver = defaultFilePathToUrl, + customConfig = {}, }: { folderPath: string; ignorePatterns?: RegExp[]; pathToUrlResolver?: (filePath: string) => string; + customConfig?: CustomConfig; }) { await resetDatabaseTables(this.db); const fileObjects = indexFolder( folderPath, pathToUrlResolver, + customConfig, ignorePatterns ); const filesToInsert = fileObjects.map(mapFileToInsert); diff --git a/src/lib/parseFile.ts b/src/lib/parseFile.ts index 953f7e8..fe207b3 100644 --- a/src/lib/parseFile.ts +++ b/src/lib/parseFile.ts @@ -28,6 +28,7 @@ export function parseFile(source: string, options?: ParsingOptions) { metadata.tasks = tasks; return { + ast, metadata, links, }; diff --git a/src/lib/process.ts b/src/lib/process.ts index 6a8f208..b30b8f2 100644 --- a/src/lib/process.ts +++ b/src/lib/process.ts @@ -4,6 +4,7 @@ import path from "path"; import { File } from "./schema.js"; import { WikiLink, parseFile } from "./parseFile.js"; +import { Root } from "remark-parse/lib/index.js"; export interface FileInfo extends File { tags: string[]; @@ -16,7 +17,8 @@ export function processFile( rootFolder: string, filePath: string, pathToUrlResolver: (filePath: string) => string, - filePathsToIndex: string[] + filePathsToIndex: string[], + computedFields: ((fileInfo: FileInfo, ast: Root) => any)[] ) { // Remove rootFolder from filePath const relativePath = path.relative(rootFolder, filePath); @@ -52,7 +54,7 @@ export function processFile( flag: "r", }); - const { metadata, links } = parseFile(source, { + const { ast, metadata, links } = parseFile(source, { from: relativePath, permalinks: filePathsToIndex, }); @@ -66,6 +68,10 @@ export function processFile( const tags = metadata?.tags || []; fileInfo.tags = tags; + for (let index = 0; index < computedFields.length; index++) { + const customFieldFunction = computedFields[index]; + customFieldFunction(fileInfo, ast); + } return fileInfo; } diff --git a/src/tests/process.spec.ts b/src/tests/process.spec.ts index 43fd910..6478294 100644 --- a/src/tests/process.spec.ts +++ b/src/tests/process.spec.ts @@ -11,6 +11,7 @@ describe("Can parse a file and get file info", () => { pathToContentFixture, fullPath, (filePath) => filePath, + [], [] );