Various fixes

README.md

@@ -28,7 +28,7 @@ _For detailed documentation, visit [pup.56k.guru](https://pup.56k.guru)._
 To install Pup, make sure you run the latest version of Deno (`deno upgrade`), then open your terminal and execute the following command:
 
 ```bash
-deno run -Ar jsr:@pup/[email protected].39 setup --channel prerelease
+deno run -Ar jsr:@pup/[email protected].40 setup --channel prerelease
 ```
 
 This command downloads the latest version of Pup and installs it on your system. The `--channel prerelease` option is included as there is no stable version of Pup yet. Read more abour release

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@pup/pup",
-  "version": "1.0.0-rc.39",
+  "version": "1.0.0-rc.40",
 
   "exports": {
     ".": "./pup.ts",
@@ -38,7 +38,7 @@
     "@cross/env": "jsr:@cross/env@~1.0.2",
     "@cross/fs": "jsr:@cross/fs@~0.1.11",
     "@cross/jwt": "jsr:@cross/jwt@~0.4.7",
-    "@cross/kv": "jsr:@cross/kv@^0.13.2",
+    "@cross/kv": "jsr:@cross/kv@^0.15.6",
     "@cross/runtime": "jsr:@cross/runtime@~1.0.0",
     "@cross/service": "jsr:@cross/service@~1.0.3",
     "@cross/test": "jsr:@cross/test@~0.0.9",

docs/src/_data.json

@@ -6,7 +6,7 @@
     "description": "Universal Process Manager"
   },
   "substitute": {
-    "$PUP_VERSION": "1.0.0-rc.39"
+    "$PUP_VERSION": "1.0.0-rc.40"
   },
   "top_links": [
     {

docs/src/changelog.md

@@ -12,7 +12,7 @@ All notable changes to this project will be documented in this section.
 ## [1.0.0-rc.40] - Unreleased
 
 - fix(core): Replace `Deno.Kv` with `@cross/kv` for cross-runtime compatibility, more compact logs and avoiding `--unstable`
-- fix(core): Make Pup work in Node and Bun
+- fix(core): Internal changes to support Node and Bun
 
 ## [1.0.0-rc.39] - 2024-05-04
 

docs/src/examples/basic-webinterface/pup.jsonc

@@ -50,7 +50,7 @@
   "plugins": [
     {
       // Use full uri to plugin, e.g. jsr:@pup/plugin-web-interface
-      "url": "jsr:@pup/plugin-web-interface",
+      "url": "jsr:@pup/plugin-web-interface@^2.0.2",
       "options": {
         "port": 8002
       }

lib/cli/main.ts

@@ -429,22 +429,13 @@ async function main() {
    * Base argument: logs
    */
   if (baseArgument === "logs") {
-    const logStore = `${await toPersistentPath(configFile as string)}/.main.log`
+    const logStore = `${await toPersistentPath(configFile as string)}/.main.log.ckvdb`
     const logger = new Logger(configuration!.logger || {}, logStore)
     await logger.init()
     const startTimestamp = checkedArgs.get("start") ? new Date(Date.parse(checkedArgs.get("start")!)).getTime() : undefined
     const endTimestamp = checkedArgs.get("end") ? new Date(Date.parse(checkedArgs.get("end")!)).getTime() : undefined
     const numberOfRows = checkedArgs.get("n") ? parseInt(checkedArgs.get("n")!, 10) : undefined
-    let logs = await logger.getLogContents(checkedArgs.get("id"), startTimestamp, endTimestamp)
-    logs = logs.filter((log) => {
-      const { processId, severity } = log
-      const severityFilter = !checkedArgs.get("severity") || checkedArgs.get("severity") === "" || checkedArgs.get("severity")!.toLowerCase() === severity.toLowerCase()
-      const processFilter = !checkedArgs.get("id") || checkedArgs.get("id") === "" || checkedArgs.get("id")!.toLowerCase() === processId.toLowerCase()
-      return severityFilter && processFilter
-    })
-    if (numberOfRows) {
-      logs = logs.slice(-numberOfRows)
-    }
+    const logs = await logger.getLogContents((!checkedArgs.get("id") || checkedArgs.get("id") === "") ? undefined : checkedArgs.get("id")!.toLowerCase(), startTimestamp, endTimestamp,  (!checkedArgs.get("severity") || checkedArgs.get("severity") === "") ? undefined : checkedArgs.get("severity")!.toLowerCase(), numberOfRows)
     if (logs && logs.length > 0) {
       const logWithColors = configuration!.logger?.colors ?? true
       for (const log of logs) {

lib/core/logger.ts

@@ -50,7 +50,7 @@ class Logger {
   }
 
   // Prepare log event selector
-  private prepareSelector(processId?: string, startTimeStamp?: number, endTimeStamp?: number): KVQuery {
+  private prepareSelector(processId?: string, startTimeStamp?: number, endTimeStamp?: number, severity?: string): KVQuery {
     const key: KVQuery = ["logs_by_time"]
     if (startTimeStamp || endTimeStamp) {
       const rangeSelector: KVQueryRange = {}
@@ -61,44 +61,46 @@ class Logger {
         rangeSelector.to = endTimeStamp
       }
       key.push(rangeSelector)
-    } else if (processId) {
+    } else if (processId || severity) {
       key.push({})
     }
     if (processId) {
       key.push(processId)
+    } else {
+      key.push({})
+    }
+    if (severity) {
+      key.push(severity)
     }
     return key
   }
 
   // Fetch logs from store
   private async fetchLogsFromStore(selector: KVQuery, nRows?: number): Promise<LogEventData[]> {
-    const result = await this.kv.listAll(selector)
-    const resultArray: LogEventData[] = []
-    for await (const res of result) {
-      resultArray.push(res.data as LogEventData)
-    }
-    if (nRows) {
-      const spliceNumber = Math.max(0, resultArray.length - nRows)
-      resultArray.splice(0, spliceNumber)
+    const resultArray: LogEventData[] = [];
+
+    // Use the generator for efficient iteration
+    for await (const { data } of this.kv.iterate<LogEventData>(selector, nRows, true)) {
+      resultArray.unshift(data);
     }
-    return resultArray
+    return resultArray;
   }
 
-  public async getLogContents(processId?: string, startTimeStamp?: number, endTimeStamp?: number, nRows?: number): Promise<LogEventData[]> {
-    const selector = this.prepareSelector(processId, startTimeStamp, endTimeStamp)
+  public async getLogContents(processId?: string, startTimeStamp?: number, endTimeStamp?: number, severity?: string, nRows?: number): Promise<LogEventData[]> {
+    const selector = this.prepareSelector(processId, startTimeStamp, endTimeStamp, severity)
     return await this.fetchLogsFromStore(selector, nRows)
   }
 
   public async getLogsByProcess(processId: string, nRows?: number): Promise<LogEventData[]> {
-    return await this.getLogContents(processId, undefined, undefined, nRows)
+    return await this.getLogContents(processId, undefined, undefined, undefined, nRows)
   }
 
   public async getLogsByTime(startTimeStamp: number, endTimeStamp: number, nRows?: number): Promise<LogEventData[]> {
-    return await this.getLogContents(undefined, startTimeStamp, endTimeStamp, nRows)
+    return await this.getLogContents(undefined, startTimeStamp, endTimeStamp, undefined, nRows)
   }
 
   public async getLogsByProcessAndTime(processId: string, startTimeStamp: number, endTimeStamp: number, nRows?: number): Promise<LogEventData[]> {
-    return await this.getLogContents(processId, startTimeStamp, endTimeStamp, nRows)
+    return await this.getLogContents(processId, startTimeStamp, endTimeStamp, undefined, nRows)
   }
 
   private async internalLog(severity: string, category: string, text: string, process?: ProcessConfiguration, timeStamp?: number) {
@@ -119,7 +121,8 @@ class Logger {
           processId: initiator,
           timeStamp: timeStamp,
         }
-        await this.kv.set(["logs_by_time", timeStamp, initiator], logObj)
+        // Append a random uuid to the key, in case two logs should arrive at the same time
+        await this.kv.set<LogEventData>(["logs_by_time", timeStamp, initiator, severity, crypto.randomUUID()], logObj)
       } catch (e) {
         console.error("Error while writing to log store", e)
       }

lib/core/pup.ts

@@ -82,7 +82,7 @@ class Pup {
 
       this.persistentStoragePath = persistentStoragePath
 
-      statusFile = `${this.persistentStoragePath}/.main.log.ckvdb` // Cross/KV store
+      statusFile = `${this.persistentStoragePath}/.main.status.ckvdb` // Cross/KV store
       secretFile = `${this.persistentStoragePath}/.main.secret` // Plain text file containing the JWT secret for the rest api
       portFile = `${this.temporaryStoragePath}/.main.port` // Plain text file containing the port number for the API
       logStore = `${this.persistentStoragePath}/.main.log.ckvdb` // Cross/KV store

lib/core/status.ts

@@ -42,7 +42,7 @@ class Status {
    */
   public async writeToStore(applicationState: ApiApplicationState) {
     try {
-      const kv = new KV({ autoSync: false })
+      const kv = new KV({ autoSync: false, disableIndex: true })
       await kv.open(this.storeName!)
 
       // Initialize lastWrite if it's not set
@@ -70,7 +70,7 @@ class Status {
    */
   public async cleanup() {
     try {
-      const kv = new KV({ autoSync: false })
+      const kv = new KV({ autoSync: false, disableIndex: true })
       await kv.open(this.storeName!)
       await kv.delete(["last_application_state"])
       await kv.close()