Documentation improvements and restructuring

pkgs/pgflow.dev/astro.config.mjs

@@ -1,6 +1,7 @@
 // @ts-check
 import { defineConfig } from 'astro/config';
 import starlight from '@astrojs/starlight';
+import starlightLinksValidator from 'starlight-links-validator';
 
 const GITHUB_REPO_URL = 'https://github.com/pgflow-dev/pgflow';
 
@@ -9,6 +10,7 @@ export default defineConfig({
   site: 'https://pgflow.dev',
   integrations: [
     starlight({
+      plugins: [starlightLinksValidator()],
       title: 'pgflow',
       editLink: {
         baseUrl: `${GITHUB_REPO_URL}/edit/main/pkgs/pgflow.dev/`,
@@ -25,15 +27,20 @@ export default defineConfig({
           items: [
             { label: 'How it works?', slug: 'edge-worker/how-it-works' },
             {
-              label: 'Prepare environment',
-              slug: 'edge-worker/prepare-environment',
+              label: 'Getting Started',
+              items: [
+                {
+                  label: 'Install Edge Worker',
+                  slug: 'edge-worker/install-edge-worker',
+                },
+                {
+                  label: 'Create your first worker',
+                  slug: 'edge-worker/create-first-worker',
+                },
+                { label: 'Configuration', slug: 'edge-worker/configuration' },
+                { label: 'Observability', slug: 'edge-worker/observability' },
+              ],
             },
-            {
-              label: 'Create your first worker',
-              slug: 'edge-worker/create-first-worker',
-            },
-            { label: 'Configuration', slug: 'edge-worker/configuration' },
-            { label: 'Observability', slug: 'edge-worker/observability' },
             { label: '⚠️ Project Status', slug: 'edge-worker/project-status' },
           ],
         },

pkgs/pgflow.dev/package.json

@@ -5,8 +5,7 @@
   "scripts": {
     "dev": "astro dev",
     "start": "astro dev",
-    "copy-migrations": "cp ../edge-worker/migrations/000_edge_worker_initial.sql public/edge-worker/",
-    "build": "npm run copy-migrations && astro check && astro build",
+    "build": "astro check && astro build",
     "preview": "astro preview",
     "astro": "astro"
   },
@@ -16,6 +15,7 @@
     "@vercel/analytics": "^1.4.1",
     "astro": "^5.1.7",
     "sharp": "^0.32.5",
+    "starlight-links-validator": "^0.14.1",
     "typescript": "^5.6.3"
   },
   "devDependencies": {

pkgs/pgflow.dev/src/components/NotProductionReady.astro

@@ -3,6 +3,6 @@ import { Aside } from "@astrojs/starlight/components";
 ---
 
 <Aside type="danger" title="Not production ready">
-	<strong>Edge Worker</strong> is currently in advanced proof of concept stage and not yet production ready.
+	<strong>Edge Worker</strong> is currently in <strong>Alpha</strong> stage and not yet production ready.
 	See <a href="/edge-worker/project-status">Project Status</a> page for more details.
 </Aside>

pkgs/pgflow.dev/src/content/docs/edge-worker/configuration.mdx

@@ -1,7 +1,7 @@
 ---
 title: Configuration
 prev:
-  link: /edge-worker/prepare-environment
+  link: /edge-worker/install-edge-worker
 next:
   link: /edge-worker/observability
 ---
@@ -11,16 +11,23 @@ import { Aside, Steps } from "@astrojs/starlight/components";
 You can pass an optional configuration object as the second argument to `EdgeWorker.start()`
 to tweak the worker's behavior.
 
-<Aside title="Zero configuration needed" type="tip">
-All configuration options are optional and have good defaults.
+<Aside title="Zero Configuration" type="tip">
+Edge Worker comes with sensible defaults for all configuration options.
 
-You only need to provide the handler function:
+You only need to provide a handler function:
 
 ```ts
 EdgeWorker.start(console.log);
 ```
 </Aside>
 
+:::caution[Configuration Validation]
+Currently, configuration options are not validated at runtime. Please double-check your settings carefully.
+
+Runtime validation is planned as part of the upcoming [configuration system improvements](/edge-worker/project-status#planned-architecture-improvements).
+:::
+
+
 ### Default configuration
 
 ```typescript
@@ -44,142 +51,152 @@ EdgeWorker.start(handler, {
   retryDelay: 5,
 
   // how many times to retry a failed job
-  retryLimit: 0,
+  retryLimit: 5,
 
   // how long a job is invisible after reading
   // if not successful, will reappear after this time
   visibilityTimeout: 3,
 });
 ```
 
-## Queue and polling
+## Queue configuration
+
+:::tip[You don't need to create queues manually]
+The worker will automatically create a logged, non-partitioned queue
+with configured name during startup if it does not exist.
+:::
 
 ### `queueName`
 **Type:** `string`
 **Default:** `'tasks'`
 
 The name of the PGMQ queue to listen to for messages.
 
-<Aside title="You don't need to create queues manually" type="tip">
-The worker will automatically create a logged, non-partitioned queue
-with configured name during startup if it does not exist.
-</Aside>
-
 ```ts
 EdgeWorker.start(handler, {
   queueName: 'my_custom_queue'
 });
 ```
 
-### `maxPollSeconds`
+## Message Processing
+
+### `visibilityTimeout`
 **Type:** `number`
-**Default:** `5`
+**Default:** `3`
 
-Amount of seconds to wait for a message to be available in the queue.
+The duration (in seconds) that a message remains invisible to other consumers while being processed.
+
+<Aside title="Message Visibility and Processing" type="caution">
+Keep `visibilityTimeout` higher than your task's maximum processing time. Here's why:
+- When a worker picks up a message, it becomes invisible for `visibilityTimeout` seconds
+- If processing takes longer than `visibilityTimeout`, the message becomes visible again
+- Other workers can then pick up and process the same message
+- This leads to duplicate processing of the same task
+- For example: with `visibilityTimeout: 3` and a task that takes 5 seconds, the message could be processed twice
+</Aside>
 
 ```ts
 EdgeWorker.start(handler, {
-  maxPollSeconds: 5 // Long-poll for 5 seconds waiting for a message
+  visibilityTimeout: 5 // message will re-appear in queue after 5 seconds if not processed
 });
 ```
 
-### `pollIntervalMs`
+### `maxConcurrent`
 **Type:** `number`
-**Default:** `200`
-
-The interval for in-database polling that `pgmq.read_with_poll` does.
-Most of the time, you don't need to change this value.
-
-<Aside type="caution">
-  This value should be lower than the `maxPollSeconds` value.
-</Aside>
+**Default:** `10`
 
+This option limits concurrency - the maximum number of messages that can be processed at the same time.
+Increase for IO-heavy tasks (network or db calls), decrease for CPU-heavy tasks.
 
 ```ts
 EdgeWorker.start(handler, {
-  pollIntervalMs: 300 // Poll every 300ms
+  maxConcurrent: 10 // Process up to 10 messages at once
 });
 ```
 
-## Retries
-
-### `retryDelay`
+### `maxPgConnections`
 **Type:** `number`
-**Default:** `5`
-
-Amount of seconds to wait between retry attempts.
+**Default:** `4`
 
-<Aside type="caution">
-  This value should be greater than the visibility timeout.
-</Aside>
+Maximum number of concurrent database connections.
+Increase for IO-heavy tasks (network or database operations), decrease for CPU-heavy tasks.
 
 ```ts
 EdgeWorker.start(handler, {
-  retryDelay: 5 // Wait 5 seconds between retries
+  maxPgConnections: 10 // Use up to 10 connections to the database
 });
 ```
 
+## Polling Behavior
 
-### `retryLimit`
+### `maxPollSeconds`
 **Type:** `number`
 **Default:** `5`
 
-Maximum number of retry attempts for failed message processing before marking the message as dead.
+Amount of seconds to wait for a message to be available in the queue.
 
-Set to `0` to disable retries.
+:::caution[Polling and Worker Health]
+Keep `maxPollSeconds` at 5 seconds or lower. Here's why:
+- Heartbeats are sent once per main loop iteration
+- Each iteration can take up to `maxPollSeconds` to complete
+- Workers are considered inactive after 6 seconds without a heartbeat
+- Therefore, `maxPollSeconds` must be lower than 5 seconds to account for any additional delays
+:::
 
 ```ts
 EdgeWorker.start(handler, {
-  retryLimit: 0 // retries are disabled
+  maxPollSeconds: 5 // Long-poll for 5 seconds waiting for a message
 });
 ```
 
-### `visibilityTimeout`
+### `pollIntervalMs`
 **Type:** `number`
-**Default:** `3`
+**Default:** `200`
 
-The duration in seconds that a message is hidden from other consumers while being processed.
+The interval (in milliseconds) between database polling attempts by `pgmq.read_with_poll`.
+The default value is suitable for most use cases.
 
-<Aside title="Timeouts" type="caution">
-Make sure to set it bigger than max time your task can take to complete, otherwise
-it will be processed multiple times.
+<Aside type="caution" title="Polling Intervals">
+Keep `pollIntervalMs` lower than `maxPollSeconds * 1000`. Here's why:
+- The worker polls the database every `pollIntervalMs` milliseconds
+- This polling continues until `maxPollSeconds` is reached
+- If `pollIntervalMs` is too high, the worker might exit after just one poll
+- For example: with `maxPollSeconds: 5` and `pollIntervalMs: 6000`, only one poll would occur
 </Aside>
 
-<Aside title="Work-in-Progress" type="danger">
-This option is probably subject to be refactored to `timeoutMs` that will
-affect timeout for the tasks (wip) and visibility timeout.
-**Do not rely on this option heavily for now!**
-</Aside>
 
 ```ts
 EdgeWorker.start(handler, {
-  visibilityTimeout: 5 // message will re-appear in queue after 5 seconds if not processed
+  pollIntervalMs: 300 // Poll every 300ms
 });
 ```
 
-## Concurrency
+## Retries
 
-### `maxConcurrent`
+### `retryDelay`
 **Type:** `number`
-**Default:** `10`
+**Default:** `5`
 
-This option limits concurrency - the maximum number of messages that can be processed at the same time.
-Increase for IO-heavy tasks (network or db calls), decrease for CPU-heavy tasks.
+Amount of seconds to wait between retry attempts.
 
 ```ts
 EdgeWorker.start(handler, {
-  maxConcurrent: 10 // Process up to 10 messages at once
+  retryDelay: 5 // Wait 5 seconds between retries
 });
 ```
 
-### `maxPgConnections`
+
+### `retryLimit`
 **Type:** `number`
-**Default:** `4`
+**Default:** `5`
 
-This option limits the number of connections to the database.
-Increase for IO-heavy tasks (network or db calls), decrease for CPU-heavy tasks.
+Maximum number of retry attempts for failed message processing before marking the message as dead.
+
+Set to `0` to disable retries.
 
 ```ts
 EdgeWorker.start(handler, {
-  maxPgConnections: 10 // Use up to 10 connections to the database
+  retryLimit: 0 // retries are disabled
 });
+```
+