doc updates

Author: moshloop

canary-checker/docs/reference/1-gcs-database-backup.mdx renamed to canary-checker/docs/reference/3-gcs-database-backup.mdx

@@ -1,5 +1,6 @@
 ---
 title: GCS Database Backup
+sidebar_class_name: beta
 ---
 
 # <Icon name="database2" /> Google Cloud SQL Backups
@@ -42,10 +43,10 @@ Durations are strings with an optional fraction and unit e.g. `300ms`, `1.5h` or
 
 | Field          | Description                                                                                                                   | Scheme                                                 | Required |
 | -------------- | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ | -------- |
-| `project`      | GCP project\ name                                                                                                             | _string_                                               | Yes      |
+| `project`      | GCP project name                                                                                                             | _string_                                               | Yes      |
 | `instance`     | Google CloudSQL instance name                                                                                                 | _string_                                               | Yes      |
 | **Connection** |                                                                                                                               |                                                        |          |
-| `connection`   | Path of an existing connection e.g. `connection://aws/instance`/. Mutually exclusive with `credentials` <br /> <Commercial /> | <CommonLink to="connection">_Connections_</CommonLink> |          |
+| `connection`   | Path of an existing connection e.g. `connection://gcp`/. Mutually exclusive with `credentials` <br /> <Commercial /> | <CommonLink to="connection">_Connections_</CommonLink> |          |
 | `credentials`  | GCP Access Token File. <SkipOSS>Mutually exclusive with `connection`</SkipOSS>                                                | <CommonLink to="authentication">_EnvVar_</CommonLink>  | Yes      |
 
 ### Connecting to GCP
@@ -72,7 +73,7 @@ spec:
 
 3. `accessKey` and `secretKey` <CommonLink to="authentication">_EnvVar_</CommonLink> with the credentials stored in a secret.
 
-```yaml title="aws.yaml"
+```yaml title="gcs-database-backup.yaml"
 apiVersion: canaries.flanksource.com/v1
 kind: Canary
 metadata:

canary-checker/docs/scripting/gotemplate.md

@@ -31,7 +31,7 @@ To send `{{ .secret }}` as it is do this
 {{`{{ .secret }}`}}
 ```
 
-## Encoding
+## base64
 
 ### Encode
 
@@ -46,8 +46,6 @@ Encode data as a Base64 string. Specifically, this uses the standard Base64 enco
 
 Decode a Base64 string. This supports both standard ([RFC4648 §4](https://tools.ietf.org/html/rfc4648#section-4)) and URL-safe ([RFC4648 §5](https://tools.ietf.org/html/rfc4648#section-5)) encodings.
 
-This function outputs the data as a string, so it may not be appropriate for decoding binary data. Use [`base64.DecodeBytes`](#base64.DecodeBytes)
-for binary data.
 
 ```go
 {{ base64.Decode "aGVsbG8gd29ybGQ=" }} // hello world
@@ -56,7 +54,6 @@ for binary data.
 
 
 :::note
-
 For the functions that return an array, a Go `[]interface{}` is returned, regardless of whether or not the input was a different type.
 :::
 
@@ -417,7 +414,7 @@ The value in decimal is {{ $val }}
 
 ### ParseFloat
 
-_**Note:**_ See [`conv.ToFloat`](#tofloat) instead for a simpler and more flexible variant of this function.
+_**Note:**_ See [`conv.ToFloat`](#tofloat64) instead for a simpler and more flexible variant of this function.
 
 Parses a string as an float64 for later use. Equivalent to [strconv.ParseFloat](https://golang.org/pkg/strconv/#ParseFloat)
 
@@ -1803,7 +1800,7 @@ Parses a timestamp defined by the given layout. This wraps [`time.Parse`](https:
 A number of pre-defined layouts are provided as constants, defined
 [here](https://golang.org/pkg/time/#pkg-constants).
 
-Just like [`time.Now`](#time-now), this is usually used in conjunction with other functions.
+Just like [`time.Now`](#now), this is usually used in conjunction with other functions.
 
 _Note: In the absence of a time zone indicator, `time.Parse` returns a time in UTC._
 

common/src/components/Fields.jsx

@@ -9,17 +9,22 @@ const schemes = {
   "EnvVar": "[EnvVar](/reference/env-var)",
   "[]EnvVar": "[[]EnvVar](/reference/env-var)",
   "CEL": "[CEL](/reference/scripting/cel)",
+  "Javascript": "[Javascript](/reference/scripting/javascript)",
+  "Gotemplate": "[Go Template](/reference/scripting/gotemplate)",
   "Duration": "[Duration](/reference/types#duration)",
+  "JSONPathOrString": "`string` or [JSONPath](https://jsonpath.com/)",
+  "[]JSONPathOrString": "`[]string` or [[]JSONPath](https://jsonpath.com/)",
+  "JSONPath": "[JSONPath](https://jsonpath.com/)",
   "Size": "[Size](/reference/types#size)",
   "Agent": "[Agent](/reference/types#agent)",
   "ResourceSelector": "[ResourceSelector](/reference/resource-selector)",
   "Connection": "[Connection](/reference/connections)",
   "string": "`string`",
   "Icon": "[Icon](/reference/types#icon)",
   "bool": "`boolean`",
-  "NotificationURL": "[url](/reference/notifications/channels)",
-  "NotificationConnection": "[Connection](/reference/connection)",
-  "NotificationProperties": "[map[string]string](/reference/notifications/channels#properties)",
+  "NotificationURL": "[Notification](/reference/notifications)",
+  "NotificationConnection": "[Connection](/reference/connections)",
+  "NotificationProperties": "[map[string]string](/reference/notifications#properties)",
 }
 export default function Fields({ common = [], rows = [], oneOf, anyOf, connection }) {
 
@@ -52,7 +57,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio
     rows = rows.concat([
       {
         field: oss ? null : "connection",
-        description: "The connection to use, mutually exclusive with `username` and `password`",
+        description: "The connection url to use, mutually exclusive with `username` and `password`",
         scheme: "Connection",
       },
       {
@@ -73,11 +78,38 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio
         scheme: "EnvVar",
       }
     ])
+  }
+
+  if (connection == "sql") {
+    rows = rows.concat([
+      {
+        field: oss ? null : "connection",
+        description: "The connection url or name to use`",
+        scheme: "Connection",
+      },
+      {
+        field: !oss && rows.filter(i => i.field == "url").length == 0 ? "url" : null,
+        description: "If `connection` is specified and it also includes a `url`, this field will take precedence",
+        scheme: "string",
+      },
+      {
+        field: oss && rows.filter(i => i.field == "url").length == 0 ? "url" : null,
+        scheme: "string",
+      },
+      {
+        field: "auth.username",
+        scheme: "EnvVar",
+      },
+      {
+        field: "auth.password",
+        scheme: "EnvVar",
+      }
+    ])
   } else if (connection == "git") {
     rows = rows.concat([
       {
         field: oss ? null : "connection",
-        description: "The connection to use, mutually exclusive with `username` and `password`",
+        description: "The connection url to use, mutually exclusive with `username` and `password`",
         scheme: "Connection",
       },
       {
@@ -106,7 +138,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio
     rows = rows.concat([
       {
         field: oss ? null : "connection",
-        description: "The connection to use, mutually exclusive with `accessKey` and `secretKey`",
+        description: "The connection url to use, mutually exclusive with `accessKey` and `secretKey`",
         scheme: "Connection",
       },
       {
@@ -136,7 +168,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio
     rows = rows.concat([
       {
         field: oss ? null : "connection",
-        description: "The connection to use, mutually exclusive with `credentials`",
+        description: "The connection url to use, mutually exclusive with `credentials`",
         scheme: "Connection",
       },
       {
@@ -149,7 +181,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio
     rows = rows.concat([
       {
         field: oss ? null : "connection",
-        description: "The connection to use, mutually exclusive with `host` and `port`",
+        description: "The connection url to use, mutually exclusive with `host` and `port`",
         scheme: "Connection",
       },
       {
@@ -175,7 +207,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio
     rows = rows.concat([
       {
         field: oss ? null : "connection",
-        description: "The connection to use, mutually exclusive with `host`, `share`, and `credentials`",
+        description: "The connection url to use, mutually exclusive with `host`, `share`, and `credentials`",
         scheme: "Connection",
       },
       {
@@ -212,7 +244,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio
     rows = rows.concat([
       {
         field: oss ? null : "connection",
-        description: "The connection to use, mutually exclusive with `tenantId`, `subscriptionId`, `clientId`, and `clientSecret`",
+        description: "The connection url to use, mutually exclusive with `tenantId`, `subscriptionId`, `clientId`, and `clientSecret`",
         scheme: "Connection",
       },
       {
@@ -273,11 +305,17 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio
         </blockquote>
       }
       {
-        oneOf &&
+        oneOf && oneOf.length == 2 &&
         <blockquote style={{ borderLeft: "2px solid var(--ifm-color-warning-light)" }}>
           <p>You must specify <code>{oneOf[0]}</code> or <code>{oneOf[1]}</code> but not both</p>
         </blockquote>
       }
+      {
+        oneOf && oneOf.length == 3 &&
+        <blockquote style={{ borderLeft: "2px solid var(--ifm-color-warning-light)" }}>
+          <p>You must specify one of <code>{oneOf[0]}</code>, <code>{oneOf[1]}</code> or <code>{oneOf[2]}</code></p>
+        </blockquote>
+      }
     </>
   )
 }

common/src/components/Scraper.jsx

@@ -0,0 +1,93 @@
+import React from 'react'
+import Admonition from '@theme/Admonition'
+import useDocusaurusContext from '@docusaurus/useDocusaurusContext'
+import Link from '@docusaurus/Link'
+import ReactMarkdown from 'react-markdown'
+import clsx from 'clsx'
+import Fields from './Fields'
+
+const customScraperFields = [
+  {
+    "field": "id",
+    "description": "A deterministic or natural id for the resource",
+    "scheme": "JSONPathOrString",
+    "required": true
+  },
+  {
+    "field": "name",
+    default: "`id`",
+    "scheme": "JSONPathOrString"
+  },
+  {
+    "field": "items",
+    "description": "Extract multiple config items from this array",
+    "scheme": "JSONPath"
+  },
+  {
+    "field": "type",
+    "description": "e.g. `File::Host`, `File::Tomcat`, `File::Pom`",
+    "scheme": "JSONPathOrString",
+    "required": "true"
+  },
+  {
+    "field": "class",
+    "scheme": "JSONPathOrString"
+  },
+  {
+    "field": "format",
+    "description": "Format of config item e.g. `xml`, `properties`",
+    "default": "`JSON`",
+    "scheme": "string"
+  },
+  {
+    "field": "timestampFormat",
+    "description": "Format to parse timestamps in `createFields` and `deletedFields`",
+    "default": "RFC3339",
+    "scheme": "Go time format"
+  },
+  {
+    "field": "createFields",
+    "description": "Identify the created time for a resource (if different to scrape time). If multiple fields are specified, the first non-empty value will be used",
+    "scheme": "[]JSONPathOrString"
+  },
+  {
+    "field": "deleteFields",
+    "description": "Identify when a config item was deleted. If multiple fields are specified, the first non-empty value will be used",
+    "scheme": "[]JSONPathOrString"
+  },
+]
+
+const commonsRows = [
+
+  {
+    "field": "ignore",
+    "description": "Fields to ignore and strip out of the config",
+    "scheme": "[]JSONPath"
+  },
+  {
+    "field": "properties",
+    "description": "Custom templatable properties for the scraped config items.",
+    "scheme": "[`[]ConfigProperty`](/reference/config-db/properties)"
+  },
+  {
+    "field": "tags",
+    "description": "Additional tags ",
+    "scheme": "map[string]string"
+  },
+]
+
+export function Scraper({ name, edition, rows, ...props }) {
+
+  const { siteConfig } = useDocusaurusContext()
+
+
+  return <Fields rows={rows} common={commonsRows} {...props} />
+
+}
+
+export function CustomScraper({ name, edition, rows, ...props }) {
+
+  const { siteConfig } = useDocusaurusContext()
+  return <Fields rows={rows} common={commonsRows.concat(customScraperFields)} {...props} />
+
+}

common/src/theme/MDXComponents/index.js

@@ -29,6 +29,9 @@ import { FullImage } from '@site/src/components/Badges'
 import { CommonLink } from '@site/src/components/Link'
 import { ConfigTransform } from '@site/src/components/ConfigTransform'
 import { CustomField } from '@site/src/components/Badges'
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import { Scraper, CustomScraper } from '@site/src/components/Scraper'
 
 const MDXComponents = {
   Icon: Icon,
@@ -48,6 +51,10 @@ const MDXComponents = {
   head: MDXHead,
   code: MDXCode,
   a: MDXA,
+  Tabs: Tabs,
+  Scraper: Scraper,
+  CustomScraper: CustomScraper,
+  TabItem: TabItem,
   pre: MDXPre,
   details: Details,
   Screenshot: Screenshot,

mission-control/docs/config-db/concepts/transform.md

@@ -2,7 +2,7 @@
 title: Transform
 ---
 
-Transformations allows you to transform the scraped config items before they are saved, common use cases include:
+Transformations allows you to modify scraped config items before they are saved, common use cases include:
 
 - Linking configuration items
 - Removing extraneous or overly verbose fields
@@ -49,7 +49,8 @@ spec:
 
 ### Masking
 
-Masking allows replacing sensitive fields with a hash or static string.
+Masking replaces sensitive fields with a hash or static string. A hash can be used to determine if a field changed without revealing original values.
+
 
 ```yaml title="file-mask-scraper.yaml"
 apiVersion: configs.flanksource.com/v1
@@ -88,10 +89,10 @@ Masks are applied in the order they are specified in the configuration file.
 ## Changes
 
 ### Exclusions
+Some configurations can change frequently and may not be relevant. For example, a `Kubernetes::Node` configuration changes often as pods launched and stopped. From the node's perspective, these image changes are irrelevant.
 
-Some configs can have changes in high volume that may not be relevant. Example: A kubernetes Node config changes frequently as the pods in the cluster update their images. From the node's perspective the image changes are irrelevant.
+This is where exclusions become useful. Here's an example that ignores all image changes in a `Kubernetes::Node` configuration:
 
-This is where exclusions can become handy. Here's an example that ignore all image changes in a kubernetes node config:
 
 ```yaml title="kubernetes-scraper.yaml"
 apiVersion: configs.flanksource.com/v1
@@ -111,7 +112,7 @@ spec:
 
 ### Mapping
 
-When you encounter a diff change, unlike an event based change, it can sometimes appear cryptic. The summary of the change may not immediately indicate what the change is about. For example, the change 'status.images' might not be self-explanatory. To address this issue, we can assign types to these diff changes using mapping.
+When you encounter a diff change, unlike an event-based change, it can sometimes appear unclear. The summary of the change may not immediately indicate its purpose. For example, the change 'status.images' might not be self-explanatory. To clarify this, you can assign types to these diff changes using mapping.
 
 ```yaml title="kubernetes-scraper.yaml"
 apiVersion: configs.flanksource.com/v1
@@ -142,11 +143,11 @@ spec:
 | `filter`  | Selects changes to apply the mapping                         | <CommonLink to="cel">CEL</CommonLink> with [Change Context](/reference/config-db/changes) |
 | `action`  | What action to take on the change, if `delete` then the corresponding config item is marked as deleted | `delete` or `ignore`                                         |
 | `type`    | New change type                                              | `string`                                                     |
-| `summary` | New summary of the change                                    | [Go Template](/reference/scripting/template)                 |
+| `summary` | New summary of the change                                    | <CommonLink to="gotemplate">Go Template</CommonLink> with [Change Context](/reference/config-db/changes)                |
 
 ## Scripting
 
-Scripting allows you to modify the scraped configuration using CEL before saving it to the database. This is useful for data normalization, default value population, sensitive field masking etc.
+Scripting modifies the scraped configuration using CEL before saving it to the database. This process is beneficial for data normalization, default value population, and sensitive field masking.
 
 | Field  | Description             | Scheme                                                       | Context                                                      |
 | ------ | ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |

mission-control/docs/config-db/index.mdx

@@ -1,6 +1,7 @@
 ---
 slug: /config-db
 hide_title: true
+title: Overview
 hide_table_of_contents: true
 pagination_next: playbooks/index
 pagination_prev:
@@ -34,7 +35,7 @@ Under the hood the catalog is powered by [github.com/flanksource/config-db](http
 The catalog is comprised of:
 
 * **Config Items** are individual resources e.g. `Pod`, `EBS`, `IAM Role`, `postgres.conf` scraped from external sources like  [AWS](/config-db/scrapers/aws), [Kubernetes](/config-db/scrapers/kubernetes), [Azure](/config-db/scrapers/azure), [Azure Devops](/config-db/scrapers/azure-devops) , [Github](/config-db/scrapers/github)
-* **Changes** recorded against config items either through automatic change detection (diffs) or from sources like [Cloud Trail](/config-db/scrapers/aws#cloud-trail) or [Kubernetes Events](/config-db/scrapers/kubernetes#events)
+* **Changes** recorded against config items either through automatic change detection (diffs) or from sources like [Cloud Trail](/config-db/scrapers/aws#cloudtrail) or [Kubernetes Events](/config-db/scrapers/kubernetes#events)
 * **Insights** recorded against config items from external sources like [AWS Trusted Advisor](/config-db/scrapers/aws#trusted-advisor) or [Trivy](/config-db/scrapers/trivy)
 * **Relationships** between configuration items