diff --git a/.github/workflows/openapi-pr.yml b/.github/workflows/openapi-pr.yml new file mode 100644 index 0000000000..c43008fc73 --- /dev/null +++ b/.github/workflows/openapi-pr.yml @@ -0,0 +1,44 @@ +# Copyright 2024-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included in +# the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +# file, in accordance with the Business Source License, use of this software +# will be governed by the Apache License, Version 2.0, included in the file +# licenses/APL2.txt. + +name: openapi-pr + +on: + pull_request: + # Only run when we change an API spec + paths: + - 'docs/api/**' + branches: + - 'master' + - 'main' + - 'release/*' + - 'beryllium' + +jobs: + redocly_preview_links: + runs-on: ubuntu-latest + steps: + - name: Find Comment + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Redocly previews + - name: Create or update comment + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + ## Redocly previews + - [Admin API](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/couchbase/sync_gateway/${{ github.event.pull_request.head.sha }}/docs/api/admin.yaml) + - [Public API](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/couchbase/sync_gateway/${{ github.event.pull_request.head.sha }}/docs/api/public.yaml) + - [Metric API](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/couchbase/sync_gateway/${{ github.event.pull_request.head.sha }}/docs/api/metric.yaml) + - [Diagnostic API](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/couchbase/sync_gateway/${{ github.event.pull_request.head.sha }}/docs/api/diagnostic.yaml) + edit-mode: replace diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml index 0a6cd56b20..fec04e3e47 100644 --- a/.github/workflows/openapi.yml +++ b/.github/workflows/openapi.yml @@ -13,9 +13,12 @@ on: # Only run when we change an API spec paths: - 'docs/api/**' - branches: + branches: - 'master' + - 'main' - 'release/*' + - 'feature/*' + - 'beryllium' - 'CBG*' - 'ci-*' - 'api-ci-*' @@ -25,17 +28,28 @@ on: - 'docs/api/**' branches: - 'master' + - 'main' - 'release/*' + - 'beryllium' jobs: api_validation: runs-on: ubuntu-latest name: OpenAPI Validation steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: r7kamura/redocly-problem-matchers@v1 - - uses: mhiew/redoc-lint-github-action@v3 + - uses: mhiew/redoc-lint-github-action@v4 with: args: '--format stylish' env: NO_COLOR: '1' + + yamllint: + name: 'yamllint' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: karancode/yamllint-github-action@master + with: + yamllint_file_or_dir: 'docs/api' diff --git a/.redocly.yaml b/.redocly.yaml index 4e30c99085..3680eaa563 100644 --- a/.redocly.yaml +++ b/.redocly.yaml @@ -11,18 +11,61 @@ apis: admin: root: "./docs/api/admin.yaml" + decorators: + remove-x-internal: on + admin-internal: + root: "./docs/api/admin.yaml" + admin-capella: + root: "./docs/api/admin-capella.yaml" + decorators: + filter-out: + property: x-capella + value: false + remove-x-internal: on public: root: "./docs/api/public.yaml" + decorators: + remove-x-internal: on + public-internal: + root: "./docs/api/public.yaml" + public-capella: + root: "./docs/api/public.yaml" + decorators: + filter-out: + property: x-capella + value: false + info-override: + description: "App Services manages access and synchronization between Couchbase Lite and Couchbase Capella" + plugin/replace-description-capella: on + plugin/replace-server-capella: + serverUrl: 'https://{hostname}:4984' + remove-x-internal: on metric: root: "./docs/api/metric.yaml" + decorators: + remove-x-internal: on + metric-internal: + root: "./docs/api/metric.yaml" + metric-capella: + root: "./docs/api/metric-capella.yaml" + decorators: + filter-out: + property: x-capella + value: false + plugin/excise-rbac-capella: on + plugin/replace-description-capella: on + remove-x-internal: on + +plugins: + - './docs/api/plugins/plugin.js' -lint: - extends: - - minimal - rules: - # disable unnecessary/invalid warnings - operation-2xx-response: off # _blipsync 101 Upgrade ... - operation-summary: off # Optional field - no-ambiguous-paths: off # /{db}/{doc} != /_debug/expvar - no-identical-paths: off # /{db} != /{targetdb} - no-path-trailing-slash: off # Some endpoints require a trailing slash +extends: + - minimal +rules: + # disable unnecessary/invalid warnings + operation-2xx-response: off # _blipsync 101 Upgrade ... + operation-summary: off # Optional field + no-ambiguous-paths: off # /{db}/{doc} != /_debug/expvar + no-identical-paths: off # /{db} != /{targetdb} + no-path-trailing-slash: off # Some endpoints require a trailing slash + security-defined: off # TODO: Denote public and authenticated API endpoints with https://redocly.com/docs/cli/rules/security-defined diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000000..9d5df0a21a --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,17 @@ +# Copyright 2023-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included +# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +# in that file, in accordance with the Business Source License, use of this +# software will be governed by the Apache License, Version 2.0, included in +# the file licenses/APL2.txt. + +--- + +extends: default + +rules: + document-start: disable + line-length: disable + comments: disable + truthy: disable diff --git a/docs/BUILD.md b/docs/BUILD.md index 98624a9839..62b7cff70b 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -1,5 +1,5 @@ -Building with Go Modules +Building with Go Modules ------------------------ Recent versions of Sync Gateway provide support for Go modules, which simplifies the process of building from source. Older versions will require building with pinned dependencies, see below. @@ -45,7 +45,7 @@ $ ./bootstrap.sh After it's complete, you should see a message that says `Bootstrap complete! Run ./build.sh to build and ./test.sh to run tests` -*Note:* if you want to run the bootstrap initialization and start on a particular Sync Gateway commit, you can provide the `-c` flag, eg `./bootstrap.sh -c y0pl33g0r425`. +*Note:* if you want to run the bootstrap initialization and start on a particular Sync Gateway commit, you can provide the `-c` flag, eg `./bootstrap.sh -c y0pl33g0r425`. **Build and Test** diff --git a/docs/BUILD_EXTRA.md b/docs/BUILD_EXTRA.md index 1388d574a1..b6404b43dc 100644 --- a/docs/BUILD_EXTRA.md +++ b/docs/BUILD_EXTRA.md @@ -45,7 +45,7 @@ NOTE: you will get a lot of warnings from running this command. ``` $ cd $GOPATH/src/github.com/couchbase/sync_gateway/ -$ go get -u +$ go get -u ``` Running `go get` here will put your Sync Gateway back on the master branch, so you'll need to go *back* to the feature branch again: diff --git a/docs/antora.yml b/docs/antora.yml index ed89483ead..275c9aa602 100644 --- a/docs/antora.yml +++ b/docs/antora.yml @@ -11,4 +11,4 @@ title: Sync Gateway version: '2.0' start_page: ROOT:installation.adoc nav: -- modules/docs/nav.adoc + - modules/docs/nav.adoc diff --git a/docs/api/README.md b/docs/api/README.md new file mode 100644 index 0000000000..a25ce439ea --- /dev/null +++ b/docs/api/README.md @@ -0,0 +1,17 @@ +# Sync Gateway OpenAPI Specs + +This directory contains the OpenAPI specs for the Sync Gateway REST API. + +The recommended tool to work with these specs is [Redocly](https://redoc.ly/). + +## Preview + +```sh +$ redocly preview-docs +``` + +## Linting + +```sh +$ redocly lint +``` diff --git a/docs/api/admin-capella.yaml b/docs/api/admin-capella.yaml new file mode 100644 index 0000000000..743ef405b8 --- /dev/null +++ b/docs/api/admin-capella.yaml @@ -0,0 +1,42 @@ +# Copyright 2022-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included +# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +# in that file, in accordance with the Business Source License, use of this +# software will be governed by the Apache License, Version 2.0, included in +# the file licenses/APL2.txt. + +openapi: 3.0.3 +info: + title: App Services Admin API + description: 'App Services manages access and synchronization between Couchbase Lite and Couchbase Capella' + version: '3.1' + license: + name: Business Source License 1.1 (BSL) + url: 'https://github.com/couchbase/sync_gateway/blob/master/LICENSE' +servers: + - url: 'https://{hostname}:4985' + description: Admin API + variables: + hostname: + description: The hostname to use + default: localhost +paths: + '/{db}/_session': + $ref: './paths/admin/db-_session.yaml' + '/{db}/_session/{sessionid}': + $ref: './paths/admin/db-_session-sessionid.yaml' + '/{db}/_user/{name}': + $ref: './paths/admin/db-_user-name.yaml' + '/{db}/_user/{name}/_session': + $ref: './paths/admin/db-_user-name-_session.yaml' + '/{db}/_user/{name}/_session/{sessionid}': + $ref: './paths/admin/db-_user-name-_session-sessionid.yaml' + '/{db}/_role/': + $ref: './paths/admin/db-_role-.yaml' + '/{db}/_role/{name}': + $ref: './paths/admin/db-_role-name.yaml' + +externalDocs: + description: Manage App Services for Mobile and Edge | Couchbase Docs + url: 'https://docs.couchbase.com/cloud/app-services/index.html' diff --git a/docs/api/admin.yaml b/docs/api/admin.yaml index caf0d464ca..f7750624d6 100644 --- a/docs/api/admin.yaml +++ b/docs/api/admin.yaml @@ -10,7 +10,7 @@ openapi: 3.0.3 info: title: Sync Gateway description: Sync Gateway manages access and synchronization between Couchbase Lite and Couchbase Server - version: 3.1.0 + version: '3.1' license: name: Business Source License 1.1 (BSL) url: 'https://github.com/couchbase/sync_gateway/blob/master/LICENSE' diff --git a/docs/api/components/parameters.yaml b/docs/api/components/parameters.yaml index fc7a357e1b..26a6c2d490 100644 --- a/docs/api/components/parameters.yaml +++ b/docs/api/components/parameters.yaml @@ -65,13 +65,16 @@ keyspace: schema: type: string examples: - Default scope and collection: + default: + summary: Default scope and collection value: db1 description: Default scope and collection - Named collection within the default scope: + namedInDefault: + summary: Named collection within the default scope value: db1.collection1 description: Named collection within the default scope - Fully-qualified scope and collection: + fullyQualified: + summary: Fully-qualified scope and collection value: db1.scope1.collection1 description: Fully-qualified scope and collection description: |- @@ -111,6 +114,14 @@ docid: type: string example: doc1 description: The document ID to run the operation against. +doc_id: + name: doc_id + in: query + required: false + schema: + type: string + example: doc1 + description: The document ID to run the operation against. endkey: name: endkey in: query diff --git a/docs/api/components/responses.yaml b/docs/api/components/responses.yaml index 350c1638f9..54c730f9e0 100644 --- a/docs/api/components/responses.yaml +++ b/docs/api/components/responses.yaml @@ -180,3 +180,15 @@ DB-config-precondition-failed: example: error: Precondition Failed reason: Provided If-Match header does not match current config version +All_user_channels_response: + description: Map of all keyspaces to all channels that the user has access to. + content: + application/json: + schema: + $ref: ./schemas.yaml#/all_user_channels +user_access_span_response: + description: Grant history entries for a user, showing which documents the user had access to, through which channels and for which sequence spans.. + content: + application/json: + schema: + $ref: ./schemas.yaml#/doc_access_spans diff --git a/docs/api/components/schemas.yaml b/docs/api/components/schemas.yaml index 2999b14040..9091a3d012 100644 --- a/docs/api/components/schemas.yaml +++ b/docs/api/components/schemas.yaml @@ -98,45 +98,72 @@ ExpVars: properties: admin_net_bytes_recv: type: integer + description: "The total number of bytes received (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound." admin_net_bytes_sent: type: integer + description: "The total number of bytes sent (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound." error_count: type: integer + description: "The total number of errors logged." go_memstats_heapalloc: type: integer + description: "HeapAlloc is bytes of allocated heap objects. Allocated heap objects include all reachable objects, as well as unreachable objects that the garbage collector has not yet freed. Specifically, HeapAlloc increases as heap objects are allocated and decreases as the heap is swept and unreachable objects are freed. Sweeping occurs incrementally between GC cycles, so these two processes occur simultaneously, and as a result HeapAlloc tends to change smoothly (in contrast with the sawtooth that is typical of stop-the-world garbage collectors)." go_memstats_heapidle: type: integer + description: "HeapIdle is bytes in idle (unused) spans. Idle spans have no objects in them. These spans could be (and may already have been) returned to the OS, or they can be reused for heap allocations, or they can be reused as stack memory. HeapIdle minus HeapReleased estimates the amount of memory that could be returned to the OS, but is being retained by the runtime so it can grow the heap without requesting more memory from the OS. If this difference is significantly larger than the heap size, it indicates there was a recent transient spike in live heap size." go_memstats_heapinuse: type: integer + description: "HeapInuse is bytes in in-use spans. In-use spans have at least one object in them. These spans an only be used for other objects of roughly the same size. HeapInuse minus HeapAlloc estimates the amount of memory that has been dedicated to particular size classes, but is not currently being used. This is an upper bound on fragmentation, but in general this memory can be reused efficiently." go_memstats_heapreleased: type: integer + description: "HeapReleased is bytes of physical memory returned to the OS. This counts heap memory from idle spans that was returned to the OS and has not yet been reacquired for the heap." go_memstats_pausetotalns: type: integer + description: "PauseTotalNs is the cumulative nanoseconds in GC stop-the-world pauses since the program started. During a stop-the-world pause, all goroutines are paused and only the garbage collector can run." go_memstats_stackinuse: type: integer + description: "StackInuse is bytes in stack spans. In-use stack spans have at least one stack in them. These spans can only be used for other stacks of the same size. There is no StackIdle because unused stack spans are returned to the heap (and hence counted toward HeapIdle)." go_memstats_stacksys: type: integer + description: "StackSys is bytes of stack memory obtained from the OS. StackSys is StackInuse, plus any memory obtained directly from the OS for OS thread stacks (which should be minimal)." go_memstats_sys: type: integer + description: "Sys is the total bytes of memory obtained from the OS. Sys is the sum of the XSys fields below. Sys measures the virtual address space reserved by the Go runtime for the heap, stacks, and other internal data structures. It's likely that not all of the virtual address space is backed by physical memory at any given moment, though in general it all was at some point." goroutines_high_watermark: type: integer + description: "Peak number of go routines since process start." num_goroutines: type: integer + description: "The total number of goroutines." num_idle_kv_ops: type: integer description: "The total number of idle kv operations." process_cpu_percent_utilization: - type: integer + type: number + format: float + description: "The CPU utilization as percentage value * 10. The extra 10 multiplier is a mistake left for backwards compatibility. Please consider using node_cpu_percent_utilization as of version 3.2. The CPU usage calculation is performed based on user and system CPU time, but it does not include components such as iowait. The derivation means that the values of process_cpu_percent_utilization and %Cpu, returned when running the top command, will differ." + node_cpu_percent_utilization: + type: number + format: float + description: "The node CPU utilization as percentage value, since the last time this stat was called. The CPU usage calculation is performed based on user and system CPU time, but it does not include components such as iowait." process_memory_resident: type: integer + description: "The memory utilization (Resident Set Size) for the process, in bytes." pub_net_bytes_recv: type: integer + description: "The total number of bytes received (since node start-up) on the network interface to which the Sync Gateway api.public_interface is bound. By default, that is the number of bytes received on 127.0.0.1:4984 since node start-up" pub_net_bytes_sent: type: integer + description: "The total number of bytes sent (since node start-up) on the network interface to which Sync Gateway api.public_interface is bound. By default, that is the number of bytes sent on 127.0.0.1:4984 since node start-up." system_memory_total: type: integer + description: "The total memory available on the system in bytes." warn_count: type: integer + description: "The total number of warnings logged." + uptime: + type: integer + description: "The total uptime." per_db: description: |- This array contains stats for all databases declared in the config file -- see the [Sync Gateway Statistics Schema](./../stats-monitoring.html) for more details on the metrics collected and reported by Sync Gateway. @@ -451,7 +478,7 @@ Document: type: string _exp: description: |- - Expiry time after which the document will be purged. The expiration time is set and managed on the Couchbase Server document. The value can be specified in two ways; in ISO-8601 format, for example the 6th of July 2022 at 17:00 in the BST timezone would be `2016-07-06T17:00:00+01:00`; it can also be specified as a numeric Couchbase Server expiry value. Couchbase Server expiries are specified as Unix time, and if the desired TTL is below 30 days then it can also represent an interval in seconds from the current time (for example, a value of 5 will remove the document 5 seconds after it is written to Couchbase Server). The document expiration time is returned in the response of `GET /{db}/{doc} ` when `show_exp=true` is included in the query. + Expiry time after which the document will be purged. The expiration time is set and managed on the Couchbase Server document. The value can be specified in two ways; in ISO-8601 format, for example the 6th of July 2022 at 17:00 in the BST timezone would be `2016-07-06T17:00:00+01:00`; it can also be specified as a numeric Couchbase Server expiry value. Couchbase Server expiry values are specified as Unix time, and if the desired TTL is below 30 days then it can also represent an interval in seconds from the current time (for example, a value of 5 will remove the document 5 seconds after it is written to Couchbase Server). The document expiration time is returned in the response of `GET /{db}/{doc} ` when `show_exp=true` is included in the query. As with the existing explicit purge mechanism, this applies only to the local database; it has nothing to do with replication. This expiration time is not propagated when the document is replicated. The purge of the document does not cause it to be deleted on any other database. type: string @@ -650,7 +677,7 @@ Retrieved-replication: **Behaviour** * *default* - In priority order, this will cause - - Deletes to always win (the delete with the logest revision history wins if both revisions are deletes) + - Deletes to always win (the delete with the longest revision history wins if both revisions are deletes) - The revision with the longest revision history to win. This means the the revision with the most changes and therefore the highest revision ID will win. * *localWins* - This will result in local revisions always being the winner in any conflict. * *remoteWins* - This will result in remote revisions always being the winner in any conflict. @@ -878,7 +905,7 @@ Replication: **Behaviour** * *default* - In priority order, this will cause - - Deletes to always win (the delete with the logest revision history wins if both revisions are deletes) + - Deletes to always win (the delete with the longest revision history wins if both revisions are deletes) - The revision with the longest revision history to win. This means the the revision with the most changes and therefore the highest revision ID will win. * *localWins* - This will result in local revisions always being the winner in any conflict. * *remoteWins* - This will result in remote revisions always being the winner in any conflict. @@ -1174,6 +1201,16 @@ Database: additionalProperties: x-additionalPropertiesName: scopename $ref: '#/Scopes' + maxProperties: 1 + example: + scopename: + collections: + collectionname1: + sync: 'function(doc){channel("collection name");}' + import_filter: 'function(doc) { if (doc.type != ''mobile'') { return false; } return true; }' + collectionname2: + sync: 'function(doc){channel("collection name");}' + import_filter: 'function(doc) { if (doc.type != ''mobile'') { return false; } return true; }' name: description: The name of the database. type: string @@ -1246,12 +1283,11 @@ Database: db_state_changed: $ref: '#/Event-config' feed_type: - description: The type of feed to use to communicate with Couchbase Server. + description: The type of feed to use to communicate with Couchbase Server. This will use DCP regardless of specification. type: string default: DCP enum: - DCP - - TAP deprecated: true allow_empty_password: description: This controls whether users that are created can have an empty password or not. @@ -1313,7 +1349,7 @@ Database: enable_star_channel: description: Used to control whether Sync Gateway should use the all documents (*) channel. type: boolean - default: false + default: true max_length: description: The maximum number of entries to maintain in the cache per channel. type: integer @@ -1490,11 +1526,11 @@ Database: kty: type: string description: The cryptographic algorithm family used with the key, such as "RSA" or "EC" - enum: [ 'RSA', 'EC' ] + enum: ['RSA', 'EC'] use: type: string description: The intended use of the public key. Only 'sig' is accepted. - enum: [ 'sig' ] + enum: ['sig'] alg: type: string description: The algorithm intended for use with the key. @@ -1504,7 +1540,7 @@ Database: crv: type: string description: For Elliptic Curve keys, the name of the curve to use. - enum: [ 'P-256', 'P-384', 'P-521' ] + enum: ['P-256', 'P-384', 'P-521'] x: type: string description: For Elliptic Curve keys, the X coordinate of the point, as a base64url string. @@ -1526,19 +1562,19 @@ Database: username_claim: description: |- Allows a different OpenID Connect field to be specified instead of the Subject (`sub`). - + The field name to use can be specified here. type: string roles_claim: description: |- If set, the value(s) of the given JSON Web Token claim will be added to the user's roles. - + The value of this claim must be either a string or an array of strings, any other type will result in an error. type: string channels_claim: description: |- If set, the value(s) of the given JSON Web Token claim will be added to the user's channels. - + The value of this claim must be either a string or an array of strings, any other type will result in an error. type: string oidc: @@ -1568,7 +1604,7 @@ Database: callback_url: description: |- The URL that the OpenID Connect will redirect to after authentication. - + If not provided, a callback URL will be generated. type: string disable_session: @@ -1595,26 +1631,26 @@ Database: disable_callback_state: description: |- Controls whether to maintain state between the auth request and callback endpoints (`/_oidc` and `/_oidc_callback`). - + **This is not recommended as it would cause OpenID Connect authentication to be vulnerable to Cross-Site Request Forgery (CSRF, XSRF).** type: boolean default: false username_claim: description: |- Allows a different OpenID Connect field to be specified instead of the Subject (`sub`). - + The field name to use can be specified here. type: string roles_claim: description: |- If set, the value(s) of the given OpenID Connect authentication token claim will be added to the user's roles. - + The value of this claim must be either a string or an array of strings, any other type will result in an error. type: string channels_claim: description: |- If set, the value(s) of the given OpenID Connect authentication token claim will be added to the user's channels. - + The value of this claim must be either a string or an array of strings, any other type will result in an error. type: string allow_unsigned_provider_tokens: @@ -1675,7 +1711,7 @@ Database: description: Force the use of views instead of GSI. type: boolean default: false - send_www_authentice_header: + send_www_authenticate_header: description: Controls whether to send a `WWW-Authenticate` header in `401 Unauthorized` HTTP responses. type: boolean default: true @@ -1760,8 +1796,8 @@ Database: default: 60 suspendable: description: |- - Set to true to allow the database to be suspended. - + Set to true to allow the database to be suspended. + Defaults to true when running in serverless mode otherwise defaults to false. type: boolean default: false @@ -1891,7 +1927,7 @@ Compact-status: description: |- **Applicable to attachment compaction only** - This is the number of references there are to legacy attachments. + This is the number of references there are to legacy attachments. type: string purged_attachments: description: |- @@ -1936,8 +1972,8 @@ Serverless: readOnly: true min_config_fetch_interval: description: |- - How long database configs should be kept for in Sync Gateway before refreshing. Set to 0 to fetch configs everytime. This is used for requested databases that SG does not know about. - + How long database configs should be kept for in Sync Gateway before refreshing. Set to 0 to fetch configs everytime. This is used for requested databases that SG does not know about. + This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. type: string default: 1s @@ -2005,7 +2041,7 @@ Startup-config: By default, this will only be accessible to the localhost. type: string default: '127.0.0.1:4985' - metric_interface: + metrics_interface: description: |- Network interface to bind metrics API to. @@ -2031,13 +2067,13 @@ Startup-config: type: boolean server_read_timeout: description: |- - Maximum duration.Second before timing out read of the HTTP(S) request. + Maximum duration before timing out read of the HTTP(S) request. This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. type: string server_write_timeout: description: |- - Maximum duration.Second before timing out write of the HTTP(S) response. + Maximum duration before timing out write of the HTTP(S) response. This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. type: string @@ -2056,8 +2092,9 @@ Startup-config: type: string default: 90s pretty: - description: Pretty-print JSON responses + description: Pretty-print JSON responses. This property is deprecated. type: boolean + deprecated: true max_connections: description: Max of incoming HTTP connections to accept type: number @@ -2107,35 +2144,7 @@ Startup-config: logging: description: The configuration settings for modifying Sync Gateway logging. type: object - properties: - log_file_path: - description: Absolute or relative path on the filesystem to the log file directory. A relative path is from the directory that contains the Sync Gateway executable file. - type: string - readOnly: true - redaction_level: - description: Redaction level to apply to log output. - type: string - default: partial - enum: - - none - - partial - - full - - unset - readOnly: true - console: - $ref: '#/Console-logging-config' - error: - $ref: '#/File-logging-config' - warn: - $ref: '#/File-logging-config' - info: - $ref: '#/File-logging-config' - debug: - $ref: '#/File-logging-config' - trace: - $ref: '#/File-logging-config' - stats: - $ref: '#/File-logging-config' + $ref: '#/Logging-config' auth: type: object properties: @@ -2160,6 +2169,21 @@ Startup-config: type: integer maximum: 9 minimum: 0 + max_concurrent_replications: + description: Maximum number of concurrent replication connections allowed. If set to 0 this limit will be ignored. + type: integer + max_concurrent_changes_batches: + description: Maximum number of changes batches to process concurrently per replication (1-5)" + type: integer + default: 2 + minimum: 1 + maximum: 5 + max_concurrent_revs: + description: Maximum number of revs to process concurrently per replication (5-200) + type: integer + default: 5 + minimum: 5 + maximum: 200 readOnly: true unsupported: description: Settings that are not officially supported. It is highly recommended these are **not** used. @@ -2170,6 +2194,7 @@ Startup-config: use_xattr_config: description: Store database configurations in system xattrs type: boolean + default: false stats_log_frequency: description: |- How often should stats be written to stats logs. @@ -2180,12 +2205,18 @@ Startup-config: use_stdlib_json: description: Bypass the jsoniter package and use Go's stdlib instead type: boolean + default: false http2: type: object properties: enabled: description: Whether HTTP2 support is enabled type: boolean + default: false + allow_dbconfig_env_vars: + description: Can be set to false to skip environment variable expansion in database configs + type: boolean + default: true readOnly: true database_credentials: description: 'A map of database name to credentials, that can be used instead of the bootstrap ones.' @@ -2208,47 +2239,305 @@ Startup-config: minimum: 0 readOnly: true couchbase_keepalive_interval: - description: TCP keep-alive interval between SG and Couchbase server + description: TCP keep-alive interval between SG and Couchbase server. This is unused. type: integer + deprecated: true + readOnly: true + heap_profile_collection_threshold: + description: Threshold in bytes for automatic collection of heap profiles. If not specified, defaults to 85% of the lesser of cgroup or system memory. + readOnly: true + type: integer + default: max memory + heap_profile_disable_collection: + description: Disables automatic heap profile collection. + default: false + type: boolean readOnly: true title: Startup-config -File-logging-config: +Runtime-config: type: object properties: - enabled: - description: Toggle for this log output - type: boolean - rotation: - $ref: '#/Log-rotation-config-readonly' - collation_buffer_size: - description: The size of the log collation buffer + logging: + $ref: "#/Logging-config" + max_concurrent_replications: + description: Maximum number of concurrent replication connections allowed. If set to 0 this limit will be ignored. type: integer + default: 0 + title: Runtime-config +Logging-config: + type: object + properties: + log_file_path: + description: Absolute or relative path on the filesystem to the log file directory. A relative path is from the directory that contains the Sync Gateway executable file. + type: string readOnly: true - title: File-logging-config -Log-rotation-config-readonly: + redaction_level: + description: Redaction level to apply to log output. + type: string + default: partial + enum: + - none + - partial + - full + - unset + readOnly: true + console: + $ref: '#/Console-logging-config' + error: + type: object + description: Error logging configuration. + properties: + enabled: + description: Toggle for this log output + type: boolean + default: true + rotation: + type: object + readOnly: true + properties: + max_size: + description: The maximum size in MB of the log file before it gets rotated. + type: integer + default: 100 + localtime: + description: 'If true, it uses the computer''s local time to format the backup timestamp.' + type: boolean + default: false + rotated_logs_size_limit: + description: Max Size (in mb) of log files before deletion + type: integer + default: 1024 + rotation_interval: + description: |- + If set, the interval at which log files are rotated, even if max_size is not reached. + + This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. + type: string + default: 0 + max_age: + description: The maximum number of days to retain old log files. + default: 360 + type: integer + collation_buffer_size: + description: The size of the log collation buffer. + default: 0 + type: integer + readOnly: true + warn: + type: object + description: Warning logging configuration. + properties: + enabled: + description: Toggle for this log output + type: boolean + default: true + rotation: + type: object + readOnly: true + properties: + max_size: + description: The maximum size in MB of the log file before it gets rotated. + type: integer + default: 100 + localtime: + description: 'If true, it uses the computer''s local time to format the backup timestamp.' + type: boolean + default: false + rotated_logs_size_limit: + description: Max Size (in mb) of log files before deletion + type: integer + default: 1024 + rotation_interval: + description: |- + If set, the interval at which log files are rotated, even if max_size is not reached. + + This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. + type: string + default: 0 + max_age: + description: The maximum number of days to retain old log files. + default: 180 + type: integer + collation_buffer_size: + description: The size of the log collation buffer + default: 0 + type: integer + readOnly: true + info: + type: object + description: Info logging configuration. + properties: + enabled: + description: Toggle for this log output + type: boolean + default: true + rotation: + type: object + readOnly: true + properties: + max_size: + description: The maximum size in MB of the log file before it gets rotated. + type: integer + default: 100 + localtime: + description: 'If true, it uses the computer''s local time to format the backup timestamp.' + type: boolean + default: false + rotated_logs_size_limit: + description: Max Size (in mb) of log files before deletion + type: integer + default: 1024 + rotation_interval: + description: |- + If set, the interval at which log files are rotated, even if max_size is not reached. + + This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. + type: string + default: 0 + max_age: + description: The maximum number of days to retain old log files. + default: 6 + type: integer + collation_buffer_size: + description: The size of the log collation buffer + default: 0 + type: integer + readOnly: true + debug: + type: object + description: Debug logging configuration. + properties: + enabled: + description: Toggle for this log output + type: boolean + default: false + rotation: + type: object + readOnly: true + properties: + max_size: + description: The maximum size in MB of the log file before it gets rotated. + type: integer + default: 100 + localtime: + description: 'If true, it uses the computer''s local time to format the backup timestamp.' + type: boolean + default: false + rotated_logs_size_limit: + description: Max Size (in mb) of log files before deletion + type: integer + default: 1024 + rotation_interval: + description: |- + If set, the interval at which log files are rotated, even if max_size is not reached. + + This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. + type: string + default: 0 + max_age: + description: The maximum number of days to retain old log files. + default: 2 + type: integer + collation_buffer_size: + description: The size of the log collation buffer + default: 1000 + type: integer + readOnly: true + trace: + type: object + description: Trace logging configuration. + properties: + enabled: + description: Toggle for this log output + type: boolean + default: false + rotation: + type: object + readOnly: true + properties: + max_size: + description: The maximum size in MB of the log file before it gets rotated. + type: integer + default: 100 + localtime: + description: 'If true, it uses the computer''s local time to format the backup timestamp.' + type: boolean + default: false + rotated_logs_size_limit: + description: Max Size (in mb) of log files before deletion + type: integer + default: 1024 + rotation_interval: + description: |- + If set, the interval at which log files are rotated, even if max_size is not reached. + + This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. + type: string + default: 0 + max_age: + description: The maximum number of days to retain old log files. + default: 2 + type: integer + collation_buffer_size: + description: The size of the log collation buffer + default: 1000 + type: integer + readOnly: true + stats: + type: object + description: Trace logging configuration. + properties: + enabled: + description: Toggle for this log output + type: boolean + default: true + rotation: + type: object + readOnly: true + properties: + max_size: + description: The maximum size in MB of the log file before it gets rotated. + type: integer + default: 100 + localtime: + description: 'If true, it uses the computer''s local time to format the backup timestamp.' + type: boolean + default: false + rotated_logs_size_limit: + description: Max Size (in mb) of log files before deletion + type: integer + default: 1024 + rotation_interval: + description: |- + If set, the interval at which log files are rotated, even if max_size is not reached. + + This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. + type: string + default: 0 + max_age: + description: The maximum number of days to retain old log files. + default: 6 + type: integer + collation_buffer_size: + description: The size of the log collation buffer + default: 0 + type: integer + readOnly: true +File-logging-config-base: type: object properties: - max_size: - description: The maximum size in MB of the log file before it gets rotated. - type: integer - max_age: - description: The maximum number of days to retain old log files. - type: integer - localtime: - description: 'If true, it uses the computer''s local time to format the backup timestamp.' + enabled: + description: Toggle for this log output type: boolean - rotated_logs_size: - description: Max Size (in mb) of log files before deletion - type: integer - readOnly: true - title: Log-rotation-config + default: false + title: File-logging-config Console-logging-config: type: object properties: log_level: description: Log Level for the console output type: string - default: none + default: info enum: - none - error @@ -2264,6 +2553,7 @@ Console-logging-config: color_enabled: description: Log with color for the console output type: boolean + default: false readOnly: true file_output: description: 'Override the default stderr output, and write to the file specified instead' @@ -2272,12 +2562,40 @@ Console-logging-config: enabled: description: Toggle for this log output type: boolean + default: false readOnly: true rotation: - $ref: '#/Log-rotation-config-readonly' + type: object + readOnly: true + title: Log-rotation-config + properties: + max_size: + description: The maximum size in MB of the log file before it gets rotated. + type: integer + default: 100 + localtime: + description: 'If true, it uses the computer''s local time to format the backup timestamp.' + type: boolean + default: false + rotated_logs_size_limit: + description: Max Size (in mb) of log files before deletion + type: integer + default: 1024 + rotation_interval: + description: |- + If set, the interval at which log files are rotated, even if max_size is not reached. + + This is a duration and therefore can be provided with units "h", "m", "s", "ms", "us", and "ns". For example, 5 hours, 20 minutes, and 30 seconds would be `5h20m30s`. + type: string + default: 0 + max_age: + description: The maximum number of days to retain old log files. By default, there is no rotation, max_age=0. + default: 0 + type: integer collation_buffer_size: - description: The size of the log collation buffer. + description: The size of the log collation buffer. The default is 10 if the output is stderr, or 1000 if to a file. type: integer + default: 10 readOnly: true title: Console-logging-config Log-update-enabled: @@ -2444,3 +2762,45 @@ CollectionNames: description: Indicates whether database initialization is in progress. type: boolean example: true +all_user_channels: + description: |- + All user channels split by how they were assigned to the user and by keyspace. + type: object + properties: + all_channels: + description: |- + All channels that the user has access to. + type: object + properties: + keyspace: + type: object + properties: + channel: + $ref: '#/channelEntry' +channelEntry: + description: Channel name + type: object + properties: + entries: + type: array + description: Start sequence to end sequence. If the channel is currently granted, the end sequence will be zero. + updated_at: + type: integer + description: Unix timestamp of last update +doc_access_spans: + description: |- + Grant history entries for a user, showing which documents the user had access to, through which channels and for which sequence spans. + type: object + properties: + doc_id: + description: |- + Document names. + type: object + properties: + channel: + description: Channel name + type: object + properties: + entries: + type: array + description: Start sequence to end sequence. If the channel is currently granted, the end sequence will be zero. diff --git a/docs/api/metric-capella.yaml b/docs/api/metric-capella.yaml new file mode 100644 index 0000000000..486b8eabcc --- /dev/null +++ b/docs/api/metric-capella.yaml @@ -0,0 +1,32 @@ +# Copyright 2022-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included +# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +# in that file, in accordance with the Business Source License, use of this +# software will be governed by the Apache License, Version 2.0, included in +# the file licenses/APL2.txt. + +openapi: 3.0.3 +info: + title: App Services Metrics API + description: 'App Services manages access and synchronization between Couchbase Lite and Couchbase Capella' + version: '3.1' + license: + name: Business Source License 1.1 (BSL) + url: 'https://github.com/couchbase/sync_gateway/blob/master/LICENSE' +servers: + - url: 'https://{hostname}:4988' + description: Metrics API + variables: + hostname: + description: The hostname to use + default: localhost +paths: + /metrics: + $ref: ./paths/metric/metrics.yaml +tags: + - name: Prometheus + description: Endpoints for use with Prometheus +externalDocs: + description: Manage App Services for Mobile and Edge | Couchbase Docs + url: 'https://docs.couchbase.com/cloud/app-services/index.html' diff --git a/docs/api/metric.yaml b/docs/api/metric.yaml index 259306e57a..d49ae47866 100644 --- a/docs/api/metric.yaml +++ b/docs/api/metric.yaml @@ -10,7 +10,7 @@ openapi: 3.0.3 info: title: Sync Gateway description: Sync Gateway manages access and synchronization between Couchbase Lite and Couchbase Server - version: 3.1.0 + version: '3.1' license: name: Business Source License 1.1 (BSL) url: 'https://github.com/couchbase/sync_gateway/blob/master/LICENSE' diff --git a/docs/api/paths/admin/_config.yaml b/docs/api/paths/admin/_config.yaml index 28dd1a84aa..0bc789a2a8 100644 --- a/docs/api/paths/admin/_config.yaml +++ b/docs/api/paths/admin/_config.yaml @@ -49,7 +49,7 @@ put: content: application/json: schema: - $ref: ../../components/schemas.yaml#/Startup-config + $ref: ../../components/schemas.yaml#/Runtime-config responses: '200': description: Successfully set runtime options diff --git a/docs/api/paths/admin/db-.yaml b/docs/api/paths/admin/db-.yaml index 90ea68e249..cd8e5de5be 100644 --- a/docs/api/paths/admin/db-.yaml +++ b/docs/api/paths/admin/db-.yaml @@ -69,30 +69,6 @@ get: init_in_progress: description: Indicates whether database initialization is in progress. type: boolean - scopes: - description: 'Scopes that are used by the database.' - type: object - example: - scope1: - collections: - collection1: - update_seq: 123456 - collection2: - update_seq: 654321 - additionalProperties: - description: 'The name of the scope.' - type: object - properties: - collections: - description: 'The set of collections within the scope.' - additionalProperties: - description: 'The name of the collection.' - type: object - properties: - update_seq: - description: 'The last sequence number that was committed to the collection.' - type: integer - example: 123456 '404': $ref: ../../components/responses.yaml#/Not-found tags: diff --git a/docs/api/paths/admin/keyspace-_bulk_docs.yaml b/docs/api/paths/admin/keyspace-_bulk_docs.yaml index abadc50d6c..f209b6c039 100644 --- a/docs/api/paths/admin/keyspace-_bulk_docs.yaml +++ b/docs/api/paths/admin/keyspace-_bulk_docs.yaml @@ -88,7 +88,8 @@ post: rev: 6-b3e8dcf825b71ccee112f3572ec4323c - id: BobSettings rev: 2-5145e1086bb8d1d71a531e9f6b543c58 - Partial success: + PartialSuccess: + summary: PartialSuccess value: - error: conflict id: FooBar diff --git a/docs/api/paths/admin/keyspace-_purge.yaml b/docs/api/paths/admin/keyspace-_purge.yaml index 2dc6ae7554..fc9ced5e4a 100644 --- a/docs/api/paths/admin/keyspace-_purge.yaml +++ b/docs/api/paths/admin/keyspace-_purge.yaml @@ -41,11 +41,13 @@ post: enum: - '*' examples: - Example: + single: + summary: 'Single document' value: doc_id: - '*' - Multiple purges example: + multiple: + summary: 'Multiple documents' value: doc_id_1: - '*' @@ -70,12 +72,14 @@ post: required: - purged examples: - Example: + single: + summary: 'Single document' value: purged: doc_id: - '*' - Multiple purges example: + multiple: + summary: 'Multiple documents' value: purged: doc_id_1: diff --git a/docs/api/paths/public/keyspace-_bulk_docs.yaml b/docs/api/paths/public/keyspace-_bulk_docs.yaml index cacf95ba15..32c836655d 100644 --- a/docs/api/paths/public/keyspace-_bulk_docs.yaml +++ b/docs/api/paths/public/keyspace-_bulk_docs.yaml @@ -84,7 +84,8 @@ post: rev: 6-b3e8dcf825b71ccee112f3572ec4323c - id: BobSettings rev: 2-5145e1086bb8d1d71a531e9f6b543c58 - Partial success: + PartialSuccess: + summary: "Partial success" value: - error: conflict id: FooBar diff --git a/docs/api/plugins/decorators/excise-rbac-capella.js b/docs/api/plugins/decorators/excise-rbac-capella.js new file mode 100644 index 0000000000..afe1feef3e --- /dev/null +++ b/docs/api/plugins/decorators/excise-rbac-capella.js @@ -0,0 +1,33 @@ +/** + * Copyright 2024-Present Couchbase, Inc. + * + * Use of this software is governed by the Business Source License included + * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified + * in that file, in accordance with the Business Source License, use of this + * software will be governed by the Apache License, Version 2.0, included in + * the file licenses/APL2.txt. + */ + +/** + * Removes the RBAC roles from capella API docs. This expects the RBAC information to be at the end of the documentation string. This is not a robust way of doing this. + * @module ExciseRBACCapella + */ + +module.exports = ExciseRBACCapella; + +const re = new RegExp("Required Sync Gateway RBAC roles"); + +/** @type {import('@redocly/cli').OasDecorator} */ +function ExciseRBACCapella() { + return { + Operation: { + leave(Operation) { + // remove all text after first regex match + idx = Operation.description.search(re); + if (idx > 0) { + Operation.description = Operation.description.substr(0, idx); + } + }, + }, + }; +} diff --git a/docs/api/plugins/decorators/replace-description-capella.js b/docs/api/plugins/decorators/replace-description-capella.js new file mode 100644 index 0000000000..252d3947fb --- /dev/null +++ b/docs/api/plugins/decorators/replace-description-capella.js @@ -0,0 +1,30 @@ +/** + * Copyright 2024-Present Couchbase, Inc. + * + * Use of this software is governed by the Business Source License included + * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified + * in that file, in accordance with the Business Source License, use of this + * software will be governed by the Apache License, Version 2.0, included in + * the file licenses/APL2.txt. + */ + +/** + * Does a string replacement on all operations (GET,PUT,POST,etc) to replace Sync Gateway with App Services. + * @module ReplaceDescriptionCapella + */ + +module.exports = ReplaceDescriptionCapella; + +/** @type {import('@redocly/cli').OasDecorator} */ +function ReplaceDescriptionCapella() { + return { + Operation: { + leave(Operation) { + Operation.description = Operation.description.replace( + "Sync Gateway", + "App Services", + ); + }, + }, + }; +} diff --git a/docs/api/plugins/decorators/replace-server-capella.js b/docs/api/plugins/decorators/replace-server-capella.js new file mode 100644 index 0000000000..6bd4808a38 --- /dev/null +++ b/docs/api/plugins/decorators/replace-server-capella.js @@ -0,0 +1,24 @@ +/** + * Copyright 2024-Present Couchbase, Inc. + * + * Use of this software is governed by the Business Source License included + * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified + * in that file, in accordance with the Business Source License, use of this + * software will be governed by the Apache License, Version 2.0, included in + * the file licenses/APL2.txt. + */ + +module.exports = ReplaceServersCapella; + +/** @type {import('@redocly/cli').OasDecorator} */ +function ReplaceServersCapella({ serverUrl }) { + return { + Server: { + leave(Server) { + if (serverUrl) { + Server.url = serverUrl; + } + }, + }, + }; +} diff --git a/docs/api/plugins/plugin.js b/docs/api/plugins/plugin.js new file mode 100644 index 0000000000..e2c875af07 --- /dev/null +++ b/docs/api/plugins/plugin.js @@ -0,0 +1,24 @@ +/** + * Copyright 2024-Present Couchbase, Inc. + * + * Use of this software is governed by the Business Source License included + * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified + * in that file, in accordance with the Business Source License, use of this + * software will be governed by the Apache License, Version 2.0, included in + * the file licenses/APL2.txt. + */ + +const ExciseRBACCapella = require("./decorators/excise-rbac-capella.js"); +const ReplaceDescriptionCapella = require("./decorators/replace-description-capella.js"); +const ReplaceServerCapella = require("./decorators/replace-server-capella.js"); + +module.exports = { + decorators: { + oas3: { + "excise-rbac-capella": ExciseRBACCapella, + "replace-description-capella": ReplaceDescriptionCapella, + "replace-server-capella": ReplaceServerCapella, + }, + }, + id: "plugin", +}; diff --git a/docs/api/public.yaml b/docs/api/public.yaml index b5b9e6df8d..08ac5afea1 100644 --- a/docs/api/public.yaml +++ b/docs/api/public.yaml @@ -10,7 +10,7 @@ openapi: 3.0.3 info: title: Sync Gateway description: Sync Gateway manages access and synchronization between Couchbase Lite and Couchbase Server - version: 3.1.0 + version: '3.1' license: name: Business Source License 1.1 (BSL) url: 'https://github.com/couchbase/sync_gateway/blob/master/LICENSE' @@ -50,8 +50,10 @@ paths: $ref: './paths/public/keyspace-_changes.yaml' '/{db}/_design/{ddoc}': $ref: './paths/public/db-_design-ddoc.yaml' + x-capella: false '/{db}/_design/{ddoc}/_view/{view}': $ref: './paths/public/db-_design-ddoc-_view-view.yaml' + x-capella: false '/{db}/_ensure_full_commit': $ref: './paths/public/db-_ensure_full_commit.yaml' '/{keyspace}/_revs_diff': @@ -101,8 +103,10 @@ tags: description: Create and manage document attachments - name: Replication description: Create and manage inter-Sync Gateway replications + x-capella: false - name: Unsupported description: Endpoints that are not supported by Sync Gateway + x-capella: false externalDocs: description: Sync Gateway Quickstart | Couchbase Docs url: 'https://docs.couchbase.com/sync-gateway/current/index.html' diff --git a/docs/api/replace-servers-capella.js b/docs/api/replace-servers-capella.js new file mode 100644 index 0000000000..7bbadb9f1d --- /dev/null +++ b/docs/api/replace-servers-capella.js @@ -0,0 +1,24 @@ +/** + * Copyright 2024-Present Couchbase, Inc. + * + * Use of this software is governed by the Business Source License included + * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified + * in that file, in accordance with the Business Source License, use of this + * software will be governed by the Apache License, Version 2.0, included in + * the file licenses/APL2.txt. + */ + +module.exports = ReplaceServersCapella; + +/** @type {import('@redocly/cli').OasDecorator} */ + +function ReplaceServersCapella({ serverUrl }) { + return { + Server: { + leave(Server) { + Server.url = serverUrl; + delete Server.protocol; + }, + }, + }; +} diff --git a/docs/design/cache/README.md b/docs/design/cache/README.md index a141f6f11b..9966591e84 100644 --- a/docs/design/cache/README.md +++ b/docs/design/cache/README.md @@ -1,6 +1,6 @@ ##Remote Change Cache -The primary goal of the distributed cache implementation for Sync Gateway is to provide an +The primary goal of the distributed cache implementation for Sync Gateway is to provide an alternative to the existing in-memory change cache. The existing cache requires each Sync Gateway node to process every mutation occuring on the Couchbase Server bucket. The intention of the distributed cache is to increase the scaling capacity of a Sync Gateway cluster. @@ -12,11 +12,11 @@ cache is to increase the scaling capacity of a Sync Gateway cluster. ###Goals - **Scale Out Sync Gateway** – Ability to scale out Sync Gateway capacity by adding additional nodes to the cluster. -- **Reduce Couchbase Server Overhead** – Add additional Sync Gateway nodes to the cluster without requiring the Couchbase Server to provide additional TAP/DCP feeds +- **Reduce Couchbase Server Overhead** – Add additional Sync Gateway nodes to the cluster without requiring the Couchbase Server to provide additional TAP/DCP feeds - **Replication Consistency** – Consistent sequence handling to ensure replication stability and consistency ###Components -- **[Cache Overview](cache_overview.md)** +- **[Cache Overview](cache_overview.md)** ###Implementations - **[Single Writer Cache](single_writer.md)** - A remote cache with a single cache writer, multiple cache readers diff --git a/docs/design/cache/cache_overview.md b/docs/design/cache/cache_overview.md index e6d6a43aa0..19ccc88822 100644 --- a/docs/design/cache/cache_overview.md +++ b/docs/design/cache/cache_overview.md @@ -16,12 +16,12 @@ For each channel, a separate document stores a counter for the channel. This is ##Document Types and Terminology -**Cache block** documents are used to store the sequences associated with a channel. A **channel cache** consists of one or more blocks. Block numbers start at '0'. +**Cache block** documents are used to store the sequences associated with a channel. A **channel cache** consists of one or more blocks. Block numbers start at '0'. * Document name: _cache:[channel name]:block[n] * Document format: see below -**Sequence document** stores the Document ID, Revision ID, and change flags associated with the sequence. +**Sequence document** stores the Document ID, Revision ID, and change flags associated with the sequence. * Document name: _cache:\_seq:[sequence number] * Document format: JSON @@ -35,13 +35,13 @@ The **channel counter** document stores a single integer counter, which is incre ###Sequence-indexed Array -Cache block documents are a fixed size byte array. The value of `byte[n]` is a flag for the presence of sequence `n` in the channel. +Cache block documents are a fixed size byte array. The value of `byte[n]` is a flag for the presence of sequence `n` in the channel. Currently two bits are used to store each sequence: * bit 0 - presence of sequence in the channel * bit 1 - whether the sequence is a removal from the channel -The cache block size is currently set at 10000 bytes, so cache block 0 stores sequences 0-9999, cache block 1 stores sequences 10000-19999, etc. More detailed performance analysis is required to identify the ideal cache block size (to balance document size with multiple retrievals). +The cache block size is currently set at 10000 bytes, so cache block 0 stores sequences 0-9999, cache block 1 stores sequences 10000-19999, etc. More detailed performance analysis is required to identify the ideal cache block size (to balance document size with multiple retrievals). Note: The POC assigns a full byte for each sequence, but if we don't identify a need for the additional 6 bits, this can be refactored to increase the number of sequences we can store per cache block. @@ -67,7 +67,7 @@ Cache writers add sequences to the cache block using an atomic append. Cache wr 1. Fast write operation (atomic append, instead of read/CAS write) 2. Reduced write contention -**Drawbacks** +**Drawbacks** 1. Writers need to iterate over entire cache block and sort results to perform usual operations (since=n) 2. More space required per sequence (8 bytes) - results in fewer sequences per block diff --git a/docs/design/cache/front_end_cache_writes.md b/docs/design/cache/front_end_cache_writes.md index af93caa927..150b22453d 100644 --- a/docs/design/cache/front_end_cache_writes.md +++ b/docs/design/cache/front_end_cache_writes.md @@ -1,8 +1,8 @@ ##Front-end Cache Writes -Remote cache scalability is capped by the ability for cache writer(s) to keep up with the inflow on the feed. A potential optimization is to have the front-end nodes write cache information when the original document is being written to the bucket, instead of waiting for it to appear on the feed. This allows a portion of the cache write work to be distributed across the whole cluster. In this scenario the cache writer's only responsibility is calculation of the **stable sequence** value. +Remote cache scalability is capped by the ability for cache writer(s) to keep up with the inflow on the feed. A potential optimization is to have the front-end nodes write cache information when the original document is being written to the bucket, instead of waiting for it to appear on the feed. This allows a portion of the cache write work to be distributed across the whole cluster. In this scenario the cache writer's only responsibility is calculation of the **stable sequence** value. -Here "front-end" just refers to any Sync Gateway node that's processing writes from clients. +Here "front-end" just refers to any Sync Gateway node that's processing writes from clients. ###Front-end processing @@ -14,13 +14,13 @@ After a successful write of the base document to the bucket, front-end nodes per ##Cache writer (controlller) processing -With the front end nodes updating the cache, the cache writer is primarily responsible for calculating and storing the **stable sequence** value. This can use the standard buffered feed reader approach. +With the front end nodes updating the cache, the cache writer is primarily responsible for calculating and storing the **stable sequence** value. This can use the standard buffered feed reader approach. -However, the cache writer can also be used to validate consistency of updates made by the front-end nodes, by validating the existence of the **sequence document** in the cache for each sequence. This covers the scenario where a front-end node fails after updating the base bucket, but before cache update is complete. +However, the cache writer can also be used to validate consistency of updates made by the front-end nodes, by validating the existence of the **sequence document** in the cache for each sequence. This covers the scenario where a front-end node fails after updating the base bucket, but before cache update is complete. ##Cache reader processing -There are two options for the cache reader. +There are two options for the cache reader. 1. Return all data available in the cache for a given request, returning a compound sequence number with **stable sequence** as the low value. This addresses the issue originally fixed by #525 - ensuring that late arriving TAP sequences aren't blocking. DocID deduplication will ensure that we don't have a scenario where a client sees revisions out of order on the client. This is a better solution than the one used for #525, as changes are available as soon as they are written, instead of waiting for them to reappear on the TAP feed. 2. Only return sequences earlier than the **stable sequence** value. Omits the fast/slow processing, but simplifies the cache reader processing and makes it more intuitive/easier to reason about. diff --git a/docs/design/cache/multi-writer.md b/docs/design/cache/multi-writer.md index 45f089385e..bc8002f112 100644 --- a/docs/design/cache/multi-writer.md +++ b/docs/design/cache/multi-writer.md @@ -9,7 +9,7 @@ **Options** 1. Multiple DCP feeds, each for a subset of vbucket -Each cache writer would instantiate a DCP feed, connected to a subset of vbuckets. This solves the "DDOS" problem - each Sync Gateway node would only be processing a subset of the revisions seen on the bucket. This doesn't address the server performance degradation as a result of serving multiple DCP feeds. +Each cache writer would instantiate a DCP feed, connected to a subset of vbuckets. This solves the "DDOS" problem - each Sync Gateway node would only be processing a subset of the revisions seen on the bucket. This doesn't address the server performance degradation as a result of serving multiple DCP feeds. **Open Issues** * Confirm with server team that multiple DCP feeds connecting to partitioned subsets of vbuckets has the same performance overhead/concerns as multiple full DCP feeds. @@ -33,12 +33,12 @@ Instead of sharding revisions by vbucket, shard them by a mod of the current SG * We have several corner cases where a single revision allocates several sequence values (unused sequences during during CAS conflicts, sequences being deduplicated by DCP). These sequences will be getting routed to the "wrong" cache writer. -###Stable Sequence +###Stable Sequence **Options** - 1. Cache controller. Cache writers post status to well-known location in the cache bucket. Controller process monitors and aggregates status, and updates the stable sequence. This would support continued use of the existing sequence model. + 1. Cache controller. Cache writers post status to well-known location in the cache bucket. Controller process monitors and aggregates status, and updates the stable sequence. This would support continued use of the existing sequence model. - 2. [DCP timestamp as sequence](timestamp_sequence.md) + 2. [DCP timestamp as sequence](timestamp_sequence.md) ###Node Failure diff --git a/docs/design/cache/single_writer.md b/docs/design/cache/single_writer.md index e03e77912b..c6e633e7f3 100644 --- a/docs/design/cache/single_writer.md +++ b/docs/design/cache/single_writer.md @@ -1,11 +1,11 @@ ##Single Writer Cache -The single writer cache aims to improve Sync Gateway scalability by avoiding the use of multiple server feeds, and centralizing the feed processing work on a single node that can be scaled up. Under this model there will still be a threshold at which the inflow rate on the feed exceeds the ability for a single Sync Gateway to process it, but this limit should be significantly higher than the existing in-memory model. +The single writer cache aims to improve Sync Gateway scalability by avoiding the use of multiple server feeds, and centralizing the feed processing work on a single node that can be scaled up. Under this model there will still be a threshold at which the inflow rate on the feed exceeds the ability for a single Sync Gateway to process it, but this limit should be significantly higher than the existing in-memory model. ###Overview -- **Single TAP/DCP feed** - Sync Gateway cluster only uses a single TAP/DCP feed, instead of one per Sync Gateway node -- **Scale up writer, scale out readers** – Can increase Sync Gateway cluster capacity by scaling up the cache writer node, and scaling out cache reader nodes. +- **Single TAP/DCP feed** - Sync Gateway cluster only uses a single TAP/DCP feed, instead of one per Sync Gateway node +- **Scale up writer, scale out readers** – Can increase Sync Gateway cluster capacity by scaling up the cache writer node, and scaling out cache reader nodes. - **Batched cache writer** - Improve processing capacity by batching cache update operations @@ -14,14 +14,14 @@ Uses sequence-indexed array, as described in the cache overview documentation. ###Cache Writer -The cache writer is responsible for writing sequences to the cache, and determining and publishing the **stable sequence** value. +The cache writer is responsible for writing sequences to the cache, and determining and publishing the **stable sequence** value. For each sequence seen on the feed, the cache writer must perform the following operations on the cache: 1. Insert a new **sequence document** to the cache. 2. For each channel associated with the revision: - Read and update of the **cache block** corresponding to the channel and sequence - Increment the **channel counter** document for the channel - 3. Update the **stable sequence** document (when appropriate) + 3. Update the **stable sequence** document (when appropriate) For a document with `n` channels, this results in: * `n` reads @@ -40,7 +40,7 @@ Incoming feed entries are placed in a queue for caching. A separate goroutine ru 3. Loop over channel sets - for each set: - Start a goroutine to read and update the **cache block(s)** corresponding to the channel and sequences 4. Wait for goroutine completion from #2 and #3 - 5. For each channel set: + 5. For each channel set: - Start a goroutine to increment the **channel counter** for the channel 6. Wait for goroutine completiong from #5 7. Update the **stable sequence** document @@ -49,8 +49,8 @@ For `m` documents distributed over `n` channels, this results in: * `m + n + 1` writes (`m` sequence docs + `n` channel blocks + `1` stable sequence) * `n` reads (`n` channel blocks) * `n` incrs (`n` channel counters) - * `(3n + m + 1)/m` total operations per document. - * If `m` is large relative to `n` (many updates in the same channels), this approaches 1 op/sequence + * `(3n + m + 1)/m` total operations per document. + * If `m` is large relative to `n` (many updates in the same channels), this approaches 1 op/sequence * If `n` approaches `m` (many updates in unrelated channels), this is closer to 4 op/sequence * If `n` is large relative to `m` (many updates, each going to several unrelated channels), this approaches (`3n + 1`) ops per sequence @@ -65,15 +65,15 @@ For `m` documents distributed over `n` channels, this results in: **Using DCP** Restart support is straightforward when using DCP. When the cache writer receives a snapshot start event on the DCP feed, it persists the current DCP vbucket timestamp to the cache database. On Sync Gateway restart, it just needs to start a new DCP feed from the previous timestamp. -**Using TAP** +**Using TAP** TAP doesn't support starting a feed from a particular position - only a full backfill from zero, or listening to new revisions. Reprocessing the entire mutation history from zero on each Sync Gateway restart isn't practical. On Sync Gateway restart, we need a way to backfill the sequences between the last `stable sequence` (as stored in the cache), and the current `_sync:seq` value (we can assume any new revisions after the _sync:seq value will appear on the new TAP feed). On Sync Gateway startup, the cache writer should: - Start the TAP feed listening for new mutations - Start a new process to backfill from the previous `stable sequence` to the current `sync:seq` -We don't currently have a view that supports a simple sequence-based range query (e.g. all sequences across all channels from seq1 to seq2). One approach would be to use the `channels` view to -query the `*` channel for a sequence range. This would require that the `*` channel always be enabled. +We don't currently have a view that supports a simple sequence-based range query (e.g. all sequences across all channels from seq1 to seq2). One approach would be to use the `channels` view to +query the `*` channel for a sequence range. This would require that the `*` channel always be enabled. ###Failover @@ -108,7 +108,7 @@ There shouldn't be any difference to the handling for non-continuous _changes re The preferred approach would be to treat the remote cache as a full index - all sequences would be present, with no need to backfill from a view query, the way we do today for the in-memory cache. The potential concern would be the number of documents in the cache bucket - we end up with one **sequence document** for each revision in the base bucket, effectively doubling the total number of documents in the cluster. -One optimization would be to expire the **sequence documents** after a relatively long period (days?), when the total number of documents in the bucket exceeds a specified threshold. The information in the sequence document is also available from the document in the base bucket - it just requires additional overhead to unmarshal the _sync metadata and retrieve that data. +One optimization would be to expire the **sequence documents** after a relatively long period (days?), when the total number of documents in the bucket exceeds a specified threshold. The information in the sequence document is also available from the document in the base bucket - it just requires additional overhead to unmarshal the _sync metadata and retrieve that data. A similar optimization could be done to purge **sequence documents** for obsolete revisions. diff --git a/docs/design/cache/timestamp_sequence.md b/docs/design/cache/timestamp_sequence.md index 05b12eb80e..397e394f36 100644 --- a/docs/design/cache/timestamp_sequence.md +++ b/docs/design/cache/timestamp_sequence.md @@ -3,7 +3,7 @@ Discussion on using Couchbase Server's native sequence handling as an alternative to Sync Gateway's current sequence allocation model ###Overview -Couchbase Server assigns an internal sequence value to each mutation. Within a given vbucket, Couchbase Server guarantees monotomically increasing sequence values. +Couchbase Server assigns an internal sequence value to each mutation. Within a given vbucket, Couchbase Server guarantees monotomically increasing sequence values. Based on this, we can construct a stable timestamp for the entire cluster, as a vector clock of the current high sequence values for each vbucket. This timestamp satisfies the requirements for use in Sync Gateway replication as a sequence. diff --git a/docs/design/channel_index/clock_handling.md b/docs/design/channel_index/clock_handling.md index 8dab1cee05..249c5fa665 100644 --- a/docs/design/channel_index/clock_handling.md +++ b/docs/design/channel_index/clock_handling.md @@ -2,15 +2,15 @@ One of the key requirements of the channel index is the ability to define a **stable sequence**, which allows replication processing to deliver the guarantee that a client has seen everything up to a given point in a channel. This is achieved in the current architecture using sequence buffering: each Sync Gateway node buffers incoming events on the DCP feed to be able to validate that it's seen everything up to a given sequence number. -When each Sync Gateway node only sees a subset of the mutations coming from the Couchbase Server (see (Feed Management)[feed_management.md] for more details), each node isn't able to identify the stable sequence on it's own. Each Sync Gateway node can only define the last seen sequence for the vbuckets it has been allocated. +When each Sync Gateway node only sees a subset of the mutations coming from the Couchbase Server (see (Feed Management)[feed_management.md] for more details), each node isn't able to identify the stable sequence on it's own. Each Sync Gateway node can only define the last seen sequence for the vbuckets it has been allocated. -Despite this, using the vbucket sequence has a significant advantage over the metadata-based sequence - we are guaranteed to see monotonically increasing sequences for a given vbucket. There isn't any buffering required - the process of determining the overall **stable sequence** for the channel index is just a matter of building the vector clock of the highest sequences that have been indexed for each vbucket, and storing this in the index as the **stable clock**. +Despite this, using the vbucket sequence has a significant advantage over the metadata-based sequence - we are guaranteed to see monotonically increasing sequences for a given vbucket. There isn't any buffering required - the process of determining the overall **stable sequence** for the channel index is just a matter of building the vector clock of the highest sequences that have been indexed for each vbucket, and storing this in the index as the **stable clock**. -####Stable Clock +####Stable Clock The initial implementation of the stable clock will store it in a single document. Each Sync Gateway node doing index writes will periodically update the stable clock sequences for the vbuckets in its partitions. This is done periodically (instead of on each index update) to optimize performance - having each sync gateway node attempt to update the stable clock after every mutation isn't going to be efficient. The default setting will be to update the stable clock at least once per second. This attempts to balance `_changes` latency with clock write performance. -The preferred approach is to maintain a single **stable clock** document, updated by all SG writer nodes, to reduce the amount of reads that index readers need to do perform to retrieve the stable clock value. This has the potential for high contention/CAS-loop processing during high write load. If that proves to be a problem, we would need to switch to multiple **stable clock** documents, each handling one or more vbucket partitions. +The preferred approach is to maintain a single **stable clock** document, updated by all SG writer nodes, to reduce the amount of reads that index readers need to do perform to retrieve the stable clock value. This has the potential for high contention/CAS-loop processing during high write load. If that proves to be a problem, we would need to switch to multiple **stable clock** documents, each handling one or more vbucket partitions. ####Channel Clock @@ -23,8 +23,8 @@ To support notification handling for continuous `_changes` feeds, Sync Gateways ####User and Role Counters -Another requirement of continuous `_changes` processing is to receive notification when a user or role document has been changed (to handle things like access grants in the middle of a continuous `_changes` request). +Another requirement of continuous `_changes` processing is to receive notification when a user or role document has been changed (to handle things like access grants in the middle of a continuous `_changes` request). -When a Sync Gateway node sees an update to a `_user` or `_role` document on the DCP feed, it will increment the corresponding **principal counter** document in the bucket. +When a Sync Gateway node sees an update to a `_user` or `_role` document on the DCP feed, it will increment the corresponding **principal counter** document in the bucket. -*TODO: is there a potential optimization here, where the Sync Gateway node only updates the **principal counter** if it already exists (so that we're not maintaining counters for users that don't have an active `_changes` feed)? The tricky question there would be how to expire user and role counters once they are no longer in use, without deleting users that are involved in long-running `_changes` requests that have gone quiet, or counters being used by multiple requests across different nodes. It might be possible to use document expirty, and do a touch on the relevant user and role document as part of heartbeat processing, but it still feels like there's a potential for missed user updates.* \ No newline at end of file +*TODO: is there a potential optimization here, where the Sync Gateway node only updates the **principal counter** if it already exists (so that we're not maintaining counters for users that don't have an active `_changes` feed)? The tricky question there would be how to expire user and role counters once they are no longer in use, without deleting users that are involved in long-running `_changes` requests that have gone quiet, or counters being used by multiple requests across different nodes. It might be possible to use document expirty, and do a touch on the relevant user and role document as part of heartbeat processing, but it still feels like there's a potential for missed user updates.* \ No newline at end of file diff --git a/docs/design/channel_index/feed_management.md b/docs/design/channel_index/feed_management.md index f355f2813d..66964dcfdc 100644 --- a/docs/design/channel_index/feed_management.md +++ b/docs/design/channel_index/feed_management.md @@ -4,14 +4,14 @@ Each Sync Gateway node will be assigned a subset of vbuckets - call this set of The number of partitions defined will be a configurable setting, defined for the cluster. The number of partitions should be greater than the number of Sync Gateway nodes initially in the cluster, to support future scale out. More detailed performance testing is required to identify the recommended partition values. -Each Sync Gateway node is given responsibility for one or more partitions. When establishing it's DCP feed, a Sync Gateway node will start streams for each vbucket it is responsible for. +Each Sync Gateway node is given responsibility for one or more partitions. When establishing it's DCP feed, a Sync Gateway node will start streams for each vbucket it is responsible for. ###Milestone 1 -The number of partitions for the cluster will be fixed at 64. Each of the Sync Gateway nodes will be assigned partitions in its config file. For this milestone, partitions will simply be made up of a continuous sequence of vbuckets - e.g. partition 0 represent vbuckets 0 through 15. When the partition assignment is moved out of the Sync Gateway config in future revisions, it will also be possible to define something other than a sequential range of vbuckets as a partition. +The number of partitions for the cluster will be fixed at 64. Each of the Sync Gateway nodes will be assigned partitions in its config file. For this milestone, partitions will simply be made up of a continuous sequence of vbuckets - e.g. partition 0 represent vbuckets 0 through 15. When the partition assignment is moved out of the Sync Gateway config in future revisions, it will also be possible to define something other than a sequential range of vbuckets as a partition. ``` -{"partitions": [0,1,2,3,4,5,6,7,8,9]} +{"partitions": [0,1,2,3,4,5,6,7,8,9]} ``` Each Sync Gateway will start a DCP feed, and start streams for each vbucket in each of their partitions. diff --git a/docs/design/channel_index/index_writes.md b/docs/design/channel_index/index_writes.md index 6f2e516e31..39381fc7ca 100644 --- a/docs/design/channel_index/index_writes.md +++ b/docs/design/channel_index/index_writes.md @@ -10,14 +10,14 @@ As with the current model, events received over the feed will be processed by Sy ####Standard Document Handling -To optimize writes to the index bucket, updates to the index will typically be done in batches. The initial implementation will be a simple loop to process the current batch, and queue entries for the next batch while the previous batch is being processed. +To optimize writes to the index bucket, updates to the index will typically be done in batches. The initial implementation will be a simple loop to process the current batch, and queue entries for the next batch while the previous batch is being processed. -Batch processing steps (in order): +Batch processing steps (in order): 1. Group the updates in the batch by channel 2. For each channel: a. Issue an update to the appropriate **channel block(s)** for the vbucket(s) and channel. - b. Update the **channel clock**, based on the high sequence values for each vbucket in the batch (by channel). + b. Update the **channel clock**, based on the high sequence values for each vbucket in the batch (by channel). c. Increment the **channel counter**. 3. Update the **stable clock**, based on the high sequence values for each vbucket in the batch. @@ -30,8 +30,8 @@ We need to track changes to user and role documents to be able to notify continu ##Milestone 1 Tasks - 1. Abstract interface for index writer, to support config-based switching between existing in-memory cache and channel index. We'll want this ability at least during development for comparing - results of the implementations. The initial implementation is already on the distributed_cache branch. This was intended for POC, though - needs a more thought out refactoring to properly + 1. Abstract interface for index writer, to support config-based switching between existing in-memory cache and channel index. We'll want this ability at least during development for comparing + results of the implementations. The initial implementation is already on the distributed_cache branch. This was intended for POC, though - needs a more thought out refactoring to properly decouple feed processing, sequence management, and index storage. 2. Implement initial implementation for channel index storage @@ -41,4 +41,4 @@ We need to track changes to user and role documents to be able to notify continu ##Milestone 2 Tasks - 1. Add support for removal of + 1. Add support for removal of diff --git a/docs/design/channel_index/overview.md b/docs/design/channel_index/overview.md index 8b6a4907e9..0057eca885 100644 --- a/docs/design/channel_index/overview.md +++ b/docs/design/channel_index/overview.md @@ -11,7 +11,7 @@ * [Index Storage](index_storage.md) * What is stored in the channel index, and how it is stored * [Clocks and Consistency](clock_handling.md) - * How the channel index will manage consistency + * How the channel index will manage consistency * [Channel Index Writes](index_writes.md) * Design for writing to the channel index * [Channel Index Reads](index_reads.md) @@ -63,11 +63,11 @@ ####Feed Partitioning * Auto-assignment of vbucket ranges to SG nodes on startup, based on interaction with shared config document in bucket * Potentially leverage cbft's infrastructure - + ####Cluster Administration - * SG node failover handling. Detect failure, and reassign partitions to other Sync Gateway nodes. + * SG node failover handling. Detect failure, and reassign partitions to other Sync Gateway nodes. -####Index Writes +####Index Writes * Compact/prune channel index entries when revisions are pruned ####Index Reads