WIP: Adapter architecture

README.md

@@ -6,8 +6,8 @@ Built to answer the question, "how can I have a queryable, browsable db that als
 
 ## Goals
 
-- **Simple**: less than 500 lines. Only one dependency, `ioredis`. You can copy-paste the code instead if you want.
-- **Serverless-friendly**: no Redis modules, only core Redis.
+- **Simple**: less than 500 lines, and no dependencies. You can copy-paste the code instead if you want.
+- **Serverless-friendly**: no Redis modules, only core Redis. Multiple Redis clients supported, including REST-based ones.
 - **Fast**: Compared to optimized Postgres, 150% faster at paginating unindexed data. See [all benchmarks](#benchmarks) below.
 - **Indexable**: Supports hierarchical [tags](#tags), a lightweight primitive for indexing your data.
 - **Browsable**: [browser-friendly API](#example-browsing-your-data) included for paginating and browsing by tag.
@@ -37,7 +37,10 @@ Exploration API:
 - [Redbase](#redbase)
   - [Goals](#goals)
   - [Install](#install)
+    - [Redis Client Compatibility](#redis-client-compatibility)
   - [Usage](#usage)
+    - [One-to-Many Relationships](#one-to-many-relationships)
+    - [Many-to-Many Relationships](#many-to-many-relationships)
   - [Core concepts](#core-concepts)
     - [Entries](#entries)
     - [Tags](#tags)
@@ -47,6 +50,8 @@ Exploration API:
     - [For the cache use-case](#for-the-cache-use-case)
     - [For the database use-case](#for-the-database-use-case)
     - [Results](#results)
+  - [Contributing](#contributing)
+    - [Writing Adapters](#writing-adapters)
   - [License](#license)
 
 ## Install
@@ -55,6 +60,13 @@ Exploration API:
 npm install redbase
 ```
 
+### Redis Client Compatibility
+
+Redbase can support arbitrary Redis clients through the use of [custom adapters](#writing-adapters). The current clients supported are:
+
+- [ioredis](https://github.com/luin/ioredis)
+- [redis](https://github.com/redis/node-redis)
+
 ## Usage
 
 ```ts
@@ -68,21 +80,21 @@ type MyValue = {
   }
 }
 
-// Options can also use your own ioredis instance if already defined,
-// as `redisInstance`
+// Options can also use your own Redis instance if already defined,
+// as `redis`
 const db = new Redbase<MyValue>('myProject', { redisUrl: 'redis://...' })
 
 const key = uuid()
 const value = { a: 'result' }
 
 await db.get(key) // undefined
 
-await db.set(key, value)
+await db.save(key, value)
 
 await db.get(key) // value
 
 // Type safety!
-await db.set(key, { c: 'result2' }) // Type error on value
+await db.save(key, { c: 'result2' }) // Type error on value
 
 // Browsing results
 let data = await db.filter()
@@ -92,9 +104,9 @@ assertEqual(await db.count(), 1)
 // Hierarchical indexes, using a customizable tag separator (default: '/')
 await Promise.all([
   // Redis auto-pipelines these calls into one fast request!
-  db.set(uuid(), { a: 'hi' }, { tags: ['user1/project1'] }),
-  db.set(uuid(), { a: 'there' }, { tags: ['user1/project2'] }),
-  db.set(uuid(), { a: 'bye' }, { tags: ['user2/project1'] })
+  db.save(uuid(), { a: 'hi' }, { tags: ['user1/project1'] }),
+  db.save(uuid(), { a: 'there' }, { tags: ['user1/project2'] }),
+  db.save(uuid(), { a: 'bye' }, { tags: ['user2/project1'] })
 ])
 
 data = await db.filter()
@@ -108,8 +120,8 @@ assertEqual(data.length, 2)
 data = await db.filter({ where: {OR: ['user1', 'user2']}})
 assertEqual(data.length, 3)
 
-const count = await db.count({ where: {OR: ['user1', 'user2']}})
-assertEqual(count, 3)
+const count = await db.count({ where: {AND: ['user1', 'user2']}})
+assertEqual(count, 0)
 
 // See all your indexes:
 const tags = await db.tags("user1")
@@ -120,6 +132,75 @@ const numberDeleted = await db.clear({ where: 'user2' })
 assertEqual(numberDeleted, 1)
 ```
 
+### One-to-Many Relationships
+
+Let's say you have User and Post interfaces that look like this:
+```ts
+interface User {
+  id: number
+  name: string
+}
+interface Post {
+  content: string
+  userId: number
+}
+```
+
+Now we want to store this data in Redbase. Rather than store all posts in an array inside of each `User` (which you might do in a NoSQL database), you can simply create separate Redbase clients for the two interfaces:
+
+```ts
+// Slightly more efficient to share the same redis instance:
+import { IORedis } from 'redbase'
+const redis = new IORedis()
+const users = new Redbase<User>('myProject-user', { redis })
+const posts = new Redbase<Post>('myProject-post', { redis })
+```
+
+When inserting a new Post for a given `userId`, make sure you tag it:
+
+```ts
+await posts.save(uuid(), post, { tags: [`user-${userId}`] })
+```
+
+Now we can query all posts for a given user:
+
+```ts
+const userPosts = await posts.filter({ where: `user-${userId}`, limit: 100 })
+```
+
+Note that tags are simple and don't have contraints; if the user's id changes, you have to re-save each of its posts with the new tag.
+
+### Many-to-Many Relationships
+
+Let's say you have Posts and Categories that look like this:
+
+```ts
+interface Post {
+  id: number
+}
+interface Category {
+  name: string
+}
+```
+
+For many-to-many relationships, e.g. `Post` <-> `Category`, you can make the save atomic inside a Redis transaction by calling `multiSet` instead of `save`:
+
+```ts
+const redis = new IORedis()
+const categories = new Redbase<Category>('myProject-category', { redis })
+const posts = new Redbase<Post>('myProject-post', { redis })
+
+const post = { id, ... }
+const categoryNames = ['tech', 'draft', ...]
+
+let multi = redis.multi()
+multi = posts.multiSet(multi, post.postId, post, { tags: categoryNames })
+for (const name of categoryNames) {
+  multi = categories.multiSet(name, { name }, { tags: [`post-${post.id}`] })
+}
+await multi.exec()
+```
+
 For all functionality, see `test/database.spec.ts`.
 
 ## Core concepts
@@ -135,18 +216,19 @@ An entry is composed of an `id` and a `value`:
 
 ### Tags
 
-Tags are a lightweight primitive for indexing your values. You attach them at insert-time, and they are schemaless. This makes them simple and flexible for many use cases. It also allows you to index values by external attributes (that aren't a part of the value itself: [example](#example-prompt-cache)).
+Tags are a lightweight primitive for indexing your values. You attach them at insert-time, and they are schemaless. This makes them simple and flexible for many use cases:
 
-Calling `db.filter({ where: { AND: [...]}})` etc. allows you to compose them together as a list.
+- Tags allow you to index values by external attributes (that aren't a part of the value itself: [example](#example-prompt-cache)).
+- Tags allow you to link values between Redbase instances in [one-to-many relationships](#one-to-many-relationships), or even have [many-to-many relationships](#many-to-many-relationships) between data types. You can shard data between multiple Redis instances this way.
+- Tags can be nested with a chosen separator, e.g. `parentindex/childindex`. This effectively allows you to group your indexes, and makes [browsing](#example-browsing-your-data) your data easier and more fun.
 
-Tags are sort of self-cleaning: indexes delete themselves during bulk-delete operations, and they shrink when entries are deleted individually. However, when the last entry for an index is deleted, the index is not. This shouldn't cause a significant zombie index issue unless you're creating and wiping out an unbounded number of tags.
+Calling `db.filter({ where: { AND: [...]}})` etc. allows you to intersect tags together, and doing the same with `OR` allows you to union them.
 
-Tags can get unruly, you you can keep them organized by nesting them:
-`parentindex/childindex`. This effectively allows you to group your indexes, and makes [browsing](#example-browsing-your-data) your data easier and more fun.
+Call `db.tags(parentIndex)` to get all the children tags of `parentIndex`.
 
-Call `db.tags("parentindex")` to get all the children tags.
+Tags are sort of self-cleaning: indexes delete themselves during bulk-delete operations, and they shrink when entries are deleted individually. However, when the last entry for an index is deleted, the index is not. This shouldn't cause a significant zombie index issue unless you're creating and wiping out an unbounded number of tags.
 
-As you might expect, when indexing an entry under `parentindex/childindex` the entry is automatically indexed under `parentindex` as well. This makes it easy to build a url-based [cache exploration server](#example-browsing-your-data). Call `db.filter({ where: 'parentindex' })` to get all entries for all children tags.
+Tags can get unruly, so you can keep them organized by nesting them. As you might expect, when indexing an entry under `parentindex/childindex`, the entry is automatically indexed under `parentindex` as well. This makes it easy to build a url-based [cache exploration server](#example-browsing-your-data). Call `db.filter({ where: parentIndex })` to get all entries for all children tags under `parentIndex`.
 
 
 ### Example: Prompt Cache
@@ -193,7 +275,7 @@ To browse, paginate, filter, and delete your data directly from a browser, just
 
 **Note:** I'm very new to benchmarking open-sourced code, and would appreciate pull requests here! One issue, for example, is that increasing the number of runs can cause the data to scale up (depending on which benchmarks you're running), which seems to e.g. make Redis win on pagination by a larger margin.
 
-This project uses [hyperfine](https://github.com/sharkdp/hyperfine) to compare Redis in a persistent mode with Postgres in an optimized mode. **Yes, this is comparing apples to oranges.** I decided to do it anyway because:
+These benchmarks use [hyperfine](https://github.com/sharkdp/hyperfine) and compare Redis in a persistent mode with Postgres in an optimized mode. Yes, this is still comparing apples to oranges in a sense, but I decided to do it anyway because:
 
 1. A big question this project answers is "how can I have a queryable, browsable db that also works well as a cache?" Redis and Postgres are two backend choices that pop up frequently.
 
@@ -219,16 +301,24 @@ Comment-out the call to `ALTER DATABASE ... SET synchronous_commit=OFF;` in `/be
 
 ### Results
 - **Inserting data**: Tie
-- **Paginating unindexed data**: Redis is ~150% faster
+- **Paginating unindexed data**: Redbase is ~150% faster
 - **Single-index pagination**: Postgres is ~50% faster
 - **Joint-index pagination**: Postgres is ~60% faster
-- **Inserting and deleting data**: Redis is ~25% faster
+- **Inserting and deleting data**: Redbase is ~25% faster
 
 Results on Apple M1 Max, 2021:
 ![Insert and scroll](files/insert_and_scroll.png)
 ![Scroll along an index](files/index_scrolling.png)
 ![Delete data](files/deleting.png)
 
+## Contributing
+
+See open pull requests and issues on [Github](https://github.com/alexanderatallah/redbase).
+
+### Writing Adapters
+
+You can add support for new [Redis clients](#redis-client-compatibility) by writing an adapter. Adapters are located in `src/adapters/`. It's easy to duplicate one and adjust.
+
 ## License
 MIT
 

src/adapters/base.ts

@@ -0,0 +1,49 @@
+export const defaultErrorHandler = (err: unknown): void | never => {
+  // Screetching halt failure by default
+  throw err
+}
+
+export type RawValue = string | number | Buffer
+export type AggregationMode = 'SUM' | 'MIN' | 'MAX'
+export type OrderingMode = 'ASC' | 'DESC'
+export type Score = number | '-inf' | '+inf'
+
+export abstract class RedisMultiAdapter {
+  abstract set(key: string, value: RawValue): RedisMultiAdapter
+  abstract expire(key: string, ttl: number): RedisMultiAdapter
+  abstract sadd(key: string, values: RawValue[]): RedisMultiAdapter
+  abstract zadd(
+    key: string,
+    scores: Score[],
+    members: RawValue[]
+  ): RedisMultiAdapter
+  abstract exec(): Promise<void>
+  abstract del(keys: string[]): RedisMultiAdapter
+  abstract zrem(key: string, values: RawValue[]): RedisMultiAdapter
+  abstract zunionstore(
+    destination: string,
+    keys: string[],
+    aggregate?: AggregationMode
+  ): RedisMultiAdapter
+  abstract zinterstore(
+    destination: string,
+    keys: string[],
+    aggregate?: AggregationMode
+  ): RedisMultiAdapter
+}
+
+export abstract class RedisAdapter {
+  abstract multi(): RedisMultiAdapter
+  abstract quit(): Promise<void>
+  abstract ttl(key: string): Promise<number>
+  abstract smembers(key: string): Promise<string[]>
+  abstract zcount(key: string, min?: Score, max?: Score): Promise<number>
+  abstract zrange(
+    key: string,
+    start: number,
+    stop: number,
+    order?: OrderingMode
+  ): Promise<string[]>
+  abstract get(key: string): Promise<string | null>
+  abstract del(keys: string[]): Promise<number>
+}