Skip to content

Commit

Permalink
fix data-storage-couchdb to work for remote CouchDB
Browse files Browse the repository at this point in the history
  • Loading branch information
@zetavg
zetavg committed Nov 18, 2023
1 parent 66159b2 commit d8674c0
Show file tree
Hide file tree
Showing 6 changed files with 133 additions and 18 deletions.
18 changes: 18 additions & 0 deletions packages/data-storage-couchdb/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,21 @@ await saveDatum({ __type: 'collection', __id: newCollection.__id, __deleted: tru
getREPL().removeListener('SIGINT', onSEGINT);
})();
```

## Run Test Suits

```bash
yarn test
```

### Run data-storage-couchdb test with debug logging

```bash
DEBUG=true yarn test lib/__tests__/data-storage-couchdb.test.ts
```

### Run data-storage-couchdb test with remote database

```bash
COUCHDB_URI='http://127.0.0.1:5984/<db_name>' COUCHDB_USERNAME=<username> COUCHDB_PASSWORD=<password> yarn test lib/__tests__/data-storage-couchdb.test.ts --runInBand
```
69 changes: 60 additions & 9 deletions packages/data-storage-couchdb/lib/__tests__/data-storage-couchdb.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ const SQLiteAdapter = require('pouchdb-adapter-react-native-sqlite/lib')({
});
PouchDB.plugin(SQLiteAdapter);

const DEBUG = process.env.DEBUG;
const COUCHDB_URI = process.env.COUCHDB_URI;
const COUCHDB_USERNAME = process.env.COUCHDB_USERNAME;
const COUCHDB_PASSWORD = process.env.COUCHDB_PASSWORD;

const contextSet = new Set();
const getContextID = () => {
let id = 1;
Expand All @@ -31,25 +36,51 @@ const getContextID = () => {
const releaseContextID = (id: number) => {
contextSet.delete(id);
};

async function withContext(fn: (c: Context) => Promise<void>) {
const contextID = getContextID();
const db = new PouchDB(`.temp_dbs/pouchdb-test-${contextID}`, {
adapter: 'react-native-sqlite',
});
const useRemoteDB = !!(COUCHDB_URI && COUCHDB_USERNAME && COUCHDB_PASSWORD);
const db = useRemoteDB
? new PouchDB(COUCHDB_URI, {
skip_setup: true,
auth: {
username: COUCHDB_USERNAME,
password: COUCHDB_PASSWORD,
},
})
: new PouchDB(`.temp_dbs/pouchdb-test-${contextID}`, {
adapter: 'react-native-sqlite',
});

const context: Context = {
dbType: 'pouchdb',
db,
logger: null,
logLevels: () => [],
logger: DEBUG ? console : null,
logLevels: DEBUG ? () => ['debug'] : () => [],
alwaysCreateIndexFirst: useRemoteDB,
};

try {
if (useRemoteDB) {
await db
.allDocs({ include_docs: true })
.then(allDocs => {
return allDocs.rows.map(row => {
return { _id: row.id, _rev: row.doc?._rev, _deleted: true };
});
})
.then(deleteDocs => {
return db.bulkDocs(deleteDocs);
});
}

const d = new CouchDBData(context);
await d.updateConfig({});
await fn(context);
} finally {
await db.destroy();
if (!useRemoteDB) {
await db.destroy();
}
releaseContextID(contextID);
}
}
Expand Down Expand Up @@ -157,6 +188,7 @@ describe('getData', () => {
{},
{ limit: 3 },
);

expect(collectionsWithLimit).toHaveLength(3);
expect(collectionsWithLimit.map(c => c.name)).toEqual(
Array.from(new Array(3)).map((_, i) => `Collection #${i + 1}`),
Expand Down Expand Up @@ -536,6 +568,18 @@ describe('getData', () => {
Array.from(new Array(5)).map((_, i) => `Item #${i + 6}`),
),
);

const itemsCreatedAfter0 = await d.getData('item', {
__created_at: { $gt: 0 },
});
expect(
itemsCreatedAfter0.every(it => (it.__raw as any)?.type === 'item'),
).toEqual(true);

const collectionsCreatedAfter0 = await d.getData('collection', {
__created_at: { $gt: 0 },
});
expect(collectionsCreatedAfter0.length).toEqual(10);
});
});

Expand Down Expand Up @@ -629,11 +673,18 @@ describe('getData', () => {
const items_b = await d.getData(
'item',
{ model_name: 'Model B' },
{ sort: [{ purchase_price_x1000: 'desc' }] },
{
sort: [
// { model_name: 'desc' }, // To prevent CouchDB error - { "error": "unsupported_mixed_sort", "reason": "Sorts currently only support a single direction for all fields.", "status": 400 } // This is now handled in getData
{ purchase_price_x1000: 'desc' },
],
},
);
expect(items_b).toHaveLength(5);
expect(items_b.map(it => it.name)).toEqual(
Array.from(new Array(5)).map((_, i) => `Item #${i + 6}`),
Array.from(new Array(5))
.map((_, i) => `Item #${i + 6}`)
.reverse(),
);
});
});
Expand Down Expand Up @@ -1197,7 +1248,7 @@ describe('saveDatum', () => {
});
const doc = await (context.db as any).get('collection-1');

(context.db as any).put({
await (context.db as any).put({
...doc,
additional_field: 'hello',
});
Expand Down
52 changes: 45 additions & 7 deletions packages/data-storage-couchdb/lib/functions/getGetData.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,13 @@ import { Context } from './types';
* We will need to update this if we change the auto ddoc generation logic, so
* that the app can generate the new and updated design docs.
*/
const AUTO_DDOC_PREFIX = 'auto_get_data';
const AUTO_DDOC_PREFIX = 'auto_get_data_v1';

export default function getGetData({
db,
logger,
logLevels,
alwaysCreateIndexFirst,
}: Context): GetData {
const getData: GetData = async function getData(
type,
Expand Down Expand Up @@ -151,6 +152,8 @@ export default function getGetData({
);

const sortKeys = (couchdbSort || []).flatMap(s => Object.keys(s));
// CouchDB limitation: sorts currently only support a single direction for all fields
const shouldUseDesc = couchdbSort?.[0]?.[sortKeys[0]] === 'desc';

const seenIndexFieldsSet = new Set();
const indexFields = [
Expand All @@ -166,12 +169,18 @@ export default function getGetData({

const ddocName_ =
indexFields.length > 0
? `${AUTO_DDOC_PREFIX}--type_${type}--${indexFields.join('-')}`
? `${AUTO_DDOC_PREFIX}--type_${type}--${indexFields.join('-')}${
// shouldUseDesc ? '--desc' : ''
''
}`
: `${AUTO_DDOC_PREFIX}--type`;
const index_ =
indexFields.length > 0
? {
fields: [...indexFields],
// fields: ['type', ...indexFields].map(f =>
// shouldUseDesc ? { [f]: 'desc' } : { [f]: 'asc' },
// ),
fields: ['type', ...indexFields],
partial_filter_selector: { type },
}
: {
Expand All @@ -180,7 +189,7 @@ export default function getGetData({
const query_ = {
use_index: ddocName_,
selector: {
...(indexFields.length <= 0 ? { type } : {}),
type,
...flattenedSelector,
},
sort: couchdbSort
Expand All @@ -192,9 +201,11 @@ export default function getGetData({
couchdbSort.flatMap(s => Object.keys(s)),
);
return [
...indexFields
...['type', ...indexFields]
.filter(f => !sortKeysSet.has(f))
.map(k => ({ [k]: 'asc' as const })),
.map(k => ({
[k]: shouldUseDesc ? ('desc' as const) : ('asc' as const),
})),
...couchdbSort,
];
})()
Expand All @@ -211,17 +222,44 @@ export default function getGetData({
}
})();

// Since remote CouchDB server may not throw error if index not found
if (alwaysCreateIndexFirst && ddocName && index) {
try {
await db.createIndex({
ddoc: ddocName,
name: ddocName,
index,
});
} catch (e) {}
}

if (logger && logDebug) {
let explain = '';
try {
explain = `, explain: ${JSON.stringify(
await ((db as any).explain as any)(query),
null,
2,
)}`;
} catch (e) {
explain = `, explain: error on getting explain - ${
e instanceof Error ? e.message : JSON.stringify(e)
}`;
}
logger.debug(
`getData query: ${JSON.stringify(
query,
null,
2,
)}, index: ${JSON.stringify(index, null, 2)}`,
)}, index: ${JSON.stringify(index, null, 2)}` + explain,
);
}

const response = await (async () => {
if (alwaysCreateIndexFirst) {
return await db.find(query);
}

let retries = 0;
while (true) {
try {
Expand Down
3 changes: 2 additions & 1 deletion packages/data-storage-couchdb/lib/functions/getGetDatum.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ export default function getGetDatum({
if (
e.message === 'not_found' /* nano */ ||
e.message ===
'missing' /* pouchdb, note that `e instanceof Error` will be false */
'missing' /* pouchdb, note that `e instanceof Error` will be false */ ||
e.name === 'not_found' /* also pouchdb */
) {
return null;
}
Expand Down
4 changes: 4 additions & 0 deletions packages/data-storage-couchdb/lib/functions/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,16 @@ export type Context =
db: nano.DocumentScope<unknown>;
logger?: Logger | null;
logLevels?: () => ReadonlyArray<string>;
/** Since remote CouchDB server may not throw error if index not found */
alwaysCreateIndexFirst?: boolean;
}
| {
dbType: 'pouchdb';
db: PouchDB.Database;
logger?: Logger | null;
logLevels?: () => ReadonlyArray<string>;
/** Since remote CouchDB server may not throw error if index not found */
alwaysCreateIndexFirst?: boolean;
};

export type CouchDBDoc = {
Expand Down
5 changes: 4 additions & 1 deletion packages/data-storage-couchdb/repl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ program
},
'nano',
)
.option('--ignore_config_missing_error', 'Ignore config missing error.')
.option('--debug', 'Enable debug output.');

program.parse(process.argv);
Expand Down Expand Up @@ -185,7 +186,9 @@ getPassword(async () => {
})();

try {
await (db as any).get('0000-config');
if (!options.ignore_config_missing_error) {
await (db as any).get('0000-config');
}
} catch (e) {
if (e instanceof Error && e.message === 'missing') {
console.warn(
Expand Down

0 comments on commit d8674c0

Please sign in to comment.