Skip to content

Commit

Permalink
db-sync: sync primary data types first & other improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@zetavg
zetavg committed Dec 3, 2023
1 parent 6f9852a commit b4ecf26
Show file tree
Hide file tree
Showing 4 changed files with 160 additions and 60 deletions.
103 changes: 101 additions & 2 deletions App/app/features/db-sync/DBSyncManager.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,20 @@ import useLogger from '@app/hooks/useLogger';

import { DBSyncServerEditableData } from './slice';

const SYNC_FILTER_DDOC_NAME = 'app_sync_v0';
const SYNC_ONLY_PRIMARY_FILTER_NAME = 'only_primary';
const SYNC_ONLY_IMAGES_FILTER_NAME = 'only_images';

const SYNC_FILTER_DDOC = {
_id: `_design/${SYNC_FILTER_DDOC_NAME}`,
filters: {
[SYNC_ONLY_PRIMARY_FILTER_NAME]:
"function (doc) { return !doc._id.startsWith('zz'); }",
[SYNC_ONLY_IMAGES_FILTER_NAME]:
"function (doc) { return doc._id.startsWith('zz20-image'); }",
},
};

const BATCH_SIZE = 16;
const BATCHES_LIMIT = 4;

Expand Down Expand Up @@ -137,6 +151,24 @@ export default function DBSyncManager() {
return null;
}
}

let createSyncFilterDDocRetries = 0;
while (true) {
try {
await remoteDB.get(`_design/${SYNC_FILTER_DDOC_NAME}`);
break;
} catch (e) {
if (createSyncFilterDDocRetries >= 5) throw e;
try {
await remoteDB.put(SYNC_FILTER_DDOC);
} catch (ee) {
if (createSyncFilterDDocRetries >= 5) throw ee;
await new Promise(resolve => setTimeout(resolve, 1000));
}
createSyncFilterDDocRetries += 1;
}
}

return remoteDB;
} catch (e) {
fLogger.error(
Expand Down Expand Up @@ -225,9 +257,11 @@ export default function DBSyncManager() {
params: PouchDB.Replication.SyncOptions,
server: ServerData,
{
filter,
onChange,
onComplete,
}: {
filter?: string;
onChange?: (arg: {
localDBUpdateSeq?: number | undefined;
remoteDBUpdateSeq?: number | undefined;
Expand All @@ -247,6 +281,7 @@ export default function DBSyncManager() {
const syncHandler = localDB.sync(remoteDB, {
...params,
retry: true,
filter,
// The patched PouchDB will accept a logger for logging during the sync
...({ logger: fLogger } as any),
});
Expand All @@ -256,21 +291,27 @@ export default function DBSyncManager() {
const lastSeq = getSeqValue(info.change.last_seq);
let localDBUpdateSeq;
let remoteDBUpdateSeq;
let localDBDocCount;
let remoteDBDocCount;
const logDetails1: any = { info, lastSeq };
const logDetails2: any = {};
try {
const localDBInfo = await localDB.info();
localDBUpdateSeq = getSeqValue(localDBInfo.update_seq);
localDBDocCount = localDBInfo.doc_count;
logDetails2.localDBInfo = localDBInfo;
logDetails1.localDBUpdateSeq = localDBUpdateSeq;
logDetails1.localDBDocCount = localDBDocCount;
} catch (e) {
logDetails1.localDBInfoError = e;
}
try {
const remoteDBInfo = await remoteDB.info();
remoteDBUpdateSeq = getSeqValue(remoteDBInfo.update_seq);
remoteDBDocCount = remoteDBInfo.doc_count;
logDetails2.remoteDBInfo = remoteDBInfo;
logDetails1.remoteDBUpdateSeq = remoteDBUpdateSeq;
logDetails1.remoteDBDocCount = remoteDBDocCount;
} catch (e) {
logDetails1.remoteDBInfoError = e;
}
Expand All @@ -279,6 +320,8 @@ export default function DBSyncManager() {
// Only update the local or remote seq based on the current operation
...(direction === 'push' ? { localDBUpdateSeq } : {}),
...(direction === 'pull' ? { remoteDBUpdateSeq } : {}),
localDBDocCount,
remoteDBDocCount,
[direction === 'push' ? 'pushLastSeq' : 'pullLastSeq']: lastSeq,
};
updateSyncProgress([server.id, payload]);
Expand Down Expand Up @@ -418,6 +461,7 @@ export default function DBSyncManager() {
}
if (shouldCancel) return;
dispatch(actions.dbSync.updateServerStatus([server.id, 'Syncing']));

startupSyncHandler = _startSync(
localDB,
remoteDB,
Expand All @@ -428,6 +472,7 @@ export default function DBSyncManager() {
},
server,
{
filter: `${SYNC_FILTER_DDOC_NAME}/${SYNC_ONLY_PRIMARY_FILTER_NAME}`,
onChange: assignSeqs,
onComplete: payload => {
assignSeqs(payload);
Expand All @@ -436,6 +481,60 @@ export default function DBSyncManager() {
},
},
);
await new Promise(resolve => {
startupSyncHandler?.on('complete', resolve);
});
logger.info('Start-up sync: primary data synced', {
function: server.id,
});
if (shouldCancel) return;

startupSyncHandler = _startSync(
localDB,
remoteDB,
{
live: false,
batch_size: 1,
batches_limit: 2,
},
server,
{
filter: `${SYNC_FILTER_DDOC_NAME}/${SYNC_ONLY_IMAGES_FILTER_NAME}`,
onChange: assignSeqs,
onComplete: payload => {
assignSeqs(payload);
initialPullLastSeq = payload.pushLastSeq;
initialPullLastSeq = payload.pullLastSeq;
},
},
);
await new Promise(resolve => {
startupSyncHandler?.on('complete', resolve);
});
logger.info('Start-up sync: images synced', {
function: server.id,
});
if (shouldCancel) return;

startupSyncHandler = _startSync(
localDB,
remoteDB,
{
live: false,
batch_size: BATCH_SIZE,
batches_limit: BATCHES_LIMIT,
},
server,
{
onChange: assignSeqs,
onComplete: payload => {
assignSeqs(payload);
initialPullLastSeq = payload.pushLastSeq;
initialPullLastSeq = payload.pullLastSeq;
},
},
);

startupSyncHandler.on('complete', info => {
if (shouldCancel) return;
if (
Expand Down Expand Up @@ -465,8 +564,8 @@ export default function DBSyncManager() {
remoteDB,
{
live: true,
batch_size: BATCH_SIZE,
batches_limit: BATCHES_LIMIT,
batch_size: 2,
batches_limit: 2,
},
server,
{
Expand Down
59 changes: 42 additions & 17 deletions App/app/features/db-sync/screens/DBSyncServerDetailScreen.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ function DBSyncServerDetailScreen({
});
}, []);

const [showAdvancedStatus, setShowAdvancedStatus] = useState(false);

return (
<ScreenContent
navigation={navigation}
Expand Down Expand Up @@ -214,26 +216,49 @@ function DBSyncServerDetailScreen({
}}
/>
</UIGroup>
<UIGroup header="Advanced">
<UIGroup>
<UIGroup.ListItem
label="Push Sequence"
adjustsDetailFontSizeToFit
detail={
typeof serverStatus.pushLastSeq === 'number'
? `${serverStatus.pushLastSeq}/${serverStatus.localDBUpdateSeq}`
: 'N/A'
}
label="Docs"
detail={`${
typeof serverStatus.localDBDocCount === 'number'
? serverStatus.localDBDocCount
: '?'
}/${
typeof serverStatus.remoteDBDocCount === 'number'
? serverStatus.remoteDBDocCount
: '?'
}`}
/>
<UIGroup.ListItemSeparator />
<UIGroup.ListItem
label="Pull Sequence"
adjustsDetailFontSizeToFit
detail={
typeof serverStatus.pullLastSeq === 'number'
? `${serverStatus.pullLastSeq}/${serverStatus.remoteDBUpdateSeq}`
: 'N/A'
}
/>
{!showAdvancedStatus ? (
<UIGroup.ListItem
label="Show Advanced Status"
onPress={() => setShowAdvancedStatus(true)}
button
/>
) : (
<>
<UIGroup.ListItem
label="Push Sequence"
adjustsDetailFontSizeToFit
detail={
typeof serverStatus.pushLastSeq === 'number'
? `${serverStatus.pushLastSeq}/${serverStatus.localDBUpdateSeq}`
: 'N/A'
}
/>
<UIGroup.ListItemSeparator />
<UIGroup.ListItem
label="Pull Sequence"
adjustsDetailFontSizeToFit
detail={
typeof serverStatus.pullLastSeq === 'number'
? `${serverStatus.pullLastSeq}/${serverStatus.remoteDBUpdateSeq}`
: 'N/A'
}
/>
</>
)}
</UIGroup>
<UIGroup>
<UIGroup.ListItem
Expand Down
6 changes: 6 additions & 0 deletions App/app/features/db-sync/slice.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ export type DBSyncServerStatusObj = {
remoteDBUpdateSeq?: number;
pushLastSeq?: number;
pullLastSeq?: number;
localDBDocCount?: number;
remoteDBDocCount?: number;
};

// Define a type for the slice state
Expand Down Expand Up @@ -159,6 +161,8 @@ export const dbSyncSlice = createSlice({
remoteDBUpdateSeq?: number;
pushLastSeq?: number;
pullLastSeq?: number;
localDBDocCount?: number;
remoteDBDocCount?: number;
},
]
>,
Expand Down Expand Up @@ -311,6 +315,8 @@ reducer.dehydrateSensitive = (state: DBSyncState) => {
remoteDBUpdateSeq: server.remoteDBUpdateSeq,
pushLastSeq: server.pushLastSeq,
pullLastSeq: server.pullLastSeq,
localDBDocCount: server.localDBDocCount,
remoteDBDocCount: server.remoteDBDocCount,
}),
),
};
Expand Down
Loading

0 comments on commit b4ecf26

Please sign in to comment.