Sync progress

.changeset/sharp-singers-search.md

@@ -0,0 +1,8 @@
+---
+'@powersync/common': minor
+'@powersync/web': minor
+'@powersync/node': minor
+'@powersync/react-native': minor
+---
+
+Report progress information about downloaded rows. Sync progress is available through `SyncStatus.downloadProgress`.

demos/django-react-native-todolist/package.json

@@ -11,7 +11,7 @@
     "@azure/core-asynciterator-polyfill": "^1.0.2",
     "@expo/metro-runtime": "^4.0.1",
     "@expo/vector-icons": "^14.0.0",
-    "@journeyapps/react-native-quick-sqlite": "^2.4.3",
+    "@journeyapps/react-native-quick-sqlite": "^2.4.4",
     "@powersync/common": "workspace:*",
     "@powersync/react": "workspace:*",
     "@powersync/react-native": "workspace:*",

demos/react-native-supabase-group-chat/package.json

@@ -22,7 +22,7 @@
     "@azure/core-asynciterator-polyfill": "^1.0.2",
     "@expo/metro-runtime": "^4.0.1",
     "@faker-js/faker": "8.3.1",
-    "@journeyapps/react-native-quick-sqlite": "^2.4.3",
+    "@journeyapps/react-native-quick-sqlite": "^2.4.4",
     "@powersync/common": "workspace:*",
     "@powersync/react": "workspace:*",
     "@powersync/react-native": "workspace:*",

demos/react-native-supabase-todolist/app/views/todos/lists.tsx

@@ -7,16 +7,16 @@ import prompt from 'react-native-prompt-android';
 import { router, Stack } from 'expo-router';
 import { LIST_TABLE, TODO_TABLE, ListRecord } from '../../../library/powersync/AppSchema';
 import { useSystem } from '../../../library/powersync/system';
-import { useQuery, useStatus } from '@powersync/react-native';
+import { useQuery } from '@powersync/react-native';
 import { ListItemWidget } from '../../../library/widgets/ListItemWidget';
+import { GuardBySync } from '../../../library/widgets/GuardBySync';
 
 const description = (total: number, completed: number = 0) => {
   return `${total - completed} pending, ${completed} completed`;
 };
 
 const ListsViewWidget: React.FC = () => {
   const system = useSystem();
-  const status = useStatus();
   const { data: listRecords } = useQuery<ListRecord & { total_tasks: number; completed_tasks: number }>(`
       SELECT
         ${LIST_TABLE}.*, COUNT(${TODO_TABLE}.id) AS total_tasks, SUM(CASE WHEN ${TODO_TABLE}.completed = true THEN 1 ELSE 0 END) as completed_tasks
@@ -78,26 +78,26 @@ const ListsViewWidget: React.FC = () => {
           );
         }}
       />
-      <ScrollView key={'lists'} style={{ maxHeight: '90%' }}>
-        {!status.hasSynced ? (
-          <Text>Busy with sync...</Text>
-        ) : (
-          listRecords.map((r) => (
-            <ListItemWidget
-              key={r.id}
-              title={r.name}
-              description={description(r.total_tasks, r.completed_tasks)}
-              onDelete={() => deleteList(r.id)}
-              onPress={() => {
-                router.push({
-                  pathname: 'views/todos/edit/[id]',
-                  params: { id: r.id }
-                });
-              }}
-            />
-          ))
-        )}
-      </ScrollView>
+      <GuardBySync>
+        <ScrollView key={'lists'} style={{ maxHeight: '90%' }}>
+          {(
+            listRecords.map((r) => (
+              <ListItemWidget
+                key={r.id}
+                title={r.name!}
+                description={description(r.total_tasks, r.completed_tasks)}
+                onDelete={() => deleteList(r.id)}
+                onPress={() => {
+                  router.push({
+                    pathname: 'views/todos/edit/[id]',
+                    params: { id: r.id }
+                  });
+                }}
+              />
+            ))
+          )}
+        </ScrollView>
+      </GuardBySync>
 
       <StatusBar style={'light'} />
     </View>

demos/react-native-supabase-todolist/library/widgets/GuardBySync.tsx

@@ -0,0 +1,40 @@
+import { useStatus } from '@powersync/react';
+import { FC, ReactNode } from 'react';
+import { View } from 'react-native';
+import { Text, LinearProgress } from '@rneui/themed';
+
+/**
+ * A component that renders its child if the database has been synced at least once and shows
+ * a progress indicator otherwise.
+ */
+export const GuardBySync: FC<{ children: ReactNode; priority?: number }> = ({ children, priority }) => {
+  const status = useStatus();
+
+  const hasSynced = priority == null ? status.hasSynced : status.statusForPriority(priority).hasSynced;
+  if (hasSynced) {
+    return children;
+  }
+
+  // If we haven't completed a sync yet, show a progress indicator!
+  const allProgress = status.downloadProgress;
+  const progress = priority == null ? allProgress : allProgress?.untilPriority(priority);
+
+  return (
+    <View>
+      {progress != null ? (
+        <>
+          <LinearProgress variant="determinate" value={progress.downloadedFraction} />
+          {progress.downloadedOperations == progress.totalOperations ? (
+            <Text>Applying server-side changes</Text>
+          ) : (
+            <Text>
+              Downloaded {progress.downloadedOperations} out of {progress.totalOperations}.
+            </Text>
+          )}
+        </>
+      ) : (
+        <LinearProgress variant="indeterminate" />
+      )}
+    </View>
+  );
+};

demos/react-native-supabase-todolist/package.json

@@ -10,7 +10,7 @@
   "dependencies": {
     "@azure/core-asynciterator-polyfill": "^1.0.2",
     "@expo/vector-icons": "^14.0.3",
-    "@journeyapps/react-native-quick-sqlite": "^2.4.3",
+    "@journeyapps/react-native-quick-sqlite": "^2.4.4",
     "@powersync/attachments": "workspace:*",
     "@powersync/common": "workspace:*",
     "@powersync/react": "workspace:*",

demos/react-supabase-todolist/src/app/views/todo-lists/page.tsx

@@ -18,6 +18,7 @@ import { LISTS_TABLE } from '@/library/powersync/AppSchema';
 import { NavigationPage } from '@/components/navigation/NavigationPage';
 import { SearchBarWidget } from '@/components/widgets/SearchBarWidget';
 import { TodoListsWidget } from '@/components/widgets/TodoListsWidget';
+import { GuardBySync } from '@/components/widgets/GuardBySync';
 
 export default function TodoListsPage() {
   const powerSync = usePowerSync();
@@ -53,7 +54,9 @@ export default function TodoListsPage() {
         </S.FloatingActionButton>
         <Box>
           <SearchBarWidget />
-          {!status.hasSynced ? <p>Busy with sync...</p> : <TodoListsWidget />}
+          <GuardBySync>
+            <TodoListsWidget />
+          </GuardBySync>
         </Box>
         {/* TODO use a dialog service in future, this is just a simple example app */}
         <Dialog

demos/react-supabase-todolist/src/components/widgets/GuardBySync.tsx

@@ -0,0 +1,41 @@
+import { Box, LinearProgress, Stack, Typography } from '@mui/material';
+import { useStatus } from '@powersync/react';
+import { FC, ReactNode } from 'react';
+
+/**
+ * A component that renders its child if the database has been synced at least once and shows
+ * a progress indicator otherwise.
+ */
+export const GuardBySync: FC<{ children: ReactNode; priority?: number }> = ({ children, priority }) => {
+  const status = useStatus();
+
+  const hasSynced = priority == null ? status.hasSynced : status.statusForPriority(priority).hasSynced;
+  if (hasSynced) {
+    return children;
+  }
+
+  // If we haven't completed a sync yet, show a progress indicator!
+  const allProgress = status.downloadProgress;
+  const progress = priority == null ? allProgress : allProgress?.untilPriority(priority);
+
+  return (
+    <Stack direction="column" spacing={1} sx={{ p: 4 }} alignItems="stretch">
+      {progress != null ? (
+        <>
+          <LinearProgress variant="determinate" value={progress.downloadedFraction * 100} />
+          <Box sx={{ alignSelf: 'center' }}>
+            {progress.downloadedOperations == progress.totalOperations ? (
+              <Typography>Applying server-side changes</Typography>
+            ) : (
+              <Typography>
+                Downloaded {progress.downloadedOperations} out of {progress.totalOperations}.
+              </Typography>
+            )}
+          </Box>
+        </>
+      ) : (
+        <LinearProgress variant="indeterminate" />
+      )}
+    </Stack>
+  );
+};

packages/common/src/client/AbstractPowerSyncDatabase.ts

@@ -32,6 +32,7 @@ import {
   type PowerSyncConnectionOptions,
   type RequiredAdditionalConnectionOptions
 } from './sync/stream/AbstractStreamingSyncImplementation.js';
+import { FULL_SYNC_PRIORITY } from '../db/crud/SyncProgress.js';
 
 export interface DisconnectAndClearOptions {
   /** When set to false, data in local-only tables is preserved. */
@@ -146,11 +147,6 @@ export const isPowerSyncDatabaseOptionsWithSettings = (test: any): test is Power
   return typeof test == 'object' && isSQLOpenOptions(test.database);
 };
 
-/**
- * The priority used by the core extension to indicate that a full sync was completed.
- */
-const FULL_SYNC_PRIORITY = 2147483647;
-
 export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDBListener> {
   /**
    * Transactions should be queued in the DBAdapter, but we also want to prevent

packages/common/src/client/SQLOpenFactory.ts

@@ -7,7 +7,7 @@ export interface SQLOpenOptions {
   dbFilename: string;
   /**
    * Directory where the database file is located.
-   * 
+   *
    * When set, the directory must exist when the database is opened, it will
    * not be created automatically.
    */

packages/common/src/client/sync/bucket/BucketStorageAdapter.ts

@@ -30,6 +30,13 @@ export interface SyncLocalDatabaseResult {
   checkpointFailures?: string[];
 }
 
+export type SavedProgress = {
+  atLast: number;
+  sinceLast: number;
+};
+
+export type BucketOperationProgress = Record<string, SavedProgress>;
+
 export interface BucketChecksum {
   bucket: string;
   priority?: number;
@@ -65,6 +72,7 @@ export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener
   startSession(): void;
 
   getBucketStates(): Promise<BucketState[]>;
+  getBucketOperationProgress(): Promise<BucketOperationProgress>;
 
   syncLocalDatabase(
     checkpoint: Checkpoint,

packages/common/src/client/sync/bucket/SqliteBucketStorage.ts

@@ -5,6 +5,7 @@ import { BaseObserver } from '../../../utils/BaseObserver.js';
 import { MAX_OP_ID } from '../../constants.js';
 import {
   BucketChecksum,
+  BucketOperationProgress,
   BucketState,
   BucketStorageAdapter,
   BucketStorageListener,
@@ -91,6 +92,13 @@ export class SqliteBucketStorage extends BaseObserver<BucketStorageListener> imp
     return result;
   }
 
+  async getBucketOperationProgress(): Promise<BucketOperationProgress> {
+    const rows = await this.db.getAll<{ name: string; count_at_last: number; count_since_last: number }>(
+      'SELECT name, count_at_last, count_since_last FROM ps_buckets'
+    );
+    return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
+  }
+
   async saveSyncData(batch: SyncDataBatch) {
     await this.writeTransaction(async (tx) => {
       let count = 0;
@@ -199,7 +207,21 @@ export class SqliteBucketStorage extends BaseObserver<BucketStorageListener> imp
         'sync_local',
         arg
       ]);
-      return result == 1;
+      if (result == 1) {
+        if (priority == null) {
+          const bucketToCount = Object.fromEntries(checkpoint.buckets.map((b) => [b.bucket, b.count]));
+          // The two parameters could be replaced with one, but: https://github.com/powersync-ja/better-sqlite3/pull/6
+          const jsonBucketCount = JSON.stringify(bucketToCount);
+          await tx.execute(
+            "UPDATE ps_buckets SET count_since_last = 0, count_at_last = ?->name WHERE name != '$local' AND ?->name IS NOT NULL",
+            [jsonBucketCount, jsonBucketCount]
+          );
+        }
+
+        return true;
+      } else {
+        return false;
+      }
     });
   }