[Limit Orders]: SQS Active Order Query (#3828)

* fix: increased cache time and refetch interval for active orders

* fix: even longer cache

* fix: WIP batch query implementation

* fix: longer delay between batches

* fix: batch size/timing

* fix: vercel max duration

* fix: wired in sqs query for active orders

* fix: build

* fix: filled orders styling

* fix: remove claimable orders test data

* fix: further reduced queries

* feat: moved orders query to passthrough

* fix: moved active orders to local router

* fix: altered orders cache to be more responsive

* chore: post typing fixes for active orders sqs query

* fix: reverted unnecessary changes

* feat: removed pool restriction on orderbooks

packages/server/src/queries/complex/orderbooks/active-orders.ts

@@ -1,13 +1,14 @@
 import { Dec, Int } from "@keplr-wallet/unit";
 import { tickToPrice } from "@osmosis-labs/math";
-import { Chain } from "@osmosis-labs/types";
+import { AssetList, Chain } from "@osmosis-labs/types";
 import { getAssetFromAssetList } from "@osmosis-labs/utils";
 import cachified, { CacheEntry } from "cachified";
 import dayjs from "dayjs";
 import { LRUCache } from "lru-cache";
 
 import { DEFAULT_LRU_OPTIONS } from "../../../utils/cache";
 import { LimitOrder, queryOrderbookActiveOrders } from "../../osmosis";
+import { queryActiveOrdersSQS } from "../../sidecar/orderbooks";
 import {
   getOrderbookTickState,
   getOrderbookTickUnrealizedCancels,
@@ -16,6 +17,46 @@ import type { MappedLimitOrder, OrderStatus } from "./types";
 
 const activeOrdersCache = new LRUCache<string, CacheEntry>(DEFAULT_LRU_OPTIONS);
 
+export function getOrderbookActiveOrdersSQS({
+  userOsmoAddress,
+  assetList,
+}: {
+  userOsmoAddress: string;
+  assetList: AssetList[];
+}) {
+  return cachified({
+    cache: activeOrdersCache,
+    key: `orderbookActiveOrders-sqs-${userOsmoAddress}`,
+    ttl: 5000, // 5 seconds
+    getFreshValue: () =>
+      queryActiveOrdersSQS({
+        userOsmoAddress,
+      }).then(async ({ orders }) => {
+        const mappedOrders: MappedLimitOrder[] = orders.map((o) => {
+          return {
+            ...o,
+            price: new Dec(o.price),
+            quantity: parseInt(o.quantity),
+            placed_quantity: parseInt(o.placed_quantity),
+            percentClaimed: new Dec(o.percentClaimed),
+            totalFilled: parseInt(o.totalFilled),
+            percentFilled: new Dec(o.percentFilled),
+            quoteAsset: getAssetFromAssetList({
+              coinMinimalDenom: o.quote_asset.symbol,
+              assetLists: assetList,
+            }),
+            baseAsset: getAssetFromAssetList({
+              coinMinimalDenom: o.base_asset.symbol,
+              assetLists: assetList,
+            }),
+            output: new Dec(o.output),
+          };
+        });
+        return mappedOrders;
+      }),
+  });
+}
+
 export function getOrderbookActiveOrders({
   orderbookAddress,
   userOsmoAddress,
@@ -32,7 +73,7 @@ export function getOrderbookActiveOrders({
   return cachified({
     cache: activeOrdersCache,
     key: `orderbookActiveOrders-${orderbookAddress}-${userOsmoAddress}`,
-    ttl: 2000, // 2 seconds
+    ttl: 10000, // 10 seconds
     getFreshValue: () =>
       queryOrderbookActiveOrders({
         orderbookAddress,

packages/server/src/queries/complex/orderbooks/pools.ts

@@ -22,16 +22,14 @@ export function getOrderbookPools() {
     ttl: 1000 * 60 * 60, // 1 hour
     getFreshValue: () =>
       queryCanonicalOrderbooks().then(async (data) => {
-        return data
-          .filter((o) => o.pool_id < 2065)
-          .map((orderbook) => {
-            return {
-              baseDenom: orderbook.base,
-              quoteDenom: orderbook.quote,
-              contractAddress: orderbook.contract_address,
-              poolId: orderbook.pool_id.toString(),
-            };
-          }) as Orderbook[];
+        return data.map((orderbook) => {
+          return {
+            baseDenom: orderbook.base,
+            quoteDenom: orderbook.quote,
+            contractAddress: orderbook.contract_address,
+            poolId: orderbook.pool_id.toString(),
+          };
+        }) as Orderbook[];
       }),
   });
 }

packages/server/src/queries/complex/orderbooks/tick-state.ts

@@ -24,7 +24,7 @@ export function getOrderbookTickState({
     key: `orderbookTickInfo-${orderbookAddress}-${tickIds
       .sort((a, b) => a - b)
       .join(",")}`,
-    ttl: 1000 * 6, // 6 seconds
+    ttl: 1000 * 10, // 6 seconds
     getFreshValue: () =>
       queryOrderbookTicks({ orderbookAddress, chainList, tickIds }).then(
         ({ data }) => data.ticks
@@ -46,7 +46,7 @@ export function getOrderbookTickUnrealizedCancels({
     key: `orderbookTickUnrealizedCancels-${orderbookAddress}-${tickIds
       .sort((a, b) => a - b)
       .join(",")}`,
-    ttl: 1000 * 6, // 6 seconds
+    ttl: 1000 * 10, // 10 seconds
     getFreshValue: () =>
       queryOrderbookTickUnrealizedCancelsById({
         orderbookAddress,

packages/server/src/queries/sidecar/orderbooks.ts

@@ -9,7 +9,47 @@ export type CanonicalOrderbooksResponse = {
   contract_address: string;
 }[];
 
+export interface SQSActiveOrder {
+  tick_id: number;
+  order_id: number;
+  order_direction: "bid" | "ask";
+  owner: string;
+  quantity: string;
+  etas: string;
+  placed_quantity: string;
+  placed_at: number;
+  price: string;
+  percentClaimed: string;
+  totalFilled: string;
+  percentFilled: string;
+  orderbookAddress: string;
+  status: "open" | "partiallyFilled";
+  output: string;
+  quote_asset: {
+    symbol: string;
+  };
+  base_asset: {
+    symbol: string;
+  };
+}
+
+export type ActiveOrdersResponse = {
+  orders: SQSActiveOrder[];
+};
+
 export async function queryCanonicalOrderbooks() {
   const url = new URL("/pools/canonical-orderbooks", SIDECAR_BASE_URL);
   return await apiClient<CanonicalOrderbooksResponse>(url.toString());
 }
+
+export async function queryActiveOrdersSQS({
+  userOsmoAddress,
+}: {
+  userOsmoAddress: string;
+}) {
+  const url = new URL(
+    `/passthrough/active-orders?userOsmoAddress=${userOsmoAddress}`,
+    SIDECAR_BASE_URL
+  );
+  return await apiClient<ActiveOrdersResponse>(url.toString());
+}

packages/trpc/src/orderbook-router.ts

@@ -3,6 +3,7 @@ import { tickToPrice } from "@osmosis-labs/math";
 import {
   CursorPaginationSchema,
   getOrderbookActiveOrders,
+  getOrderbookActiveOrdersSQS,
   getOrderbookHistoricalOrders,
   getOrderbookMakerFee,
   getOrderbookPools,
@@ -12,12 +13,17 @@ import {
   OrderStatus,
 } from "@osmosis-labs/server";
 import { getAssetFromAssetList } from "@osmosis-labs/utils";
+import { z } from "zod";
 
 import { createTRPCRouter, publicProcedure } from "./api";
 import { OsmoAddressSchema, UserOsmoAddressSchema } from "./parameter-types";
 
 const GetInfiniteLimitOrdersInputSchema = CursorPaginationSchema.merge(
   UserOsmoAddressSchema.required()
+).merge(
+  z.object({
+    filter: z.enum(["open", "filled", "historical"]).optional(),
+  })
 );
 
 const orderStatusOrder: Record<OrderStatus, number> = {
@@ -106,7 +112,48 @@ export const orderbookRouter = createTRPCRouter({
           return allOrders.sort(defaultSortOrders);
         },
         cacheKey: `all-active-orders-${input.userOsmoAddress}`,
-        ttl: 2000,
+        ttl: 15000,
+        cursor: input.cursor,
+        limit: input.limit,
+      });
+    }),
+  getAllOrdersSQS: publicProcedure
+    .input(GetInfiniteLimitOrdersInputSchema)
+    .query(async ({ input, ctx }) => {
+      return maybeCachePaginatedItems({
+        getFreshItems: async () => {
+          const { userOsmoAddress, filter } = input;
+
+          const shouldFetchActive =
+            !filter || filter === "open" || filter === "filled";
+          const shouldFetchHistorical = !filter || filter === "historical";
+          const promises: Promise<MappedLimitOrder[]>[] = [];
+          if (shouldFetchActive) {
+            promises.push(
+              getOrderbookActiveOrdersSQS({
+                userOsmoAddress,
+                assetList: ctx.assetLists,
+              })
+            );
+          }
+          if (shouldFetchHistorical) {
+            promises.push(
+              getOrderbookHistoricalOrders({
+                userOsmoAddress,
+                assetLists: ctx.assetLists,
+                chainList: ctx.chainList,
+              })
+            );
+          }
+          const orders = await Promise.all(promises);
+          const allOrders = orders.flat().sort(defaultSortOrders);
+          if (filter === "filled") {
+            return allOrders.filter((o) => o.status === "filled");
+          }
+          return allOrders;
+        },
+        cacheKey: `all-active-orders-sqs-${input.userOsmoAddress}`,
+        ttl: 10000,
         cursor: input.cursor,
         limit: input.limit,
       });

packages/web/components/complex/orders-history/index.tsx

@@ -84,9 +84,8 @@ export const OrderHistory = observer(() => {
   } = useOrderbookAllActiveOrders({
     userAddress: wallet?.address ?? "",
     pageSize: 20,
-    refetchInterval: 15000,
+    refetchInterval: 10000,
   });
-
   const groupedOrders = useMemo(() => groupOrdersByStatus(orders), [orders]);
   const groups = useMemo(
     () =>
@@ -116,15 +115,16 @@ export const OrderHistory = observer(() => {
     scrollMargin: listRef.current?.offsetTop ?? 0,
     paddingStart: 45,
   });
-
-  const filledOrders = orders.filter((o) => o.status === "filled");
-  const filledOrdersCount = filledOrders.length;
-
-  const { claimAllOrders } = useOrderbookClaimableOrders({
-    userAddress: wallet?.address ?? "",
-    disabled: isLoading || orders.length === 0 || isRefetching,
-    orders: filledOrders,
-  });
+  const filledOrdersInDisplay = useMemo(() => {
+    return orders.filter((o) => o.status === "filled");
+  }, [orders]);
+
+  const { claimAllOrders, count: filledOrdersCount } =
+    useOrderbookClaimableOrders({
+      userAddress: wallet?.address ?? "",
+      disabled: isLoading || filledOrdersInDisplay.length === 0 || isRefetching,
+      refetchInterval: 10000,
+    });
 
   const claimOrders = useCallback(async () => {
     try {

packages/web/hooks/limit-orders/use-orderbook.ts

@@ -265,7 +265,7 @@ export type DisplayableLimitOrder = MappedLimitOrder;
 export const useOrderbookAllActiveOrders = ({
   userAddress,
   pageSize = 10,
-  refetchInterval = 2000,
+  refetchInterval = 5000,
 }: {
   userAddress: string;
   pageSize?: number;
@@ -282,7 +282,7 @@ export const useOrderbookAllActiveOrders = ({
     hasNextPage,
     refetch,
     isRefetching,
-  } = api.edge.orderbooks.getAllOrders.useInfiniteQuery(
+  } = api.local.orderbooks.getAllOrdersSQS.useInfiniteQuery(
     {
       userOsmoAddress: userAddress,
       limit: pageSize,
@@ -311,8 +311,7 @@ export const useOrderbookAllActiveOrders = ({
 
   const refetchOrders = useCallback(async () => {
     if (isRefetching) return;
-
-    return refetch();
+    await refetch();
   }, [refetch, isRefetching]);
 
   return {
@@ -328,33 +327,44 @@ export const useOrderbookAllActiveOrders = ({
 };
 
 export const useOrderbookClaimableOrders = ({
-  userAddress: _,
-  disabled: __,
-  orders = [],
+  userAddress,
+  disabled = false,
+  refetchInterval = 5000,
 }: {
   userAddress: string;
   disabled?: boolean;
-  orders: MappedLimitOrder[];
+  refetchInterval?: number;
 }) => {
   const { orderbooks } = useOrderbooks();
   const { accountStore } = useStore();
   const account = accountStore.getWallet(accountStore.osmosisChainId);
   const addresses = orderbooks.map(({ contractAddress }) => contractAddress);
-  // const {
-  //   data: orders,
-  //   isLoading,
-  //   isFetching,
-  //   refetch,
-  // } = api.edge.orderbooks.getClaimableOrders.useQuery(
-  //   {
-  //     userOsmoAddress: userAddress,
-  //   },
-  //   {
-  //     enabled: !!userAddress && addresses.length > 0 && !disabled,
-  //     refetchOnMount: true,
-  //   }
-  // );
+  const { data: claimableOrders, isLoading } =
+    api.local.orderbooks.getAllOrdersSQS.useInfiniteQuery(
+      {
+        userOsmoAddress: userAddress,
+        filter: "filled",
+        limit: 100,
+      },
+      {
+        getNextPageParam: (lastPage) => lastPage.nextCursor,
+        initialCursor: 0,
+        refetchInterval,
+        enabled: !!userAddress && addresses.length > 0 && !disabled,
+        refetchOnMount: true,
+        keepPreviousData: false,
+        trpc: {
+          abortOnUnmount: true,
+          context: {
+            skipBatch: true,
+          },
+        },
+      }
+    );
 
+  const orders = useMemo(() => {
+    return claimableOrders?.pages?.flatMap((page) => page.items) ?? [];
+  }, [claimableOrders?.pages]);
   const claimAllOrders = useCallback(async () => {
     if (!account || !orders) return;
     const msgs = addresses
@@ -390,7 +400,7 @@ export const useOrderbookClaimableOrders = ({
   return {
     orders: orders ?? [],
     count: orders?.length ?? 0,
-    isLoading: false,
+    isLoading,
     claimAllOrders,
   };
 };

packages/web/server/api/local-router.ts

@@ -4,6 +4,7 @@ import {
   concentratedLiquidityRouter,
   createTRPCRouter,
   oneClickTradingRouter,
+  orderbookRouter,
   paramsRouter,
   portfolioRouter,
   swapRouter,
@@ -23,4 +24,5 @@ export const localRouter = createTRPCRouter({
   bridgeTransfer: localBridgeTransferRouter,
   portfolio: portfolioRouter,
   params: paramsRouter,
+  orderbooks: orderbookRouter,
 });

packages/web/stores/index.tsx

@@ -28,8 +28,7 @@ export function refetchUserQueries(apiUtils: ReturnType<typeof api.useUtils>) {
   apiUtils.local.balances.getUserBalances.invalidate();
   apiUtils.local.bridgeTransfer.getSupportedAssetsBalances.invalidate();
   apiUtils.edge.assets.getImmersiveBridgeAssets.invalidate();
-  apiUtils.edge.orderbooks.getAllOrders.invalidate();
-  apiUtils.edge.orderbooks.getClaimableOrders.invalidate();
+  apiUtils.local.orderbooks.getAllOrdersSQS.invalidate();
 }
 
 const EXCEEDS_1CT_NETWORK_FEE_LIMIT_TOAST_ID = "exceeds-1ct-network-fee-limit";