get avail slots/timestamps from light client instead of rpc

packages/engine/paima-funnel/src/funnels/avail/baseFunnel.ts

@@ -99,7 +99,7 @@ export class AvailBlockFunnel extends BaseFunnel implements ChainFunnel {
     );
 
     const parallelHeaders = await timeout(
-      getMultipleHeaderData(this.api, numbers),
+      getMultipleHeaderData(this.api, this.config.lightClient, numbers),
       GET_DATA_TIMEOUT
     );
 

packages/engine/paima-funnel/src/funnels/avail/parallelFunnel.ts

@@ -8,18 +8,17 @@ import type { PoolClient } from 'pg';
 import { FUNNEL_PRESYNC_FINISHED } from '@paima/utils';
 import { createApi } from './createApi.js';
 import { getLatestProcessedCdeBlockheight } from '@paima/db';
-import type { Header } from './utils.js';
 import {
   getDAData,
-  getLatestBlockNumber,
   getMultipleHeaderData,
-  getSlotFromHeader,
   getTimestampForBlockAt,
   slotToTimestamp,
   timestampToSlot,
   GET_DATA_TIMEOUT,
   getLatestAvailableBlockNumberFromLightClient,
+  getBlockHeaderDataFromLightClient,
 } from './utils.js';
+import type { HeaderData } from './utils.js';
 import {
   addInternalCheckpointingEvent,
   buildParallelBlockMappings,
@@ -137,10 +136,16 @@ export class AvailParallelFunnel extends BaseFunnel implements ChainFunnel {
         }
 
         // get only headers for block that have data
-        parallelHeaders = await timeout(getMultipleHeaderData(this.api, numbers), GET_DATA_TIMEOUT);
+        parallelHeaders = await timeout(
+          getMultipleHeaderData(this.api, this.config.lightClient, numbers),
+          GET_DATA_TIMEOUT
+        );
       } else {
         // unless the range is empty
-        parallelHeaders = await timeout(getMultipleHeaderData(this.api, [to]), GET_DATA_TIMEOUT);
+        parallelHeaders = await timeout(
+          getMultipleHeaderData(this.api, this.config.lightClient, [to]),
+          GET_DATA_TIMEOUT
+        );
       }
 
       for (const blockData of roundParallelData) {
@@ -290,10 +295,11 @@ export class AvailParallelFunnel extends BaseFunnel implements ChainFunnel {
       const mappedStartingBlockHeight = await findBlockByTimestamp(
         // the genesis doesn't have a slot to extract a timestamp from
         1,
-        await getLatestBlockNumber(api),
+        await getLatestAvailableBlockNumberFromLightClient(config.lightClient),
         applyDelay(config, Number(startingBlock.timestamp)),
         chainName,
-        async (blockNumber: number) => await getTimestampForBlockAt(api, blockNumber)
+        async (blockNumber: number) =>
+          await getTimestampForBlockAt(config.lightClient, api, blockNumber)
       );
 
       availFunnelCacheEntry.initialize(mappedStartingBlockHeight);
@@ -324,26 +330,27 @@ export class AvailParallelFunnel extends BaseFunnel implements ChainFunnel {
     const config = this.config;
 
     const latestHeader = await timeout(
-      (async (): Promise<Header> => {
+      (async (): Promise<HeaderData> => {
         const latestNumber = await getLatestAvailableBlockNumberFromLightClient(config.lightClient);
 
-        const latestBlockHash = await this.api.rpc.chain.getBlockHash(latestNumber);
-        const latestHeader = await this.api.rpc.chain.getHeader(latestBlockHash);
+        const latestHeader = await getBlockHeaderDataFromLightClient(
+          config.lightClient,
+          latestNumber,
+          this.api
+        );
 
-        return latestHeader as unknown as Header;
+        return latestHeader;
       })(),
       LATEST_BLOCK_UPDATE_TIMEOUT
     );
 
-    const slot = getSlotFromHeader(latestHeader, this.api);
-
     this.sharedData.cacheManager.cacheEntries[AvailFunnelCacheEntry.SYMBOL]?.updateLatestBlock({
-      number: latestHeader.number.toNumber(),
+      number: latestHeader.number,
       hash: latestHeader.hash.toString(),
-      slot: slot,
+      slot: latestHeader.slot,
     });
 
-    return latestHeader.number.toNumber();
+    return latestHeader.number;
   }
 
   private getCacheEntry(): AvailFunnelCacheEntry {

packages/engine/paima-funnel/src/funnels/avail/utils.ts

@@ -1,40 +1,24 @@
 import type { ApiPromise } from 'avail-js-sdk';
-import type { Header as PolkadotHeader } from '@polkadot/types/interfaces/types';
+import type { Header as PolkadotHeader, DigestItem } from '@polkadot/types/interfaces/types';
+import { Bytes } from '@polkadot/types-codec';
 import type { SubmittedData } from '@paima/sm';
 import { base64Decode } from '@polkadot/util-crypto';
 import { BaseFunnelSharedApi } from '../BaseFunnel.js';
 import { createApi } from './createApi.js';
+import { GenericConsensusEngineId } from '@polkadot/types/generic/ConsensusEngineId';
 
 export const GET_DATA_TIMEOUT = 10000;
 
 export type Header = PolkadotHeader;
 
-export function getSlotFromHeader(header: Header, api: ApiPromise): number {
-  const preRuntime = header.digest.logs.find(log => log.isPreRuntime)!.asPreRuntime;
-
-  const rawBabeDigest = api.createType('RawBabePreDigest', preRuntime[1]);
-
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const babeDigest = rawBabeDigest.toPrimitive() as unknown as any;
-
-  // the object is an enumeration, but all the variants have a slotNumber field
-  const slot = babeDigest[Object.getOwnPropertyNames(babeDigest)[0]].slotNumber;
-  return slot;
-}
-
-export async function getLatestBlockNumber(api: ApiPromise): Promise<number> {
-  let highHash = await api.rpc.chain.getFinalizedHead();
-  let high = (await api.rpc.chain.getHeader(highHash)).number.toNumber();
-  return high;
-}
-
-export async function getTimestampForBlockAt(api: ApiPromise, mid: number): Promise<number> {
-  const hash = await api.rpc.chain.getBlockHash(mid);
-  // FIXME: why is the conversion needed?
-  const header = (await api.rpc.chain.getHeader(hash)) as unknown as Header;
+export async function getTimestampForBlockAt(
+  lc: string,
+  api: ApiPromise,
+  bn: number
+): Promise<number> {
+  const header = await getBlockHeaderDataFromLightClient(lc, bn, api);
 
-  const slot = getSlotFromHeader(header, api);
-  return slotToTimestamp(slot, api);
+  return slotToTimestamp(header.slot, api);
 }
 
 export function slotToTimestamp(slot: number, api: ApiPromise): number {
@@ -55,32 +39,82 @@ export function timestampToSlot(timestamp: number, api: ApiPromise): number {
   return timestamp / slotDuration;
 }
 
-type HeaderData = { number: number; hash: string; slot: number };
+export type HeaderData = { number: number; hash: string; slot: number };
 
 export async function getMultipleHeaderData(
   api: ApiPromise,
+  lc: string,
   blockNumbers: number[]
 ): Promise<HeaderData[]> {
   const results = [] as HeaderData[];
 
   for (const bn of blockNumbers) {
-    // NOTE: the light client allows getting header directly from block number,
-    // but it doesn't provide the babe data for the slot
-    const hash = await api.rpc.chain.getBlockHash(bn);
-    const header = (await api.rpc.chain.getHeader(hash)) as unknown as Header;
-
-    const slot = getSlotFromHeader(header, api);
-
-    results.push({
-      number: header.number.toNumber(),
-      hash: header.hash.toString(),
-      slot: slot,
-    });
+    results.push(await getBlockHeaderDataFromLightClient(lc, bn, api));
   }
 
   return results;
 }
 
+export async function getBlockHeaderDataFromLightClient(
+  lc: string,
+  bn: number,
+  api: ApiPromise
+): Promise<{ number: number; hash: string; slot: number }> {
+  const responseRaw = await fetch(`${lc}/v2/blocks/${bn}/header`);
+
+  if (responseRaw.status !== 200) {
+    // we don't want to accidentally skip blocks if there is something wrong
+    // with the light client. We only fetch blocks in range, so a not found
+    // here it's a logic error.
+    throw new Error(
+      `Unexpected error encountered when fetching headers from Avail's light client. Error: ${responseRaw.status}`
+    );
+  }
+
+  const response = (await responseRaw.json()) as unknown as {
+    hash: string;
+    number: number;
+    digest: {
+      logs: {
+        [key in DigestItem['type']]: [number[], number[]];
+      }[];
+    };
+  };
+
+  const preRuntimeJson = response.digest.logs.find(log => log.PreRuntime)?.PreRuntime;
+
+  if (!preRuntimeJson) {
+    throw new Error("Couldn't find preruntime digest");
+  }
+
+  // using ts-expect-error because for some reason the types of the registry are
+  // different, but avail-js-sdk doesn't seem to re-export @polkadot/types in
+  // order to access these constructors directly.
+  const preRuntime = [
+    // this is not used, but we parse it just in case it fails.
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-expect-error
+    new GenericConsensusEngineId(api.registry, preRuntimeJson[0]),
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-expect-error
+    new Bytes(api.registry, preRuntimeJson[1]),
+  ];
+
+  const rawBabeDigest = api.createType('RawBabePreDigest', preRuntime[1]);
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const babeDigest = rawBabeDigest.toPrimitive() as unknown as any;
+
+  // the object is an enumeration, but all the variants have a slotNumber field
+  const slot = babeDigest[Object.getOwnPropertyNames(babeDigest)[0]].slotNumber;
+
+  return {
+    number: response.number,
+    hash: response.hash,
+    slot: slot,
+  };
+}
+
 export async function getDAData(
   api: ApiPromise,
   lc: string,
@@ -157,7 +191,10 @@ export async function getLatestAvailableBlockNumberFromLightClient(lc: string):
 }
 
 export class AvailSharedApi extends BaseFunnelSharedApi {
-  public constructor(private rpc: string) {
+  public constructor(
+    private rpc: string,
+    private lightClient: string
+  ) {
     super();
     this.getBlock.bind(this);
   }
@@ -167,7 +204,7 @@ export class AvailSharedApi extends BaseFunnelSharedApi {
   ): Promise<{ timestamp: number | string } | undefined> {
     const api = await createApi(this.rpc);
 
-    const headerData = await getMultipleHeaderData(api, [height]);
+    const headerData = await getMultipleHeaderData(api, this.lightClient, [height]);
 
     const timestamp = slotToTimestamp(headerData[0].slot, api);
 

packages/engine/paima-funnel/src/index.ts

@@ -52,7 +52,7 @@ export class FunnelFactory implements IFunnelFactory {
     }
 
     if (mainConfig.type === ConfigNetworkType.AVAIL_MAIN) {
-      mainNetworkApi = new AvailSharedApi(mainConfig.rpc);
+      mainNetworkApi = new AvailSharedApi(mainConfig.rpc, mainConfig.lightClient);
     }
 
     const web3s = await Promise.all(