various progress, add metrics

docker/docker-compose.local.yml

@@ -1,4 +1,4 @@
-version: "3.4"
+version: "3.7"
 
 # Configuration to work with a local non-dockerized Gossip node
 # For local testing and quick debugging
@@ -12,25 +12,27 @@ services:
     build:
       context: prometheus
       args:
-        # Linux:  http://localhost:8008
-        # MacOSX: http://host.docker.internal:8008
-        BEACON_URL: localhost:8008
+        # Linux:  localhost:8088
+        # MacOSX: host.docker.internal:8088
+        GOSSIP_URL: host.docker.internal:8088
     restart: always
-    network_mode: host
     volumes:
       - "prometheus:/prometheus"
+    ports:
+      - 9090:9090
 
   grafana:
     build: grafana
     restart: always
-    network_mode: host
     volumes:
       - "grafana:/var/lib/grafana"
       - "../dashboards:/dashboards"
     environment:
       # Linux:  http://localhost:9090
       # MacOSX: http://host.docker.internal:9090
-      PROMETHEUS_URL: http://localhost:9090
+      PROMETHEUS_URL: http://host.docker.internal:9090
+    ports:
+      - 3000:3000
 
 volumes:
   prometheus:

docker/grafana/Dockerfile

@@ -1,5 +1,4 @@
-# Same version as our ansible deployments, to minimize the diff in the dashboard on export
-FROM grafana/grafana:8.4.2
+FROM grafana/grafana:9.3.0
 
 # Datasource URL is configured with ENV variables
 COPY datasource.yml /etc/grafana/provisioning/datasources/datasource.yml

docker/grafana/dashboard.yml

@@ -1,7 +1,7 @@
 apiVersion: 1
 
 providers:
-  - name: lodestar_github
+  - name: seaport_gossip_github
     type: file
     updateIntervalSeconds: 60
     allowUiUpdates: true

docker/prometheus/Dockerfile

@@ -3,11 +3,11 @@ FROM prom/prometheus:latest
 COPY prometheus.yml /etc/prometheus/prometheus.yml
 
 # Modified datasource to work with a network_mode: host
-# Docker DNS: "beacon_node:8008"
-# net host: "localhost:8008"
-# MacOSX: "host.docker.internal:8008"
-ARG BEACON_URL=beacon_node:8008
-RUN sed -i 's/#BEACON_URL/'"$BEACON_URL"'/' /etc/prometheus/prometheus.yml
+# Docker DNS: "gossip_node:8088"
+# net host: "localhost:8088"
+# MacOSX: "host.docker.internal:8088"
+ARG GOSSIP_URL=gossip_node:8088
+RUN sed -i 's/#GOSSIP_URL/'"$GOSSIP_URL"'/' /etc/prometheus/prometheus.yml
 RUN cat /etc/prometheus/prometheus.yml
 
 CMD [ \

docker/prometheus/prometheus.yml

@@ -1,9 +1,9 @@
 scrape_configs:
-  - job_name: Lodestar
+  - job_name: SeaportGossip 
     scrape_interval: 20s
     scrape_timeout: 20s
     metrics_path: /metrics
     static_configs:
       # This tag is to be replaced in the Dockerfile with sed
       # Modified datasource to work with a network_mode: host
-      - targets: ["#BEACON_URL"]
+      - targets: ['#GOSSIP_URL']

example.env

@@ -1,8 +1,11 @@
 export WEB3_PROVIDER="https://api.mycryptoapi.com/eth"
 export OPENSEA_API_KEY=""
-export COLLECTION_ADDRESSES="0x942bc2d3e7a589fe5bd4a5c6ef9727dfd82f5c8a,0xe4d20bc4a845aa35b008f5f9f85e50b581df7263"
-export INGEST_OPENSEA_ORDERS="true"
-export LOG_LEVEL="debug"
+
+export SEAPORT_GOSSIP_COLLECTION_ADDRESSES="0x942bc2d3e7a589fe5bd4a5c6ef9727dfd82f5c8a,0xe4d20bc4a845aa35b008f5f9f85e50b581df7263"
+export SEAPORT_GOSSIP_LOG_LEVEL="debug"
+
+export SEAPORT_GOSSIP_INGEST_OPENSEA_ORDERS="true"
+export SEAPORT_GOSSIP_GET_ALL_ORDERS_FROM_PEERS="true"
 
 export SEAPORT_GOSSIP_HOSTNAME="your.external.ip"
 export SEAPORT_GOSSIP_BOOTNODES="/ip4/127.0.0.1/tcp/8998/ws/p2p/12D3KooWCchvPx33SSNnxBntQmt9UN3GDVWtrw5B3egCC8eaDsRb"

package.json

@@ -9,8 +9,8 @@
     "src",
     "prisma"
   ],
-  "engines" : { 
-    "node" : ">=16.15.1"
+  "engines": {
+    "node": ">=16.15.1"
   },
   "type": "module",
   "types": "./dist/index.d.ts",
@@ -43,16 +43,17 @@
     "seed": "ts-node-esm prisma/seed.ts"
   },
   "dependencies": {
-    "@chainsafe/libp2p-gossipsub": "^4.1.1",
-    "@chainsafe/libp2p-noise": "^8.0.1",
+    "@chainsafe/libp2p-gossipsub": "^5.2.1",
+    "@chainsafe/libp2p-noise": "^10.2.0",
     "@chainsafe/ssz": "^0.9.2",
     "@graphql-yoga/node": "^2.13.13",
     "@libp2p/interfaces": "^3.0.3",
-    "@libp2p/kad-dht": "^3.0.5",
-    "@libp2p/mplex": "^5.2.4",
+    "@libp2p/kad-dht": "^6.0.1",
+    "@libp2p/mplex": "^7.1.0",
     "@libp2p/peer-id": "^1.1.16",
     "@libp2p/peer-id-factory": "^1.0.18",
-    "@libp2p/websockets": "^3.0.4",
+    "@libp2p/prometheus-metrics": "^1.1.2",
+    "@libp2p/websockets": "^5.0.0",
     "@multiformats/multiaddr": "^11.0.0",
     "@opensea/seaport-order-validator": "^0.1.1",
     "@opensea/stream-js": "^0.0.21-rc.1",
@@ -65,7 +66,7 @@
     "graphql-fields": "^2.0.3",
     "graphql-scalars": "^1.18.0",
     "it-pipe": "^2.0.4",
-    "libp2p": "^0.39.2",
+    "libp2p": "0.40.0-bae32ba",
     "merkletreejs": "^0.3.3",
     "node-fetch": "^3.2.10",
     "prom-client": "^14.1.0",

prisma/migrations/20221116203342_/migration.sql → prisma/migrations/20221122200922_/migration.sql

@@ -40,15 +40,15 @@ CREATE TABLE "Order" (
     "numerator" TEXT,
     "denominator" TEXT,
     "extraData" TEXT,
-    "chainId" TEXT NOT NULL
+    "chainId" TEXT NOT NULL,
+    "auctionType" INTEGER NOT NULL
 );
 
 -- CreateTable
 CREATE TABLE "OrderMetadata" (
     "orderHash" TEXT NOT NULL PRIMARY KEY,
     "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
     "isValid" BOOLEAN NOT NULL,
-    "isAuction" BOOLEAN NOT NULL,
     "isFullyFulfilled" BOOLEAN NOT NULL,
     "lastFulfilledAt" TEXT,
     "lastFulfilledPrice" TEXT,

prisma/schema.prisma

@@ -66,8 +66,9 @@ model Order {
   extraData   String?
 
   // Metadata
-  chainId  String /// string decimal
-  metadata OrderMetadata?
+  chainId     String /// string decimal
+  metadata    OrderMetadata?
+  auctionType Int /// 0: basic, 1: english, 2: dutch
 }
 
 model OrderMetadata {
@@ -76,8 +77,6 @@ model OrderMetadata {
 
   isValid Boolean
 
-  isAuction Boolean /// when order is restricted and zone is an EOA. in the future may have whitelisted "auction" zones
-
   isFullyFulfilled   Boolean
   lastFulfilledAt    String?
   lastFulfilledPrice String?

scripts/simulate/run-local-net.ts

@@ -117,5 +117,5 @@ const invalidOrder = invalidBasicOrders[0]
 await node3.addOrders([invalidOrder] as any)
 
 // Force gossip invalid order to connected peer
-await (node3 as any)._publishOrder(invalidOrder)
+await node3.publishOrder(invalidOrder)
 */

src/ingestors/opensea.ts

@@ -1,9 +1,12 @@
 import { EventType, OpenSeaStreamClient } from '@opensea/stream-js'
-import { BigNumber } from 'ethers'
+import { BigNumber, ethers } from 'ethers'
 import fetch from 'node-fetch'
 import { WebSocket } from 'ws'
 
+import { addOrder, exceedsMaxOrderLimits } from '../query/order.js'
 import { RateLimit, short } from '../util/helpers.js'
+import { deriveOrderHash } from '../util/order.js'
+import { AuctionType, OrderEvent } from '../util/types.js'
 
 import type { SeaportGossipNode } from '../node.js'
 import type { Address, ItemType, OrderJSON, OrderType } from '../util/types.js'
@@ -14,7 +17,6 @@ import type {
   ItemReceivedOfferEventPayload,
 } from '@opensea/stream-js'
 import type { RequestInit } from 'node-fetch'
-import type winston from 'winston'
 
 enum OpenSeaOrderType {
   LISTINGS,
@@ -23,7 +25,6 @@ enum OpenSeaOrderType {
 
 interface IngestorOpts {
   node: SeaportGossipNode
-  logger: winston.Logger
 }
 
 interface OpenSeaOfferItem {
@@ -45,6 +46,7 @@ interface OpenSeaOrder {
   listing_time: number
   expiration_time: number
   order_hash: string
+  order_type: string
   protocol_data: {
     parameters: {
       offerer: string
@@ -66,12 +68,10 @@ interface OpenSeaOrder {
 
 export class OpenSeaOrderIngestor {
   private node: SeaportGossipNode
-  private logger: winston.Logger
   private client: OpenSeaStreamClient
 
   private running = false
 
-  private OPENSEA_API_KEY: string
   private CONTRACT_ENDPOINT = 'https://api.opensea.io/api/v1/asset_contract/'
   private ORDERS_ENDPOINT = 'https://api.opensea.io/v2/orders/ethereum/seaport/'
   private LISTINGS_ENDPOINT = `${this.ORDERS_ENDPOINT}listings`
@@ -83,20 +83,18 @@ export class OpenSeaOrderIngestor {
 
   constructor(opts: IngestorOpts) {
     this.node = opts.node
-    this.logger = opts.logger
-    this.OPENSEA_API_KEY = (this.node as any).opts.openSeaAPIKey
     this.client = new OpenSeaStreamClient({
-      token: this.OPENSEA_API_KEY,
+      token: this.node.opts.openSeaAPIKey,
       connectOptions: {
         transport: WebSocket,
       },
     })
   }
 
   public async start() {
-    this.logger.info('OpenSea Ingestor: Starting...')
+    this.node.logger.info('OpenSea Ingestor: Starting...')
     this.running = true
-    const collectionAddresses = (this.node as any).opts.collectionAddresses
+    const collectionAddresses = this.node.opts.collectionAddresses
     for (const address of collectionAddresses) {
       const slug = await this._getCollectionSlug(address)
       if (slug === undefined) continue
@@ -126,11 +124,6 @@ export class OpenSeaOrderIngestor {
       event.payload.item.nft_id.split('/')
     if (chain !== 'ethereum') return
     const orderHash = event.payload.order_hash
-    let log = `OpenSea Ingestor: New event ${
-      event.event_type
-    } for collection ${short(collectionAddress)}, order hash: ${short(
-      orderHash
-    )}`
     const orderType =
       event.event_type === EventType.ITEM_LISTED
         ? OpenSeaOrderType.LISTINGS
@@ -142,13 +135,36 @@ export class OpenSeaOrderIngestor {
       orderType
     )
     if (order === undefined) return
-    const added = await this.node.addOrders([order])
-    if (added === 1) {
-      log += '. Added valid order to db.'
+    if (await exceedsMaxOrderLimits(order, this.node)) return
+    const [isAdded, metadata] = await addOrder(
+      this.node,
+      order,
+      false,
+      false,
+      order.auctionType
+    )
+    let log = `OpenSea Ingestor: New event ${
+      event.event_type
+    } for collection ${short(collectionAddress)}, order hash: ${short(
+      orderHash
+    )}`
+    if (isAdded) {
+      log += '. Added order to db.'
     } else {
-      log += '. Order not added to db.'
+      log += `. Order not added to db (${
+        metadata.isValid ? 'valid, already existed' : 'invalid'
+      }).`
+    }
+    this.node.logger.info(log)
+    const gossipsubEvent = {
+      event: OrderEvent.NEW,
+      orderHash: deriveOrderHash(order),
+      order,
+      blockNumber: metadata.lastValidatedBlockNumber ?? '0',
+      blockHash: metadata.lastValidatedBlockHash ?? ethers.constants.HashZero,
     }
-    this.logger.info(log)
+    await this.node.publishEvent(gossipsubEvent)
+    this.node.metrics?.ordersIngestedOpenSea.inc()
   }
 
   private async _getOrder(
@@ -178,22 +194,28 @@ export class OpenSeaOrderIngestor {
       if (order === undefined) return undefined
       return this._orderToOrderJSON(order)
     } catch (error: any) {
-      this.logger.error(
+      this.node.logger.error(
         `Error fetching order from OpenSea: ${error.message ?? error}`
       )
     }
   }
 
-  private _orderToOrderJSON(order: OpenSeaOrder): OrderJSON {
+  private _orderToOrderJSON(
+    order: OpenSeaOrder
+  ): OrderJSON & { auctionType: AuctionType } {
     const { parameters, signature } = order.protocol_data
     delete (parameters as any).totalOriginalConsiderationItems
+    let auctionType = AuctionType.BASIC
+    if (order.order_type === 'english') auctionType = AuctionType.ENGLISH
+    if (order.order_type === 'dutch') auctionType = AuctionType.DUTCH
     return {
       ...parameters,
       startTime: Number(parameters.startTime),
       endTime: Number(parameters.endTime),
       salt: BigNumber.from(parameters.salt).toString(),
       signature,
       chainId: '1',
+      auctionType,
     }
   }
 
@@ -207,7 +229,7 @@ export class OpenSeaOrderIngestor {
         throw new Error('slug not present in returned collection metadata')
       return data.collection.slug
     } catch (error: any) {
-      this.logger.error(
+      this.node.logger.error(
         `Error fetching collection slug for ${address} from OpenSea: ${
           error.message ?? error
         }`
@@ -223,15 +245,15 @@ export class OpenSeaOrderIngestor {
       ...opts,
       headers: {
         accept: 'application/json',
-        'X-API-KEY': this.OPENSEA_API_KEY,
+        'X-API-KEY': this.node.opts.openSeaAPIKey,
         ...opts.headers,
       },
     })
   }
 
   public stop() {
     if (!this.running) return
-    this.logger.info('OpenSea Ingestor: Stopping...')
+    this.node.logger.info('OpenSea Ingestor: Stopping...')
     this.client.disconnect()
     this.abortController.abort()
     this.running = false