add fallback for web3 json-rpc.

package.json

@@ -44,8 +44,8 @@
     "@types/node": "^14.14.16",
     "@types/node-fetch": "^2.5.5",
     "@types/yargs": "^15.0.4",
-    "@typescript-eslint/eslint-plugin": "^2.25.0",
-    "@typescript-eslint/parser": "^2.25.0",
+    "@typescript-eslint/eslint-plugin": "^2.34.0",
+    "@typescript-eslint/parser": "^2.34.0",
     "ava": "^3.5.1",
     "docker-compose-mocha": "^1.2.0",
     "eslint": "^6.8.0",

src/api/render-node.test.ts

@@ -12,7 +12,7 @@ test.serial('[integration] getNodeManagement responds according to Ethereum and
   t.timeout(5 * 60 * 1000);
 
   const ethereum = new EthereumTestDriver(true);
-  const ethereumEndpoint = 'http://localhost:7545';
+  const ethereumEndpoint = ['http://localhost:7545'];
   const maticEndpoint = 'mock-endpoint';
   const finalityBufferBlocks = 5;
 

src/cli-args.test.ts

@@ -22,7 +22,7 @@ const configPath = 'some/path/config.json';
 
 const minimalConfigValue = {
   EthereumGenesisContract: 'bar',
-  EthereumEndpoint: 'http://localhost:7545',
+  EthereumEndpoint: ['http://localhost:7545'],
   'node-address': 'ecfcccbc1e54852337298c7e90f5ecee79439e67',
 };
 const inputConfigValue = {
@@ -71,7 +71,7 @@ test('parseOptions: environment variables and no config', (t) => {
 
   t.assert((output.ExternalLaunchConfig = {}));
   t.assert((output.StatusJsonPath = './status/status.json'));
-  t.assert((output.EthereumEndpoint = mockEthereumEndpoint));
+  t.assert((output.EthereumEndpoint = [mockEthereumEndpoint]));
   t.assert((output['node-address'] = mockNodeAddress));
 });
 
@@ -87,7 +87,7 @@ test('parseOptions: env vars take precedence', (t) => {
 
   const output = parseArgs(['--config', configPath]);
 
-  t.assert((output.EthereumEndpoint = mockEthereumEndpoint));
+  t.assert((output.EthereumEndpoint = [mockEthereumEndpoint]));
   t.assert((output['node-address'] = mockNodeAddress));
 });
 

src/config.example.ts

@@ -5,7 +5,7 @@ export const exampleConfig: ServiceConfiguration = {
   Port: 8080,
   EthereumGenesisContract: '0xD859701C81119aB12A1e62AF6270aD2AE05c7AB3',
   EthereumFirstBlock: 11191390,
-  EthereumEndpoint: 'http://ganache:7545',
+  EthereumEndpoint: ['http://ganache:7545'],
   DeploymentDescriptorUrl: 'https://deployment.orbs.network/mainnet.json',
   ElectionsAuditOnly: false,
   StatusJsonPath: './status/status.json',

src/config.test.ts

@@ -7,7 +7,7 @@ test('accepts legal config', (t) => {
       BootstrapMode: false,
       Port: 2,
       EthereumGenesisContract: 'foo',
-      EthereumEndpoint: 'http://localhost:7545',
+      EthereumEndpoint: ['http://localhost:7545'],
       EthereumPollIntervalSeconds: 0.1,
       EthereumRequestsPerSecondLimit: 0,
       ElectionsStaleUpdateSeconds: 7 * 24 * 60 * 60,
@@ -34,7 +34,7 @@ test('declines illegal config (1)', (t) => {
       BootstrapMode: false,
       Port: 2,
       EthereumGenesisContract: 'foo',
-      EthereumEndpoint: 'http://localhost:7545',
+      EthereumEndpoint: ['http://localhost:7545'],
       EthereumPollIntervalSeconds: 0.1,
       EthereumRequestsPerSecondLimit: 0,
       ElectionsStaleUpdateSeconds: 7 * 24 * 60 * 60,
@@ -60,7 +60,7 @@ test('declines illegal config (2)', (t) => {
       BootstrapMode: false,
       Port: 2,
       EthereumGenesisContract: 'foo',
-      EthereumEndpoint: 'foo-bar:123',
+      EthereumEndpoint: ['foo-bar:123'],
       EthereumPollIntervalSeconds: 0.1,
       EthereumRequestsPerSecondLimit: 0,
       ElectionsStaleUpdateSeconds: 7 * 24 * 60 * 60,
@@ -77,6 +77,6 @@ test('declines illegal config (2)', (t) => {
       Verbose: true,
       'node-address': 'ecfcccbc1e54852337298c7e90f5ecee79439e67',
     }),
-    ['Ethereum endpoint is not a valid url']
+    ['Ethereum endpoint Item 1: must be a valid URL']
   );
 });

src/config.ts

@@ -4,7 +4,7 @@ export interface ServiceConfiguration {
   BootstrapMode: boolean;
   Port: number;
   EthereumGenesisContract: string;
-  EthereumEndpoint: string;
+  EthereumEndpoint: string[];
   /** @deprecated Use `EthereumEndpoint` instead */
   MaticEndpoint?: string;
   DeploymentDescriptorUrl: string;
@@ -47,6 +47,39 @@ export const defaultServiceConfiguration = {
   Verbose: false,
 };
 
+// Define the types for the custom validator function
+validate.validators.array = function (
+  value: unknown,
+  options: { item?: validate.ValidateOption },
+  key: string
+): string | undefined {
+  // Check if the value is an array
+  if (!Array.isArray(value)) {
+    return `${key} must be an array.`;
+  }
+
+  // If there are item-level validation options, validate each item
+  if (options && options.item) {
+    const errors = value
+      .map((item, index) => {
+        const error = validate.single(item, options.item);
+        if (error) {
+          return `Item ${index + 1}: ${error.join(', ')}`;
+        }
+        return undefined;
+      })
+      .filter((error): error is string => !!error); // Narrow the type to strings
+
+    // If there are errors, return them as a single string
+    if (errors.length > 0) {
+      return errors.join('; ');
+    }
+  }
+
+  // Return undefined if there are no errors
+  return undefined;
+};
+
 export function validateServiceConfiguration(c: Partial<ServiceConfiguration>): string[] | undefined {
   const serviceConfigConstraints = {
     BootstrapMode: {
@@ -101,12 +134,20 @@ export function validateServiceConfiguration(c: Partial<ServiceConfiguration>):
       numericality: { noStrings: true },
     },
     EthereumEndpoint: {
-      presence: { allowEmpty: false },
-      type: 'string',
-      url: {
-        allowLocal: true,
-      },
-    },
+    presence: true, // Ensure the attribute is present
+    type: "array",  // Ensure it's an array
+    array: {
+      item: {
+        presence: true, // Ensure each item is not empty
+        type: "string", // Ensure each item in the array is a string
+        format: {
+          pattern: /^(https?:\/\/[^\s$.?#].[^\s]*)$/i, // URL regex pattern
+          message: "must be a valid URL"
+        }
+      }
+    }
+  },
+
     EthereumGenesisContract: {
       presence: { allowEmpty: false },
       type: 'string',

src/env-var-args.test.ts

@@ -63,6 +63,12 @@ test('setConfigEnvVars uses environment variables when set', (t) => {
       continue;
     }
 
+    if (key == 'EthereumEndpoint') {
+      t.deepEqual(input[key as keyof ServiceConfiguration], [mockEnv.ETHEREUM_ENDPOINT]);
+      continue;
+    }
+
+    //console.log (key, input[key as keyof ServiceConfiguration], mockEnv[camelCaseToSnakeCase(key) as keyof typeof mockEnv]);
     t.assert(input[key as keyof ServiceConfiguration] === mockEnv[camelCaseToSnakeCase(key) as keyof typeof mockEnv]);
   }
 });

src/env-var-args.ts

@@ -12,7 +12,9 @@ export function setConfigEnvVars(config: ServiceConfiguration): void {
   config.BootstrapMode = process.env.BOOTSTRAP_MODE ? process.env.BOOTSTRAP_MODE === 'true' : config.BootstrapMode;
   config.Port = process.env.PORT ? Number(process.env.PORT) : config.Port;
   config.EthereumGenesisContract = process.env.ETHEREUM_GENESIS_CONTRACT ?? config.EthereumGenesisContract;
-  config.EthereumEndpoint = process.env.ETHEREUM_ENDPOINT ?? config.EthereumEndpoint;
+  // parse ETHEREUM_ENDPOINT, if it has multiple values, split by comma
+  console.log ('fuck off', process.env.ETHEREUM_ENDPOINT);
+  config.EthereumEndpoint = process.env.ETHEREUM_ENDPOINT ? process.env.ETHEREUM_ENDPOINT.split(',') : config.EthereumEndpoint;
   config.DeploymentDescriptorUrl = process.env.DEPLOYMENT_DESCRIPTOR_URL ?? config.DeploymentDescriptorUrl;
   config.ElectionsAuditOnly = process.env.ELECTIONS_AUDIT_ONLY
     ? process.env.ELECTIONS_AUDIT_ONLY === 'true'

src/ethereum/block-sync.test.ts

@@ -9,7 +9,7 @@ test.serial('[integration] BlockSync reads registry for contract addresses', asy
   t.timeout(5 * 60 * 1000);
 
   const ethereum = new EthereumTestDriver(true);
-  const ethereumEndpoint = 'http://localhost:7545';
+  const ethereumEndpoint = ['http://localhost:7545'];
   const finalityBufferBlocks = 5;
 
   // setup Ethereum state

src/ethereum/ethereum-reader.ts

@@ -9,15 +9,17 @@ import https from 'https';
 const HTTP_TIMEOUT_SEC = 20;
 
 const subDomain =  'eth-api'
-const domain = 'orbs.network' 
+const domain = 'orbs.network'
+let timer: NodeJS.Timeout | null = null;
 
 export type EthereumConfiguration = {
-  EthereumEndpoint: string;
+  EthereumEndpoint: string[];
   EthereumRequestsPerSecondLimit: number;
 };
 
 export class EthereumReader {
-  private web3: Web3;
+  private currentWeb3Index = 0;
+  private web3s: Web3[];
   private throttled?: pThrottle.ThrottledFunction<[], void>;
   private agent: https.Agent;
   private blockTimeSinceFail: number;
@@ -29,21 +31,49 @@ export class EthereumReader {
       maxSockets: 5,
     });
     this.blockTimeSinceFail = 0;
-    this.web3 = new Web3(
-      new Web3.providers.HttpProvider(config.EthereumEndpoint, {
+
+    this.web3s = config.EthereumEndpoint.map(endpoint => new Web3(
+      new Web3.providers.HttpProvider(endpoint, {
         keepAlive: true,
         timeout: HTTP_TIMEOUT_SEC * 1000,
       })
-    );
+    ));
+
     if (config.EthereumRequestsPerSecondLimit > 0) {
       this.throttled = pThrottle(() => Promise.resolve(), config.EthereumRequestsPerSecondLimit, 1000);
     }
   }
 
+  getWeb3(): Web3 {
+    return this.web3s[this.currentWeb3Index];
+  }
+
+  switchWeb3() {
+    this.currentWeb3Index = (this.currentWeb3Index + 1) % this.web3s.length;
+
+    if (this.currentWeb3Index != 0) {
+      if (timer!=null) {
+        clearTimeout (timer);
+      }
+
+      timer = setTimeout(() => {
+        this.currentWeb3Index = 0;
+        console.log('switchWeb3: switching to web3 to first provider.');
+      }, 1000 * 60 * 60); // after an hour, return to the first web3
+    }
+  }
+
   async getBlockNumber(): Promise<number> {
     if (this.throttled) await this.throttled();
     this.requestStats.add(1);
-    return this.web3.eth.getBlockNumber();
+
+    try {
+      return await this.getWeb3().eth.getBlockNumber();
+    } catch (error) {
+      console.error("Error fetching block number:", error);
+      this.switchWeb3();
+      return await this.getWeb3().eth.getBlockNumber();
+    }
   }
 
   // orbs GET api dediated to serve block time from cache
@@ -92,7 +122,16 @@ export class EthereumReader {
     // fallback to web3
     if (this.throttled) await this.throttled();
     this.requestStats.add(1);
-    const block = await this.web3.eth.getBlock(blockNumber);
+
+    let block
+    try {
+      block = await this.getWeb3().eth.getBlock(blockNumber);
+    } catch (error) {
+      console.error("Error fetching block number:", error);
+      this.switchWeb3();
+      block = await this.getWeb3().eth.getBlock(blockNumber);
+    }
+
     if (!block) {
       throw new Error(`web3.eth.getBlock for ${blockNumber} return empty block.`);
     }
@@ -102,7 +141,18 @@ export class EthereumReader {
   getContractForEvent(eventName: EventName, address: string): Contract {
     const contractName = contractByEventName(eventName);
     const abi = getAbiForContract(address, contractName);
-    return new this.web3.eth.Contract(abi, address);
+
+    try {
+      const web3instance = this.getWeb3();
+      //return new this.getWeb3().eth.Contract(abi, address);
+      return new web3instance.eth.Contract(abi, address);
+    } catch (error) {
+      console.error("Error fetching contract:", error);
+      this.switchWeb3();
+      const web3instance = this.getWeb3();
+      return new web3instance.eth.Contract(abi, address);
+      //return new this.getWeb3().eth.Contract(abi, address);
+    }
   }
 
   // throws error if fails, caller needs to decrease page size if needed

src/ethereum/event-fetcher.ts

@@ -16,7 +16,7 @@ export abstract class EventFetcher {
   }
 
   // every fetcher instance should override this function
-  abstract async fetchBlock(blockNumber: number, latestAllowedBlock: number): Promise<EventData[]>;
+  abstract fetchBlock(blockNumber: number, latestAllowedBlock: number): Promise<EventData[]>;
 }
 
 // the simplest fetcher, yet inefficient, good for testing

src/ethereum/test-driver.ts

@@ -155,8 +155,8 @@ export class EthereumTestDriver {
     return await d.web3.eth.getBlockNumber();
   }
 
-  async getCurrentBlockPreDeploy(ethereumEndpoint: string): Promise<number> {
-    const web3 = new Web3(ethereumEndpoint);
+  async getCurrentBlockPreDeploy(ethereumEndpoint: string[]): Promise<number> {
+    const web3 = new Web3(ethereumEndpoint[0]);
     return await web3.eth.getBlockNumber();
   }
 

src/main.ts

@@ -10,13 +10,15 @@ process.on('uncaughtException', function (err) {
 });
 
 function censorConfig(conf: ServiceConfiguration) {
+  const censoredEthEndpointArray = conf.EthereumEndpoint.map((endpoint) => endpoint.slice(0, 30) + "**********");
+
   const external: {[key: string]: any} = {
     ...conf.ExternalLaunchConfig,
-    EthereumEndpoint: conf.EthereumEndpoint.slice(0, -10) + "**********",
+    EthereumEndpoint: censoredEthEndpointArray,
   }
   const censored = {
     ...conf,
-    EthereumEndpoint: conf.EthereumEndpoint.slice(0, -10) + "**********",
+    EthereumEndpoint: censoredEthEndpointArray,
     ExternalLaunchConfig: external
   }