Implement Retry Mechanism for HttpExtractor

libs/execution/src/lib/blocks/execution-result.ts

@@ -5,7 +5,7 @@
 import * as E from 'fp-ts/lib/Either';
 import { AstNode, DiagnosticInfo } from 'langium';
 
-interface ExecutionErrorDetails<N extends AstNode = AstNode> {
+export interface ExecutionErrorDetails<N extends AstNode = AstNode> {
   message: string;
   diagnostic: DiagnosticInfo<N>;
 }

libs/extensions/std/exec/src/http-extractor-executor.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0-only
 
+import { strict as assert } from 'assert';
 import * as http from 'http';
 import * as https from 'https';
 import * as path from 'path';
@@ -12,18 +13,24 @@ import {
   BinaryFile,
   BlockExecutorClass,
   ExecutionContext,
+  ExecutionErrorDetails,
   FileExtension,
   MimeType,
   None,
   implementsStatic,
 } from '@jvalue/jayvee-execution';
 import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server';
+import { AstNode } from 'langium';
 
 import {
   inferFileExtensionFromContentTypeString,
   inferFileExtensionFromFileExtensionString,
   inferMimeTypeFromContentTypeString,
 } from './file-util';
+import {
+  createBackoffStrategy,
+  isBackoffStrategyHandle,
+} from './util/backoff-strategy';
 
 type HttpGetFunction = typeof http.get;
 
@@ -43,14 +50,53 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor<
     context: ExecutionContext,
   ): Promise<R.Result<BinaryFile>> {
     const url = context.getPropertyValue('url', PrimitiveValuetypes.Text);
-
-    const file = await this.fetchRawDataAsFile(url, context);
-
-    if (R.isErr(file)) {
-      return file;
+    const retries = context.getPropertyValue(
+      'retries',
+      PrimitiveValuetypes.Integer,
+    );
+    assert(retries >= 0); // loop executes at least once
+    const retryBackoffMilliseconds = context.getPropertyValue(
+      'retryBackoffMilliseconds',
+      PrimitiveValuetypes.Integer,
+    );
+    const retryBackoffStrategy = context.getPropertyValue(
+      'retryBackoffStrategy',
+      PrimitiveValuetypes.Text,
+    );
+    assert(isBackoffStrategyHandle(retryBackoffStrategy));
+    const backoffStrategy = createBackoffStrategy(
+      retryBackoffStrategy,
+      retryBackoffMilliseconds,
+    );
+
+    let failure: ExecutionErrorDetails<AstNode> | undefined;
+    for (let attempt = 0; attempt <= retries; ++attempt) {
+      const isLastAttempt = attempt === retries;
+      const file = await this.fetchRawDataAsFile(url, context);
+
+      if (R.isOk(file)) {
+        context.logger.logDebug(`Successfully fetched raw data`);
+        return R.ok(file.right);
+      }
+
+      failure = file.left;
+
+      if (!isLastAttempt) {
+        context.logger.logDebug(failure.message);
+
+        const currentBackoff = backoffStrategy.getBackoffMilliseconds(
+          attempt + 1,
+        );
+        context.logger.logDebug(
+          `Waiting ${currentBackoff}ms before trying again...`,
+        );
+        await new Promise((p) => setTimeout(p, currentBackoff));
+        continue;
+      }
     }
 
-    return R.ok(file.right);
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return R.err(failure!);
   }
 
   private fetchRawDataAsFile(
@@ -91,7 +137,6 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor<
 
         // When all data is downloaded, create file
         response.on('end', () => {
-          context.logger.logDebug(`Successfully fetched raw data`);
           response.headers;
 
           // Infer Mimetype from HTTP-Header, if not inferrable, then default to application/octet-stream

libs/extensions/std/exec/src/util/backoff-strategy.spec.ts

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import {
+  BackoffStrategy,
+  ExponentialBackoffStrategy,
+  LinearBackoffStrategy,
+} from './backoff-strategy';
+
+describe('BackoffStrategy', () => {
+  describe('ExponentialBackoffStrategy', () => {
+    describe('getBackoffMilliseconds', () => {
+      it('should calculate exponential backoff correctly with 5 retries', () => {
+        const backoffStrategy: BackoffStrategy = new ExponentialBackoffStrategy(
+          2,
+        );
+
+        expect(backoffStrategy.getBackoffMilliseconds(1)).toEqual(2);
+        expect(backoffStrategy.getBackoffMilliseconds(2)).toEqual(4);
+        expect(backoffStrategy.getBackoffMilliseconds(3)).toEqual(8);
+        expect(backoffStrategy.getBackoffMilliseconds(4)).toEqual(16);
+        expect(backoffStrategy.getBackoffMilliseconds(5)).toEqual(32);
+      });
+    });
+  });
+
+  describe('LinearBackoffStrategy', () => {
+    describe('getNextBackoffMilliseconds', () => {
+      it('should calculate exponential backoff correctly with 5 retries', () => {
+        const backoffStrategy: BackoffStrategy = new LinearBackoffStrategy(2);
+
+        expect(backoffStrategy.getBackoffMilliseconds(1)).toEqual(2);
+        expect(backoffStrategy.getBackoffMilliseconds(2)).toEqual(2);
+        expect(backoffStrategy.getBackoffMilliseconds(3)).toEqual(2);
+        expect(backoffStrategy.getBackoffMilliseconds(4)).toEqual(2);
+        expect(backoffStrategy.getBackoffMilliseconds(5)).toEqual(2);
+      });
+    });
+  });
+});

libs/extensions/std/exec/src/util/backoff-strategy.ts

@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+export type BackoffStrategyHandle = 'exponential' | 'linear';
+export function isBackoffStrategyHandle(
+  v: unknown,
+): v is BackoffStrategyHandle {
+  return v === 'exponential' || v === 'linear';
+}
+
+export function createBackoffStrategy(
+  handle: BackoffStrategyHandle,
+  backoffMilliseconds: number,
+): BackoffStrategy {
+  if (handle === 'linear') {
+    return new LinearBackoffStrategy(backoffMilliseconds);
+  }
+  return new ExponentialBackoffStrategy(backoffMilliseconds);
+}
+
+export interface BackoffStrategy {
+  /**
+   * Calculates the backoff interval in milliseconds.
+   * @param retry the number of the current retry, starts counting with 1
+   */
+  getBackoffMilliseconds(retry: number): number;
+}
+
+/**
+ * Strategy for exponential backoffs.
+ * Uses seconds as unit for calculating the exponent.
+ */
+export class ExponentialBackoffStrategy implements BackoffStrategy {
+  constructor(private initialBackoffMilliseconds: number) {}
+
+  getBackoffMilliseconds(retry: number): number {
+    const initialBackoffSeconds = this.initialBackoffMilliseconds / 1000;
+    return Math.pow(initialBackoffSeconds, retry) * 1000;
+  }
+}
+
+/**
+ * Strategy for linear backoffs.
+ */
+export class LinearBackoffStrategy implements BackoffStrategy {
+  constructor(private backoffMilliseconds: number) {}
+
+  getBackoffMilliseconds(): number {
+    return this.backoffMilliseconds;
+  }
+}

libs/extensions/std/lang/src/http-extractor-meta-inf.ts

@@ -6,6 +6,7 @@ import {
   BlockMetaInformation,
   IOType,
   PrimitiveValuetypes,
+  evaluatePropertyValue,
 } from '@jvalue/jayvee-language-server';
 
 export class HttpExtractorMetaInformation extends BlockMetaInformation {
@@ -28,7 +29,126 @@ export class HttpExtractorMetaInformation extends BlockMetaInformation {
             ],
           },
         },
+        retries: {
+          type: PrimitiveValuetypes.Integer,
+          defaultValue: 0,
+          docs: {
+            description:
+              'Configures how many retries should be executed after a failure fetching the data.',
+            examples: [
+              {
+                code: 'retries: 3',
+                description:
+                  'Executes up to 3 retries if the original retry fails (so in total max. 4 requests).',
+              },
+            ],
+          },
+          validation: (property, validationContext, evaluationContext) => {
+            const encodingValue = evaluatePropertyValue(
+              property,
+              evaluationContext,
+              PrimitiveValuetypes.Integer,
+            );
+            if (encodingValue === undefined) {
+              return;
+            }
+
+            if (encodingValue < 0) {
+              validationContext.accept(
+                'error',
+                'Only not negative integers allowed',
+                {
+                  node: property,
+                  property: 'value',
+                },
+              );
+            }
+          },
+        },
+        retryBackoffMilliseconds: {
+          type: PrimitiveValuetypes.Integer,
+          defaultValue: 2000,
+          docs: {
+            description:
+              'Configures the wait time in milliseconds before executing a retry.',
+            examples: [
+              {
+                code: 'retryBackoff: 5000',
+                description: 'Waits 5s (5000 ms) before executing a retry.',
+              },
+            ],
+          },
+          validation: (property, validationContext, evaluationContext) => {
+            const minBockoffValue = 1000;
+
+            const encodingValue = evaluatePropertyValue(
+              property,
+              evaluationContext,
+              PrimitiveValuetypes.Integer,
+            );
+            if (encodingValue === undefined) {
+              return;
+            }
+
+            if (encodingValue < minBockoffValue) {
+              validationContext.accept(
+                'error',
+                `Only integers larger or equal to ${minBockoffValue} are allowed`,
+                {
+                  node: property,
+                  property: 'value',
+                },
+              );
+            }
+          },
+        },
+        retryBackoffStrategy: {
+          type: PrimitiveValuetypes.Text,
+          defaultValue: 'exponential',
+          docs: {
+            description:
+              'Configures the wait strategy before executing a retry. Can have values "exponential" or "linear".',
+            examples: [
+              {
+                code: 'retryBackoffStrategy: "linear"',
+                description:
+                  'Waits always the same amount of time before executing a retry.',
+              },
+              {
+                code: 'retryBackoffStrategy: "exponential"',
+                description:
+                  'Exponentially increases the wait time before executing a retry.',
+              },
+            ],
+          },
+          validation: (property, validationContext, evaluationContext) => {
+            const allowedValues = ['exponential', 'linear'];
+
+            const encodingValue = evaluatePropertyValue(
+              property,
+              evaluationContext,
+              PrimitiveValuetypes.Text,
+            );
+            if (encodingValue === undefined) {
+              return;
+            }
+
+            if (!allowedValues.includes(encodingValue)) {
+              validationContext.accept(
+                'error',
+                `Only the following values are allowed: ${allowedValues
+                  .map((v) => `"${v}"`)
+                  .join(', ')}`,
+                {
+                  node: property,
+                  property: 'value',
+                },
+              );
+            }
+          },
+        },
       },
+
       // Input type:
       IOType.NONE,