ci: Implement URL checker ESLint rule (#351)

* Implement URL checker rule * Consolidate ESLint tasks * Fix URLs in the comments
awslabs · Jan 18, 2024 · ead3e0e · ead3e0e
1 parent 946a0e0
commit ead3e0e
Show file tree

Hide file tree

Showing 20 changed files with 226 additions and 42 deletions.
diff --git a/.projenrc.ts b/.projenrc.ts
@@ -135,6 +135,7 @@ const fwkProject = new awscdk.AwsCdkConstructLibrary({
     '@types/eslint',
     'eslint-plugin-local-rules',
     'esbuild',
+    'sync-request-curl'
   ],
 
   bundledDeps: [
@@ -188,6 +189,7 @@ fwkProject.addPackageIgnore("!*.lit.ts");
 
 fwkProject.testTask.reset('jest --passWithNoTests --updateSnapshot --group=-e2e', {receiveArgs: true});
 fwkProject.testTask.spawn(new Task('eslint'));
+fwkProject.tasks.tryFind('eslint')!.exec(`eslint --ext .ts,.tsx --rule 'local-rules/url-checker: error' src`, { condition: '[ -n "$CI" ]' });
 
 fwkProject.addTask('test:e2e', {
   description: 'Run framework end-to-end tests',

diff --git a/framework/.projen/deps.json b/framework/.projen/deps.json
diff --git a/framework/.projen/tasks.json b/framework/.projen/tasks.json
diff --git a/framework/API.md b/framework/API.md
diff --git a/framework/eslint-local-rules/rules.ts b/framework/eslint-local-rules/rules.ts
@@ -1,11 +1,12 @@
-// Put the following comment above a code line you want to exclude from this check, e.g. after review:
-// eslint-disable-next-line local-rules/no-tokens-in-construct-id
-
-import type { Rule } from "eslint";
-import * as ts from "typescript";
+import { RuleTester, type Rule } from 'eslint';
+import * as ts from 'typescript';
+import request from 'sync-request-curl';
+import * as ESTree from 'estree';
 
 export default {
-    "no-tokens-in-construct-id": {
+    // Put the following comment above a code line you want to exclude from this check, e.g. after review:
+    // eslint-disable-next-line local-rules/no-tokens-in-construct-id
+    'no-tokens-in-construct-id': {
         meta: {
             docs: {
                 description: 'Checks whether a CDK token might appear in a construct ID on instantiation',
@@ -56,4 +57,50 @@ export default {
             };
         },
     },
-} satisfies Record<string, Rule.RuleModule>;
+    'url-checker': {
+        meta: {
+            docs: {
+                description: 'Checks the URLs in the comment-docs for accessibility'
+            },
+            schema: [],
+        },        
+        create: function (context: Rule.RuleContext) : Rule.RuleListener {
+            function checkProgramComment(program: ESTree.Program): void {
+                if (!program.comments) {
+                    return;
+                }
+
+                for (const c of program.comments.filter(c => c.type == 'Block')) {
+                    const matches = c.value.match(/\bhttps?:\/\/[\-A-Za-z0-9+&@#\/%?=~_|!:,.;]*[\-A-Za-z0-9+&@#\/%=~_|]/);
+                    if (!matches || matches.length == 0) {
+                        continue;
+                    }
+
+                    for (const url of matches) {
+                        try {
+                            const res = request('GET', url, { headers: { 'user-agent': 'libcurl/1.0.6'} });
+                            if (res.statusCode != 200) {
+                                context.report({
+                                    loc: c.loc!,                                    
+                                    message: `Fetching the URL '${url}' resulted in HTTP ${res.statusCode}`,
+                                });
+                            }
+                        }
+                        catch (e) {
+                            context.report({
+                                loc: c.loc!,
+                                message: `Failed to check the URL '${url}': ${e}`,
+                            });
+                        }
+                    }
+                }
+            }
+
+            return {
+                Program: checkProgramComment,
+            };
+        },
+    },
+} satisfies Record<string, Rule.RuleModule>;
+
+
diff --git a/framework/package.json b/framework/package.json
diff --git a/framework/src/governance/lib/data-catalog-database.ts b/framework/src/governance/lib/data-catalog-database.ts
@@ -11,7 +11,7 @@ import { Context, TrackedConstruct, TrackedConstructProps, Utils } from '../../u
 
 /**
  * An AWS Glue Data Catalog Database configured with the location and a crawler.
- * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/data-catalog-database
+ * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Governance/data-catalog-database
  *
  * @example
  * import { Bucket } from 'aws-cdk-lib/aws-s3';

diff --git a/framework/src/governance/lib/data-lake-catalog.ts b/framework/src/governance/lib/data-lake-catalog.ts
@@ -11,7 +11,7 @@ import { Context, TrackedConstruct, TrackedConstructProps } from '../../utils';
 
 /**
 * Creates AWS Glue Catalog Database for each storage layer. Composed of 3 {@link DataCatalogDatabase} for Bronze, Silver, and Gold data.
-* @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/data-lake-catalog
+* @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Governance/data-lake-catalog
  *
 * @example
 * import { Key } from 'aws-cdk-lib/aws-kms';

diff --git a/framework/src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts b/framework/src/processing/lib/cicd-pipeline/spark-emr-cicd-pipeline.ts
@@ -22,7 +22,7 @@ import { DEFAULT_SPARK_IMAGE, SparkImage } from '../emr-releases';
 
 /**
  * A CICD Pipeline that tests and deploys a Spark application in cross-account environments using CDK Pipelines.
- * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/spark-cicd-pipeline
+ * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-cicd-pipeline
  *
  * @exampleMetadata fixture=imports-only
  * @example

diff --git a/framework/src/processing/lib/spark-job/pyspark-application-package.ts b/framework/src/processing/lib/spark-job/pyspark-application-package.ts
@@ -15,7 +15,7 @@ import { Context, TrackedConstruct, TrackedConstructProps } from '../../../utils
 /**
  * A construct that takes your PySpark application, packages its virtual environment and uploads it along its entrypoint to an Amazon S3 bucket
  * This construct requires Docker daemon installed locally to run.
- * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/pyspark-application-package
+ * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/pyspark-application-package
  *
  * @example
  * let pysparkPacker = new dsf.processing.PySparkApplicationPackage (this, 'pysparkPacker', {

diff --git a/framework/src/processing/lib/spark-job/spark-job-emr-eks.ts b/framework/src/processing/lib/spark-job/spark-job-emr-eks.ts
@@ -16,7 +16,7 @@ import { EMR_DEFAULT_VERSION } from '../emr-releases';
 /**
  * A construct to run Spark Jobs using EMR on EKS.
  * Creates a Step Functions State Machine that orchestrates the Spark Job.
- * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/spark-job
+ * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-emr-serverless-job
  *
  * @example
  * import { JsonPath } from 'aws-cdk-lib/aws-stepfunctions';

diff --git a/framework/src/processing/lib/spark-job/spark-job-emr-serverless.ts b/framework/src/processing/lib/spark-job/spark-job-emr-serverless.ts
@@ -15,7 +15,7 @@ import { SparkEmrServerlessRuntime } from '../spark-runtime';
 /**
  * A construct to run Spark Jobs using EMR Serverless.
  * Creates a State Machine that orchestrates the Spark Job.
- * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/spark-job
+ * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-emr-serverless-job
  *
  * @example
  * import { PolicyDocument, PolicyStatement } from 'aws-cdk-lib/aws-iam';

diff --git a/framework/src/processing/lib/spark-job/spark-job.ts b/framework/src/processing/lib/spark-job/spark-job.ts
@@ -18,7 +18,7 @@ import { Context, TrackedConstruct, TrackedConstructProps, Utils } from '../../.
  * A base construct to run Spark Jobs.
  *
  * Creates an AWS Step Functions State Machine that orchestrates the Spark Job.
- * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/spark-job
+ * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-emr-serverless-job
  *
  * Available implementations:
  * * {@link SparkEmrServerlessJob} for Emr Serverless implementation

diff --git a/framework/src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts b/framework/src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts
@@ -47,7 +47,7 @@ import { DEFAULT_KARPENTER_VERSION } from '../../karpenter-releases';
 
 /**
  * A construct to create an EKS cluster, configure it and enable it with EMR on EKS
- * @see https://awslabs.github.io/aws-data-solutions-framework/docs/constructs/library/spark-emr-containers-runtime
+ * @see https://awslabs.github.io/data-solutions-framework-on-aws/docs/constructs/library/Processing/spark-emr-containers-runtime
 */
 export class SparkEmrContainersRuntime extends TrackedConstruct {
 
@@ -107,6 +107,8 @@ export class SparkEmrContainersRuntime extends TrackedConstruct {
    * The security group used by the EC2NodeClass of the default nodes
    */
   public readonly karpenterSecurityGroup?: ISecurityGroup;
+
+  // eslint-disable-next-line local-rules/url-checker
   /**
    * The rules used by Karpenter to track node health, rules are defined in the cloudformation below
    * https://raw.githubusercontent.com/aws/karpenter/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml