[Components] Support file URLS and file paths in a single prop input

.github/workflows/pull-request-checks.yaml

@@ -96,6 +96,8 @@ jobs:
           format: 'csv'
       # NOTE: These steps are kept in this workflow to avoid re-rerunning the rest of the lint job
       # in the Components Checks workflow 
+      - name: Build TypeScript Components
+        run: pnpm build
       - name: Check component keys
         run: node scripts/findBadKeys.js ${{ steps.changed_files.outputs.added_modified }} ${{ steps.changed_files.outputs.renamed }}
       - name: Check component app prop

components/aws/actions/s3-upload-file/s3-upload-file.mjs → components/aws/actions/s3-upload-base64-as-file/s3-upload-base64-as-file.mjs

@@ -1,15 +1,11 @@
 import common from "../../common/common-s3.mjs";
-import { toSingleLineString } from "../../common/utils.mjs";
 
 export default {
   ...common,
-  key: "aws-s3-upload-file",
-  name: "S3 - Upload File - Base64",
-  description: toSingleLineString(`
-    Accepts a base64-encoded string and a filename, then uploads as a file to S3.
-    [See the docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html)
-  `),
-  version: "0.3.2",
+  key: "aws-s3-upload-base64-as-file",
+  name: "S3 - Upload Base64 As File",
+  description: "Accepts a base64-encoded string and a filename, then uploads as a file to S3. [See the documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html)",
+  version: "0.0.1",
   type: "action",
   props: {
     aws: common.props.aws,

components/aws/actions/s3-upload-file-tmp/s3-upload-file-tmp.mjs → components/aws/actions/s3-upload-files/s3-upload-files.mjs

@@ -1,18 +1,17 @@
-import common from "../../common/common-s3.mjs";
-import fs from "fs";
 import { join } from "path";
-import { toSingleLineString } from "../../common/utils.mjs";
-import { ConfigurationError } from "@pipedream/platform";
+import fs from "fs";
+import {
+  getFileStreamAndMetadata,
+  ConfigurationError,
+} from "@pipedream/platform";
+import common from "../../common/common-s3.mjs";
 
 export default {
   ...common,
-  key: "aws-s3-upload-file-tmp",
-  name: "S3 - Upload Files - /tmp",
-  description: toSingleLineString(`
-    Accepts a file path or folder path starting from /tmp, then uploads the contents to S3.
-    [See the docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html)
-  `),
-  version: "1.0.3",
+  key: "aws-s3-upload-files",
+  name: "S3 - Upload Files",
+  description: "Upload files to S3. Accepts either a file URL, a local file path, or a directory path. [See the documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html)",
+  version: "0.0.1",
   type: "action",
   props: {
     aws: common.props.aws,
@@ -25,8 +24,8 @@ export default {
     },
     path: {
       type: "string",
-      label: "File Or Folder Path",
-      description: "Path starting from `/tmp`. If it's a directory, all files will be uploaded.",
+      label: "File Path, Url, Or Folder Path",
+      description: "Provide either a file URL, a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`), or a directory path to upload all files.",
     },
     customFilename: {
       type: common.props.key.type,
@@ -37,6 +36,17 @@ export default {
   },
   methods: {
     ...common.methods,
+    streamToBase64(stream) {
+      return new Promise((resolve, reject) => {
+        const chunks = [];
+        stream.on("data", (chunk) => chunks.push(chunk));
+        stream.on("end", () => {
+          const buffer = Buffer.concat(chunks);
+          resolve(buffer.toString("base64"));
+        });
+        stream.on("error", reject);
+      });
+    },
     getFilesRecursive(dir) {
       let results = [];
       const items = fs.readdirSync(dir);
@@ -59,16 +69,19 @@ export default {
       } = this;
       const files = this.getFilesRecursive(folderPath);
       const response = await Promise.all(files.map(async (filePath) => {
-        const fileContent = fs.readFileSync(filePath, {
-          encoding: "base64",
-        });
+        const {
+          stream,
+          metadata,
+        } = await getFileStreamAndMetadata(filePath);
         const relativePath = filePath.substring(folderPath.length + 1);
         const s3Key = join(prefix, relativePath);
 
         await uploadFile({
           Bucket: bucket,
           Key: s3Key,
-          Body: Buffer.from(fileContent, "base64"),
+          Body: stream,
+          ContentType: metadata.contentType,
+          ContentLength: metadata.size,
         });
         return {
           filePath,
@@ -87,15 +100,18 @@ export default {
         customFilename,
       } = this;
 
-      const file = fs.readFileSync(filePath, {
-        encoding: "base64",
-      });
+      const {
+        stream,
+        metadata,
+      } = await getFileStreamAndMetadata(filePath);
       const filename = customFilename || filePath.split("/").pop();
 
       const response = await uploadFile({
         Bucket: bucket,
         Key: join(prefix, filename),
-        Body: Buffer.from(file, "base64"),
+        Body: stream,
+        ContentType: metadata.contentType,
+        ContentLength: metadata.size,
       });
 
       $.export("$summary", `Uploaded file ${filename} to S3`);
@@ -108,9 +124,17 @@ export default {
       uploadFolderFiles,
       path,
     } = this;
+
+    // If path is a URL, treat it as a single file
+    if (path.startsWith("http://") || path.startsWith("https://")) {
+      return await uploadSingleFile($, path);
+    }
+
+    // For local paths, check if it exists
     if (!fs.existsSync(path)) {
       throw new ConfigurationError(`The file or directory path \`${path}\` does not exist. Please verify the path and include the leading /tmp if needed.`);
     }
+
     const stat = fs.statSync(path);
     return stat.isDirectory()
       ? await uploadFolderFiles($, path)

components/aws/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/aws",
-  "version": "0.7.9",
+  "version": "1.0.0",
   "description": "Pipedream Aws Components",
   "main": "aws.app.mjs",
   "keywords": [
@@ -29,7 +29,7 @@
     "@aws-sdk/s3-request-presigner": "^3.609.0",
     "@aws-sdk/signature-v4-crt": "^3.731.0",
     "@pipedream/helper_functions": "^0.3.6",
-    "@pipedream/platform": "^3.0.3",
+    "@pipedream/platform": "^3.1.0",
     "adm-zip": "^0.5.10",
     "dedent": "^1.5.1",
     "mailparser": "^3.6.6",

components/click2mail2/actions/create-document/create-document.mjs

@@ -1,26 +1,16 @@
 import FormData from "form-data";
-import fs from "fs";
+import { getFileStreamAndMetadata } from "@pipedream/platform";
 import click2mail2 from "../../click2mail2.app.mjs";
 import { FORMATS } from "../../common/constants.mjs";
 
 export default {
   key: "click2mail2-create-document",
   name: "Create Document",
-  version: "0.0.1",
+  version: "1.0.0",
   description: "Creates a new document in your account from an uploaded file or a URL. [See the documentation for file](https://developers.click2mail.com/reference/createdocument_1).  [See the documentation for URL](https://developers.click2mail.com/reference/createdocumentfromurl)",
   type: "action",
   props: {
     click2mail2,
-    uploadType: {
-      type: "string",
-      label: "Upload Type",
-      description: "The type of the upload.",
-      reloadProps: true,
-      options: [
-        "URL",
-        "File",
-      ],
-    },
     documentName: {
       type: "string",
       label: "Document Name",
@@ -39,49 +29,51 @@ export default {
         "documentClass",
       ],
     },
-  },
-  async additionalProps() {
-    const props = {};
-    if (this.uploadType === "URL") {
-      props.url = {
-        type: "string",
-        label: "URL",
-        description: "Document url",
-      };
-    } else {
-      props.file = {
-        type: "string",
-        label: "File",
-        description: "Path of the file in /tmp folder. To upload a file to /tmp folder, please follow the [doc here](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)",
-      };
-    }
-    return props;
+    file: {
+      type: "string",
+      label: "File Path Or Url",
+      description: "Provide either a file URL or a path to a file in the `/tmp` directory (for example, `/tmp/myFile.pdf`).",
+    },
   },
   async run({ $ }) {
     const {
       click2mail2,
-      uploadType,
       file,
       ...params
     } = this;
 
+    const isUrl = file.startsWith("http://") || file.startsWith("https://");
     let objToSend = {};
 
-    if (uploadType === "File") {
+    if (!isUrl) {
+      const {
+        stream, metadata,
+      } = await getFileStreamAndMetadata(file);
       const formData = new FormData();
-      formData.append("file", fs.createReadStream(file));
+      formData.append("file", stream, {
+        contentType: metadata.contentType,
+        knownLength: metadata.size,
+        filename: metadata.name,
+      });
 
       objToSend = {
         data: formData,
         headers: formData.getHeaders(),
       };
+    } else {
+      objToSend = {
+        params: {
+          ...params,
+          url: file,
+        },
+      };
     }
+
     const response = await click2mail2.create({
       $,
-      path: `${uploadType === "File"
+      path: `${!isUrl
         ? "documents"
         : "documents/url"}`,
-      params,
       ...objToSend,
     });
 

components/click2mail2/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/click2mail2",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "description": "Pipedream Click2Mail Components",
   "main": "click2mail2.app.mjs",
   "keywords": [
@@ -13,9 +13,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@pipedream/platform": "^1.5.1",
+    "@pipedream/platform": "^3.1.0",
     "form-data": "^4.0.0",
     "fs": "^0.0.1-security"
   }
 }
-