Add ACL to S3 bucket and object uploads (#5)

* Add ACL to S3 bucket and object uploads

* Update comments

* Update ACL value to undefined and add new function createValidInputFixtureWithACLBucketOwner

* Add extractAfterSubdirectory function and use it in getLocalFiles

* Remove commented out code for S3 credentials

* Add commands to SyncCloudStorage class

* Refactor tags method and handle error when getting existing tags

* test: added test case for getLocalFiles

* Improved multiple files test case

* Fix storage existence check and remove unnecessary code

.eslintrc.json

@@ -10,7 +10,8 @@
   "plugins": ["import"],
   "parserOptions": {
     "ecmaVersion": 2023,
-    "sourceType": "module"
+    "sourceType": "module",
+    "project": "./tsconfig.json"
   },
   "rules": {
     "semi": "off",
@@ -52,7 +53,8 @@
     ],
     "no-console": 2,
     "no-unused-vars": 2,
-    "@typescript-eslint/no-unused-vars": ["error"]
+    "@typescript-eslint/no-unused-vars": ["error"],
+    "import/named": "error"
   },
   "env": {
     "node": true
@@ -64,7 +66,11 @@
     "import/resolver": {
       "typescript": {
         "alwaysTryTypes": true,
-        "project": "./tsconfig.json"
+        "project": [
+          "./tsconfig.json",
+          "./src/tsconfig.json",
+          "./test/tsconfig.json"
+        ]
       }
     }
   }

docs/global.html

@@ -218,7 +218,7 @@ <h5>Parameters:</h5>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="utils_objects.js.html">utils/objects.js</a>, <a href="utils_objects.js.html#line23">line 23</a>
+        <a href="utils_objects.js.html">utils/objects.js</a>, <a href="utils_objects.js.html#line27">line 27</a>
     </li></ul></dd>
 
 
@@ -292,7 +292,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/index.html

@@ -70,7 +70,6 @@ <h3>Serverless</h3>
       metadata:
         foo: bar
         bar: foo
-      acl: public-read
 </code></pre>
 <h2>Deployments</h2>
 <h3><a href="https://github.com/common-fate/granted">Granted</a> (assume)</h3>
@@ -93,7 +92,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/index.js.html

@@ -34,7 +34,6 @@ <h1 class="page-title">Source: index.js</h1>
 const client_s3_1 = require("@aws-sdk/client-s3");
 const errors_1 = require("./errors");
 const buckets_1 = require("./providers/s3/buckets");
-// import { getCredentials } from './providers/s3/credentials'
 const input_1 = require("./schemas/input");
 const logger_1 = __importDefault(require("./utils/logger"));
 /**
@@ -45,6 +44,7 @@ <h1 class="page-title">Source: index.js</h1>
     serverless;
     options;
     hooks;
+    commands;
     servicePath;
     config;
     logging;
@@ -83,6 +83,7 @@ <h1 class="page-title">Source: index.js</h1>
         this.config = validConfig;
         this.client = this.getS3Client();
         this._storages = this.config.syncCloudStorage.storages.filter((bucket) => bucket.enabled);
+        this.commands = this.setCommands();
         this.hooks = this.setHooks();
     }
     /**
@@ -103,6 +104,22 @@ <h1 class="page-title">Source: index.js</h1>
             endpoint,
         });
     }
+    /**
+     * Set commands.
+     * @returns {ServerlessPlugin.Commands} Commands
+     * @memberof SyncCloudStorage
+     *
+     * @example
+     * const commands = this.setCommands()
+     */
+    setCommands() {
+        return {
+            scs: {
+                usage: 'Sync Cloud Storage',
+                lifecycleEvents: ['storages', 'tags'],
+            },
+        };
+    }
     /**
      * Set hooks.
      * @returns {ServerlessPlugin.Hooks} Hooks
@@ -115,10 +132,10 @@ <h1 class="page-title">Source: index.js</h1>
         const syncStoragesHook = () => this.storages();
         const syncTagsHook = () => this.tags();
         return {
-            'before:offline:start:init': syncStoragesHook,
-            'scs:buckets': syncStoragesHook,
+            'scs:storages': syncStoragesHook,
             'scs:tags': syncTagsHook,
-            'before:deploy:deploy': () => syncStoragesHook(),
+            'before:offline:start:init': syncStoragesHook,
+            'before:deploy:deploy': syncStoragesHook,
         };
     }
     /**
@@ -153,20 +170,18 @@ <h1 class="page-title">Source: index.js</h1>
     /**
      * Sync tags.
      * @private
-     * @returns {TagsMethodPromiseResult}
      * @memberof SyncCloudStorage
      * @see https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3
-     *
+     * @returns {Promise&lt;TagsSyncResults>}
      * @example
      * const result = await this.tags()
      */
     async tags() {
-        const isPluginEnable = this.disableCheck().result;
-        if (!isPluginEnable) {
-            return [];
+        const isPluginDisable = this.disableCheck().result;
+        if (isPluginDisable) {
+            return [{ error: 'Plugin is disabled' }];
         }
-        const syncedStorages = (await Promise.allSettled(this._storages.map((bucket) => (0, buckets_1.syncTags)(this.client, bucket))));
-        return syncedStorages;
+        return (await Promise.allSettled(this._storages.map((bucket) => (0, buckets_1.syncTags)(this.client, bucket))));
     }
     /**
      * On exit.
@@ -211,7 +226,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/module-SyncCloudStorage-SyncCloudStorage.html

@@ -253,7 +253,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/module-SyncCloudStorage.html

@@ -69,7 +69,7 @@ <h1 class="page-title">Module: SyncCloudStorage</h1>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="index.js.html">index.js</a>, <a href="index.js.html#line12">line 12</a>
+        <a href="index.js.html">index.js</a>, <a href="index.js.html#line11">line 11</a>
     </li></ul></dd>
 
 
@@ -127,7 +127,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/providers_local_objects.js.html

@@ -58,6 +58,7 @@ <h1 class="page-title">Source: providers/local/objects.js</h1>
 const crypto_1 = require("crypto");
 const fs = __importStar(require("fs"));
 const path_1 = __importDefault(require("path"));
+const objects_1 = require("../../utils/objects");
 /**
  * Returns a list of local files recursively.
  * @memberof Local
@@ -76,9 +77,14 @@ <h1 class="page-title">Source: providers/local/objects.js</h1>
             recursiveFiles.push(...innerFiles);
         }
         else {
+            const internalPath = (0, objects_1.extractAfterSubdirectory)(fullPath, storage.localPath);
+            let key = internalPath;
+            if (storage.prefix) {
+                key = path_1.default.join(storage.prefix, internalPath);
+            }
             const file = {
                 LocalPath: fullPath,
-                Key: storage.prefix ? path_1.default.join(`${storage.prefix}/${item}`) : item,
+                Key: key,
                 LastModified: new Date(stat.mtime),
                 Size: stat.size,
                 ETag: await getFileETag(fs.createReadStream(fullPath)),
@@ -112,7 +118,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/providers_s3_buckets.js.html

@@ -50,7 +50,7 @@ <h1 class="page-title">Source: providers/s3/buckets.js</h1>
 };
 exports.storageExists = storageExists;
 /**
- * Syncs storage with upload and delete actions by comparing local file and storage's object checksums.
+ * Syncs storage with upload and delete actions by comparing local file and storage's object `${Key}-${ETag}`.
  * @memberof S3
  * @param {S3Client} client
  * @param {Storage} storage
@@ -101,7 +101,6 @@ <h1 class="page-title">Source: providers/s3/buckets.js</h1>
  * @returns {Promise&lt;SyncMetadataReturn>}
  */
 const syncMetadata = async (client, storage) => {
-    // Get list of existing objects
     const existingObjects = await (0, objects_1.listObjects)(client, storage);
     const syncedMetadata = [];
     for (const file of existingObjects) {
@@ -116,14 +115,14 @@ <h1 class="page-title">Source: providers/s3/buckets.js</h1>
             ContentType: (0, objects_2.getContentType)(file.Key),
             MetadataDirective: client_s3_1.MetadataDirective.REPLACE,
             Metadata: storage.metadata,
+            ACL: storage.acl,
         });
         const result = await client.send(copyCommand);
         logger_1.default.info('Metadata synced', {
             storage: storage.name,
             Key: file.Key,
             result,
         });
-        // Get Object metadata
         const headCommand = await client.send(new client_s3_1.HeadObjectCommand({
             Bucket: storage.name,
             Key: storage.prefix
@@ -148,19 +147,32 @@ <h1 class="page-title">Source: providers/s3/buckets.js</h1>
  */
 const syncTags = async (client, storage) => {
     logger_1.default.info("Syncing storage's tags", { storage: storage.name });
+    let existingTagSet = { TagSet: [] };
     try {
-        const existingTagSetCommand = new client_s3_1.GetBucketTaggingCommand({
-            Bucket: storage.name,
-        });
-        const existingTagSet = await client.send(existingTagSetCommand);
+        try {
+            const existingTagSetCommand = new client_s3_1.GetBucketTaggingCommand({
+                Bucket: storage.name,
+            });
+            existingTagSet = await client.send(existingTagSetCommand);
+        }
+        catch (error) {
+            if (error.name === 'NoSuchTagSet') {
+                existingTagSet = { TagSet: [] };
+            }
+            else {
+                logger_1.default.error('Failed to get existing tags', {
+                    storage: storage.name,
+                    error: JSON.stringify(error),
+                });
+            }
+        }
         const mergedTagSet = (0, tags_1.mergeTags)(existingTagSet.TagSet, storage.tags ?? {});
-        const command = new client_s3_1.PutBucketTaggingCommand({
+        await client.send(new client_s3_1.PutBucketTaggingCommand({
             Bucket: storage.name,
             Tagging: {
                 TagSet: mergedTagSet,
             },
-        });
-        await client.send(command);
+        }));
         logger_1.default.info("Synced storage's tags", {
             storage: storage.name,
             existingTagSet: existingTagSet.TagSet,
@@ -219,7 +231,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/providers_s3_credentials.js.html

@@ -79,7 +79,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/providers_s3_objects.js.html

@@ -86,6 +86,7 @@ <h1 class="page-title">Source: providers/s3/objects.js</h1>
                 Key: fileToUpload.Key,
                 Body: fs_1.default.createReadStream(fileToUpload.LocalPath),
                 ContentType: (0, objects_1.getContentType)(fileToUpload.Key),
+                ACL: storage.acl,
             },
         });
         logger_1.default.info('Uploaded file to bucket', {
@@ -182,7 +183,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/utils_objects.js.html

@@ -27,8 +27,12 @@ <h1 class="page-title">Source: utils/objects.js</h1>
     <section>
         <article>
             <pre class="prettyprint source linenums"><code>"use strict";
+var __importDefault = (this &amp;&amp; this.__importDefault) || function (mod) {
+    return (mod &amp;&amp; mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getContentType = exports.getChecksum = void 0;
+exports.extractAfterSubdirectory = exports.getContentType = exports.getChecksum = void 0;
+const path_1 = __importDefault(require("path"));
 const mrmime_1 = require("mrmime");
 /**
  * Get a checksum for an object
@@ -53,6 +57,25 @@ <h1 class="page-title">Source: utils/objects.js</h1>
     return key ? (0, mrmime_1.lookup)(key) ?? defaultMimeType : defaultMimeType;
 };
 exports.getContentType = getContentType;
+const extractAfterSubdirectory = (fullPath, subdirectory) => {
+    // Normalize both paths to ensure consistent separators
+    const normalizedFullPath = path_1.default.normalize(fullPath);
+    const normalizedSubdirectory = path_1.default.normalize(subdirectory);
+    // Find the start index of the subdirectory in the full path
+    const startIndex = normalizedFullPath.indexOf(normalizedSubdirectory);
+    if (startIndex !== -1) {
+        // Calculate the end index of the subdirectory within the full path
+        const endIndex = startIndex + normalizedSubdirectory.length + 1;
+        // Extract the part of the full path after the subdirectory
+        const afterSubdirectory = normalizedFullPath.substring(endIndex);
+        // Normalize the extracted path to clean up any leading separators
+        return path_1.default.normalize(afterSubdirectory);
+    }
+    else {
+        return ''; // Subdirectory not found in the full path
+    }
+};
+exports.extractAfterSubdirectory = extractAfterSubdirectory;
 </code></pre>
         </article>
     </section>
@@ -69,7 +92,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/utils_tags.js.html

@@ -70,7 +70,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-Sy
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Sun Feb 04 2024 22:41:43 GMT+0100 (Central European Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.2</a> on Fri Feb 09 2024 00:49:08 GMT+0100 (Central European Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>