feat: add aws test suites

makeomatic · Aug 24, 2021 · 2414957 · 2414957
1 parent 99c7a1c
commit 2414957
Show file tree

Hide file tree

Showing 27 changed files with 2,701 additions and 285 deletions.
diff --git a/.mdeprc.js b/.mdeprc.js
@@ -8,6 +8,14 @@ module.exports = {
   post_exec: 'yarn coverage:report'
 };
 
+if (process.env.PROVIDER === 'aws') {
+  module.exports.tests = './test/suites/providers/aws/*.js';
+}
+
+if (process.env.PROVIDER !== 'aws') {
+  module.exports.tests = './test/suites/**/!(providers)/*.js';
+}
+
 switch (process.env.DB) {
   case 'sentinel':
     module.exports.services.push('redisSentinel');

diff --git a/package.json b/package.json
@@ -7,9 +7,10 @@
     "compile": "rimraf ./lib && babel -d ./lib --copy-files ./src",
     "pretest": "yarn compile",
     "test": "rimraf ./coverage && yarn lint && yarn test:e2e",
-    "test:e2e": "yarn test:e2e:cluster && yarn test:e2e:sentinel",
-    "test:e2e:cluster": "DB=cluster mdep test run",
-    "test:e2e:sentinel": "DB=sentinel mdep test run --docker_compose ./test/docker-compose.sentinel.yml",
+    "test:e2e": "yarn test:e2e:cluster && yarn test:e2e:sentinel && test:e2e:aws-provider",
+    "test:e2e:cluster": "PROVIDER=gce DB=cluster mdep test run",
+    "test:e2e:sentinel": "PROVIDER=gce DB=sentinel mdep test run --docker_compose ./test/docker-compose.sentinel.yml",
+    "test:e2e:aws-provider": "PROVIDER=aws DB=sentinel mdep test run --docker_compose ./test/docker-compose.sentinel.yml",
     "start": "mfleet",
     "lint": "eslint ./src ./test",
     "prepublishOnly": "yarn compile",

diff --git a/src/actions/upload.js b/src/actions/upload.js
@@ -38,6 +38,7 @@ const {
  * @return {Promise}
  */
 async function initFileUpload({ params }) {
+  console.log('init file upload');
   const {
     files,
     meta,

diff --git a/src/configs/core.js b/src/configs/core.js
@@ -93,6 +93,7 @@ exports.transport = [{
  * @returns {Provider}
  */
 exports.selectTransport = function selectTransport() {
+  // console.log('my provider 1', this.providers[0]);
   return this.providers[0];
 };
 

diff --git a/src/index.js b/src/index.js
@@ -164,8 +164,8 @@ class Files extends Microfleet {
     await super.connect();
     await this.initWebhook();
     await Promise.mapSeries(this.providers, (provider) => {
-      // @todo
-      if (provider.config.name !== 'gce' || provider.config.name !== 'aws') return null;
+      if (!['aws', 'gce'].includes(provider.config.name)) return null;
+
       if (!provider.config.bucket.channel.pubsub) return null;
       return provider.subscribe(this.handleUploadNotification.bind(this));
     });

diff --git a/src/providers/aws.js b/src/providers/aws.js
@@ -2,6 +2,7 @@ const Promise = require('bluebird');
 const AbstractFileTransfer = require('ms-files-transport');
 const { merge } = require('lodash');
 const S3 = require('aws-sdk/clients/s3');
+const SNS = require('aws-sdk/clients/sns');
 
 const DOWNLOAD_URL_EXPIRES_IN_SEC = 60000;
 
@@ -61,7 +62,25 @@ class AWSTransport extends AbstractFileTransfer {
    * @return {Subscription}
    */
   async subscribe() {
-    this.log.warn('the method is not implemented yet');
+    const { Topics: topics } = await new SNS({ region: this._config.aws.credentials.region }).listTopics({}).promise();
+
+    const topicArn = `arn:aws:sns:us-west-2:178085672309:${this._config.aws.credentials.topicName}`;
+
+    const topic = topics.find(
+      (_topic) => _topic.TopicArn === topicArn
+    );
+
+    if (!topic) {
+      await new SNS({ region: this._config.aws.credentials.region }).createTopic({ Name: this._config.aws.credentials.topicName }).promise();
+    }
+
+    const params = {
+      Protocol: 'https',
+      TopicArn: topicArn,
+      Endpoint: 'localhost:443',
+    };
+
+    await new SNS().subscribe(params).promise();
   }
 
   /**
@@ -93,6 +112,7 @@ class AWSTransport extends AbstractFileTransfer {
         if (!isBucketExist) {
           const bucketParams = {
             Bucket: bucketName,
+            region: this._config.aws.credentials.region,
           };
 
           await aws.createBucket(bucketParams, function handleCreateBucket(_err) {
@@ -125,11 +145,9 @@ class AWSTransport extends AbstractFileTransfer {
   }
 
   // @todo interface
-  getDownloadUrlSigned(filename, downloadName) {
-    this.log.warn(`${downloadName} is not implemented yet`);
-
+  getDownloadUrlSigned(filename) {
     const params = {
-      Bucket: this._config.bucketName,
+      Bucket: this.getBucketName(),
       Expires: DOWNLOAD_URL_EXPIRES_IN_SEC,
       Key: filename,
     };
@@ -174,11 +192,9 @@ class AWSTransport extends AbstractFileTransfer {
     return new Promise((resolve, reject) => {
       this._aws.getSignedUrl('putObject', params, (err, url) => {
         if (err) {
-          console.log('init resumable upload err', err);
           return reject(err);
         }
 
-        console.log('init resumable upload url', url);
         return resolve(url);
       });
     });
@@ -239,15 +255,11 @@ class AWSTransport extends AbstractFileTransfer {
     const params = {
       Bucket: this._config.bucket.name,
       Expires: DOWNLOAD_URL_EXPIRES_IN_SEC,
-      Key: opts.filename,
+      Key: opts.resource,
     };
 
-    params.ContentType = opts.contentType;
-
     return new Promise((resolve, reject) => {
-      console.log('signed url params', params);
       this._aws.getSignedUrl('putObject', params, (err, url) => {
-        console.log('signed url err', err);
         if (err) {
           return reject(err);
         }
@@ -305,12 +317,10 @@ class AWSTransport extends AbstractFileTransfer {
     console.log(`exists method aws for: ${filename}`);
 
     return new Promise((resolve) => {
-      this._aws.headObject({ Key: filename, Bucket: this._config.bucket.name }, (err, data) => {
+      this._aws.headObject({ Key: filename, Bucket: this._config.bucket.name }, (err) => {
         if (err) {
-          console.log('exists err', err);
           return resolve(false);
         }
-        console.log('exists data', data);
         return resolve(true);
       });
     });
@@ -322,7 +332,14 @@ class AWSTransport extends AbstractFileTransfer {
    * @return {Promise}
    */
   remove(filename) {
-    this.log.warn('the method is not implemented yet', { filename });
+    return new Promise((resolve) => {
+      this._aws.deleteObject({ Key: filename, Bucket: this._config.bucket.name }, (err) => {
+        if (err) {
+          return resolve(false);
+        }
+        return resolve(true);
+      });
+    });
   }
 }
 
@@ -348,7 +365,7 @@ AWSTransport.defaultOpts = {
         token: undefined,
       },
     },
-    mсetadata: {},
+    metadata: {},
   },
 };
 

diff --git a/src/utils/fetch-data.js b/src/utils/fetch-data.js
@@ -102,6 +102,7 @@ async function selectMaster(redis) {
  * @param  {String[]} [fieldsFilter.pick]
  */
 module.exports = function fetchData(key, fieldFilter = {}) {
+  console.log('fetch data for key', key);
   const { redis } = this;
   const timer = perf(`fetchData:${key}`);
 

diff --git a/src/utils/process.js b/src/utils/process.js
@@ -31,6 +31,7 @@ module.exports = function processFile(key, data) {
   return Promise
     .using(acquireLock(this, `postprocess:${key}`), (lock) => {
       const { uploadId } = data;
+      console.log('port srocess uploadId', uploadId);
       const { redis } = this;
 
       return Promise

diff --git a/test/configs/generic/core.js b/test/configs/generic/core.js
@@ -19,36 +19,16 @@ exports.amqp = {
   },
 };
 
-exports.transport = [
-  //   {
-  //   name: 'gce',
-  //   options: {
-  //     gce: {
-  //       projectId: env.GCLOUD_PROJECT_ID,
-  //       credentials: {
-  //         client_email: env.GCLOUD_PROJECT_EMAIL,
-  //         private_key: env.GCLOUD_PROJECT_PK,
-  //       },
-  //     },
-  //     bucket: {
-  //       name: env.TEST_BUCKET,
-  //       metadata: {
-  //         location: env.GCLOUD_BUCKET_LOCATION || 'EUROPE-WEST1',
-  //         dra: true,
-  //       },
-  //     },
-  //     // test for direct public URLs
-  //   },
-  //   // its not a public name!
-  //   cname: 'gce',
-  // },
+const awsTransport = [
   {
     name: 'aws',
     options: {
       aws: {
         credentials: {
+          region: env.AWS_REGION,
           accessKeyId: env.AWS_ACCESS_KEY_ID,
           secretAccessKey: env.AWS_SECRET_ACCESS_KEY,
+          topicName: env.TOPIC_NAME,
         },
       },
       bucket: {
@@ -58,7 +38,35 @@ exports.transport = [
     },
     // its not a public name!
     cname: 'aws',
-  }];
+  },
+];
+
+// const gceTransport = [
+//   {
+//     name: 'gce',
+//     options: {
+//       gce: {
+//         projectId: env.GCLOUD_PROJECT_ID,
+//         credentials: {
+//           client_email: env.GCLOUD_PROJECT_EMAIL,
+//           private_key: env.GCLOUD_PROJECT_PK,
+//         },
+//       },
+//       bucket: {
+//         name: env.TEST_BUCKET,
+//         metadata: {
+//           location: env.GCLOUD_BUCKET_LOCATION || 'EUROPE-WEST1',
+//           dra: true,
+//         },
+//       },
+//       // test for direct public URLs
+//     },
+//     // its not a public name!
+//     cname: 'gce',
+//   },
+// ];
+
+exports.transport = env.PROVIDER === 'aws' ? awsTransport : awsTransport;
 
 exports.hooks = {
   // return input, assume there are models

diff --git a/test/docker-compose.aws-provider.yml b/test/docker-compose.aws-provider.yml
@@ -0,0 +1,16 @@
+version: '3'
+
+services:
+  tester:
+    volumes:
+      - ${PWD}:/src
+      - ${PWD}/test/configs/generic:/configs/core:ro
+      # - ${PWD}/test/configs/generic:/configs/aws:ro
+      - ${PWD}/test/configs/redis-sentinel:/configs/redis:ro
+    environment:
+      NODE_ENV: "test"
+      DEBUG: "${DEBUG}"
+      TEST_BUCKET: "makeomatic-test"
+      DOTENV_FILE_PATH: "/src/test/.env"
+      NCONF_FILE_PATH: '["/configs/core","/configs/redis"]'
+
diff --git a/test/helpers/utils.js b/test/helpers/utils.js
@@ -177,38 +177,41 @@ function modelSimpleUpload({
 // upload single file
 //
 function upload(location, file) {
+  console.log('upload location', location);
   return request.put({
     url: location,
     body: file,
-    headers: {
-      'content-length': file.length,
-    },
+    // headers: {
+    //   'content-length': file.length,
+    // },
     simple: false,
     resolveWithFullResponse: true,
   });
 }
 
-async function uploadSimple(meta, file, isPublic) {
-  const { query: { Expires } } = url.parse(meta.location);
+async function uploadSimple(meta, file) {
+  // const { query: { Expires } } = url.parse(meta.location);
 
-  const headers = {
-    'Content-MD5': meta.md5Hash,
-    'Cache-Control': `public,max-age=${Expires}`,
-    'Content-Type': meta.contentType,
-  };
+  // const headers = {
+  //   'Content-MD5': meta.md5Hash,
+  //   'Cache-Control': `public,max-age=${Expires}`,
+  //   'Content-Type': meta.contentType,
+  // };
 
-  if (isPublic) {
-    headers['x-goog-acl'] = 'public-read';
-  }
+  // if (isPublic) {
+  //   headers['x-goog-acl'] = 'public-read';
+  // }
 
-  return request.put({
+  const res = request.put({
     url: meta.location,
     body: file,
     // headers,
     // simple: false,
     resolveWithFullResponse: true,
-    ACL: 'public-read',
+    // ACL: 'public-read',
   });
+
+  return res;
 }
 
 //
@@ -247,10 +250,12 @@ function finishMessage(rsp, skipProcessing = true) {
 // Initializes upload
 //
 function initUpload(data) {
+  console.log('init upload data', data);
   return function init() {
     return this.amqp
       .publishAndWait('files.upload', data.message, { timeout: 30000 })
       .tap((rsp) => {
+        console.log('files upload res', rsp);
         this.response = rsp;
       })
       .tap((rsp) => uploadFiles(data, rsp));
-Original file line number
+Diff line change
@@ Expand Up / @@ -38,6 +38,7 @@ const { @@
      * @return {Promise}
      */
     async function initFileUpload({ params }) {
+      console.log('init file upload');
       const {
         files,
         meta,
@@ Expand Down @@