CLDSRV-409: use latest s3cmd with python3

.github/workflows/tests.yaml

@@ -222,13 +222,10 @@ jobs:
           3.9
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
-    - name: Setup python2 test environment
+    - name: Setup python test environment
       run: |
         sudo apt-get install -y libdigest-hmac-perl
-        pip install virtualenv==20.21.0
-        virtualenv -p $(which python2) ~/.virtualenv/py2
-        source ~/.virtualenv/py2/bin/activate
-        pip install 's3cmd==1.6.1'
+        pip install 's3cmd==2.3.0'
     - name: Setup CI services
       run: docker-compose up -d
       working-directory: .github/docker

tests/functional/s3cmd/tests.js

@@ -165,7 +165,9 @@ function readJsonFromChild(child, lineFinder, cb) {
         const findBrace = data.indexOf('{', findLine);
         const findEnd = findEndJson(data, findBrace);
         const endJson = data.substring(findBrace, findEnd + 1)
-            .replace(/"/g, '\\"').replace(/'/g, '"');
+            .replace(/"/g, '\\"').replace(/'/g, '"')
+            .replace(/b'/g, '\'')
+            .replace(/b"/g, '"');
         return cb(JSON.parse(endJson));
     });
 }
@@ -262,7 +264,7 @@ describe('s3cmd putBucket', () => {
         exec([
             'mb', `s3://${bucket}`,
             '--bucket-location=scality-us-west-1',
-        ], done, 13);
+        ], done, 11);
     });
 
     it('put an invalid bucket, should fail', done => {
@@ -344,18 +346,18 @@ describe('s3cmd getService', () => {
 
     it("should have response headers matching AWS's response headers",
         done => {
-            provideLineOfInterest(['ls', '--debug'], 'DEBUG: Response: {',
+            provideLineOfInterest(['ls', '--debug'], '\'headers\': {',
             parsedObject => {
-                assert(parsedObject.headers['x-amz-id-2']);
-                assert(parsedObject.headers['transfer-encoding']);
-                assert(parsedObject.headers['x-amz-request-id']);
-                const gmtDate = new Date(parsedObject.headers.date)
+                assert(parsedObject['x-amz-id-2']);
+                assert(parsedObject['transfer-encoding']);
+                assert(parsedObject['x-amz-request-id']);
+                const gmtDate = new Date(parsedObject.date)
                     .toUTCString();
-                assert.strictEqual(parsedObject.headers.date, gmtDate);
+                assert.strictEqual(parsedObject.date, gmtDate);
                 assert.strictEqual(parsedObject
-                    .headers['content-type'], 'application/xml');
+                    ['content-type'], 'application/xml');
                 assert.strictEqual(parsedObject
-                    .headers['set-cookie'], undefined);
+                    ['set-cookie'], undefined);
                 done();
             });
         });
@@ -395,11 +397,11 @@ describe('s3cmd getObject', function toto() {
     });
 
     it('get non existing file in existing bucket, should fail', done => {
-        exec(['get', `s3://${bucket}/${nonexist}`, 'fail'], done, 12);
+        exec(['get', `s3://${bucket}/${nonexist}`, 'fail'], done, 64);
     });
 
     it('get file in non existing bucket, should fail', done => {
-        exec(['get', `s3://${nonexist}/${nonexist}`, 'fail2'], done, 12);
+        exec(['get', `s3://${nonexist}/${nonexist}`, 'fail2'], done, 64);
     });
 });
 
@@ -511,22 +513,22 @@ describe('s3cmd delObject', () => {
 
     it('delete an already deleted object, should return a 204', done => {
         provideLineOfInterest(['rm', `s3://${bucket}/${upload}`, '--debug'],
-        'DEBUG: Response: {', parsedObject => {
+        'DEBUG: Response:\n{', parsedObject => {
             assert.strictEqual(parsedObject.status, 204);
             done();
         });
     });
 
     it('delete non-existing object, should return a 204', done => {
         provideLineOfInterest(['rm', `s3://${bucket}/${nonexist}`, '--debug'],
-        'DEBUG: Response: {', parsedObject => {
+        'DEBUG: Response:\n{', parsedObject => {
             assert.strictEqual(parsedObject.status, 204);
             done();
         });
     });
 
     it('try to get the deleted object, should fail', done => {
-        exec(['get', `s3://${bucket}/${upload}`, download], done, 12);
+        exec(['get', `s3://${bucket}/${upload}`, download], done, 64);
     });
 });
 
@@ -621,7 +623,7 @@ describe('s3cmd multipart upload', function titi() {
     });
 
     it('should not be able to get deleted object', done => {
-        exec(['get', `s3://${bucket}/${MPUpload}`, download], done, 12);
+        exec(['get', `s3://${bucket}/${MPUpload}`, download], done, 64);
     });
 });
 
@@ -660,7 +662,7 @@ MPUploadSplitter.forEach(file => {
         });
 
         it('should not be able to get deleted object', done => {
-            exec(['get', `s3://${bucket}/${file}`, download], done, 12);
+            exec(['get', `s3://${bucket}/${file}`, download], done, 64);
         });
     });
 });
@@ -728,15 +730,15 @@ describe('s3cmd info', () => {
 
     // test that POLICY and CORS are returned as 'none'
     it('should find that policy has a value of none', done => {
-        checkRawOutput(['info', `s3://${bucket}`], 'policy', 'none',
+        checkRawOutput(['info', `s3://${bucket}`], 'Policy', 'none',
         'stdout', foundIt => {
             assert(foundIt);
             done();
         });
     });
 
     it('should find that cors has a value of none', done => {
-        checkRawOutput(['info', `s3://${bucket}`], 'cors', 'none',
+        checkRawOutput(['info', `s3://${bucket}`], 'CORS', 'none',
         'stdout', foundIt => {
             assert(foundIt);
             done();
@@ -762,7 +764,7 @@ describe('s3cmd info', () => {
         });
 
         it('should find that cors has a value', done => {
-            checkRawOutput(['info', `s3://${bucket}`], 'cors', corsConfig,
+            checkRawOutput(['info', `s3://${bucket}`], 'CORS', corsConfig,
             'stdout', foundIt => {
                 assert(foundIt, 'Did not find value for cors');
                 done();