From 31d1734d5c66b4794a6e9d4074dad46810cbd992 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 22 Jun 2023 13:52:14 +0200 Subject: [PATCH] CLDSRV-409: use latest s3cmd with python3 --- .github/workflows/tests.yaml | 7 ++---- tests/functional/s3cmd/tests.js | 42 +++++++++++++++++---------------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9387ad0a7f..532796b3a4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -222,13 +222,10 @@ jobs: 3.9 - name: Setup CI environment uses: ./.github/actions/setup-ci - - name: Setup python2 test environment + - name: Setup python test environment run: | sudo apt-get install -y libdigest-hmac-perl - pip install virtualenv==20.21.0 - virtualenv -p $(which python2) ~/.virtualenv/py2 - source ~/.virtualenv/py2/bin/activate - pip install 's3cmd==1.6.1' + pip install 's3cmd==2.3.0' - name: Setup CI services run: docker-compose up -d working-directory: .github/docker diff --git a/tests/functional/s3cmd/tests.js b/tests/functional/s3cmd/tests.js index a0c09feade..a1bff602bc 100644 --- a/tests/functional/s3cmd/tests.js +++ b/tests/functional/s3cmd/tests.js @@ -165,7 +165,9 @@ function readJsonFromChild(child, lineFinder, cb) { const findBrace = data.indexOf('{', findLine); const findEnd = findEndJson(data, findBrace); const endJson = data.substring(findBrace, findEnd + 1) - .replace(/"/g, '\\"').replace(/'/g, '"'); + .replace(/"/g, '\\"').replace(/'/g, '"') + .replace(/b'/g, '\'') + .replace(/b"/g, '"'); return cb(JSON.parse(endJson)); }); } @@ -262,7 +264,7 @@ describe('s3cmd putBucket', () => { exec([ 'mb', `s3://${bucket}`, '--bucket-location=scality-us-west-1', - ], done, 13); + ], done, 11); }); it('put an invalid bucket, should fail', done => { @@ -344,18 +346,18 @@ describe('s3cmd getService', () => { it("should have response headers matching AWS's response headers", done => { - provideLineOfInterest(['ls', '--debug'], 'DEBUG: Response: {', + provideLineOfInterest(['ls', '--debug'], '\'headers\': {', parsedObject => { - assert(parsedObject.headers['x-amz-id-2']); - assert(parsedObject.headers['transfer-encoding']); - assert(parsedObject.headers['x-amz-request-id']); - const gmtDate = new Date(parsedObject.headers.date) + assert(parsedObject['x-amz-id-2']); + assert(parsedObject['transfer-encoding']); + assert(parsedObject['x-amz-request-id']); + const gmtDate = new Date(parsedObject.date) .toUTCString(); - assert.strictEqual(parsedObject.headers.date, gmtDate); + assert.strictEqual(parsedObject.date, gmtDate); assert.strictEqual(parsedObject - .headers['content-type'], 'application/xml'); + ['content-type'], 'application/xml'); assert.strictEqual(parsedObject - .headers['set-cookie'], undefined); + ['set-cookie'], undefined); done(); }); }); @@ -395,11 +397,11 @@ describe('s3cmd getObject', function toto() { }); it('get non existing file in existing bucket, should fail', done => { - exec(['get', `s3://${bucket}/${nonexist}`, 'fail'], done, 12); + exec(['get', `s3://${bucket}/${nonexist}`, 'fail'], done, 64); }); it('get file in non existing bucket, should fail', done => { - exec(['get', `s3://${nonexist}/${nonexist}`, 'fail2'], done, 12); + exec(['get', `s3://${nonexist}/${nonexist}`, 'fail2'], done, 64); }); }); @@ -511,7 +513,7 @@ describe('s3cmd delObject', () => { it('delete an already deleted object, should return a 204', done => { provideLineOfInterest(['rm', `s3://${bucket}/${upload}`, '--debug'], - 'DEBUG: Response: {', parsedObject => { + 'DEBUG: Response:\n{', parsedObject => { assert.strictEqual(parsedObject.status, 204); done(); }); @@ -519,14 +521,14 @@ describe('s3cmd delObject', () => { it('delete non-existing object, should return a 204', done => { provideLineOfInterest(['rm', `s3://${bucket}/${nonexist}`, '--debug'], - 'DEBUG: Response: {', parsedObject => { + 'DEBUG: Response:\n{', parsedObject => { assert.strictEqual(parsedObject.status, 204); done(); }); }); it('try to get the deleted object, should fail', done => { - exec(['get', `s3://${bucket}/${upload}`, download], done, 12); + exec(['get', `s3://${bucket}/${upload}`, download], done, 64); }); }); @@ -621,7 +623,7 @@ describe('s3cmd multipart upload', function titi() { }); it('should not be able to get deleted object', done => { - exec(['get', `s3://${bucket}/${MPUpload}`, download], done, 12); + exec(['get', `s3://${bucket}/${MPUpload}`, download], done, 64); }); }); @@ -660,7 +662,7 @@ MPUploadSplitter.forEach(file => { }); it('should not be able to get deleted object', done => { - exec(['get', `s3://${bucket}/${file}`, download], done, 12); + exec(['get', `s3://${bucket}/${file}`, download], done, 64); }); }); }); @@ -728,7 +730,7 @@ describe('s3cmd info', () => { // test that POLICY and CORS are returned as 'none' it('should find that policy has a value of none', done => { - checkRawOutput(['info', `s3://${bucket}`], 'policy', 'none', + checkRawOutput(['info', `s3://${bucket}`], 'Policy', 'none', 'stdout', foundIt => { assert(foundIt); done(); @@ -736,7 +738,7 @@ describe('s3cmd info', () => { }); it('should find that cors has a value of none', done => { - checkRawOutput(['info', `s3://${bucket}`], 'cors', 'none', + checkRawOutput(['info', `s3://${bucket}`], 'CORS', 'none', 'stdout', foundIt => { assert(foundIt); done(); @@ -762,7 +764,7 @@ describe('s3cmd info', () => { }); it('should find that cors has a value', done => { - checkRawOutput(['info', `s3://${bucket}`], 'cors', corsConfig, + checkRawOutput(['info', `s3://${bucket}`], 'CORS', corsConfig, 'stdout', foundIt => { assert(foundIt, 'Did not find value for cors'); done();