Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

deps(cw): regen clients with codegen at 0.25.x #6439

Merged
merged 36 commits into from
Feb 14, 2025
Merged
Changes from 1 commit
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
b6a853e
deps: add in generated client
Hweinstock Jan 27, 2025
3c37c57
fix: add missing type annotation
Hweinstock Jan 27, 2025
a3c9adf
fix: pin client versions used by toolkit
Hweinstock Jan 29, 2025
7450af0
deps: regen other client to fix to same version
Hweinstock Jan 29, 2025
7a33dcc
deps: fix other @aws-sdk/* packages
Hweinstock Jan 29, 2025
b1470ed
deps: update package-lock
Hweinstock Jan 29, 2025
83a18cc
deps: avoid fixing to specific version
Hweinstock Jan 29, 2025
b3e02c9
deps: pin smithy major versions
Hweinstock Jan 29, 2025
046c3fb
deps: add back missing module
Hweinstock Jan 29, 2025
8b44fdd
refactor: gross workarounds for it to build
Hweinstock Jan 29, 2025
548fb51
refactor: implement custom credentials provider
Hweinstock Jan 30, 2025
96c7759
refactor: rename profile field to match expected
Hweinstock Jan 30, 2025
a4a2709
docs: add documentation link in implementation
Hweinstock Jan 30, 2025
49195c3
deps: remove deprecated dependencies
Hweinstock Jan 30, 2025
d2f3c4f
refactor: add stronger typing
Hweinstock Jan 30, 2025
d7bede6
docs: remove documentation link since its already in the types
Hweinstock Jan 31, 2025
d7a9ca3
test: implement tests for behavior of credentials provider
Hweinstock Jan 31, 2025
cee9f55
merge: resolve package.json conflicts
Hweinstock Jan 31, 2025
d180291
Merge branch 'master' into auth/loadedConfigFile
Hweinstock Jan 31, 2025
4cd53bd
merge: undo merge
Hweinstock Jan 31, 2025
cea3f93
deps: consume single credential providers package
Hweinstock Jan 31, 2025
d2c81f3
Merge branch 'master' into regenClient/0.25.x
Hweinstock Jan 31, 2025
2450ed4
deps: consume only cred providers needed
Hweinstock Jan 31, 2025
1c8f322
deps: avoid relying on deprecated package
Hweinstock Jan 31, 2025
c358a2a
refactor: avoid unnecessary type annotation
Hweinstock Feb 3, 2025
c5aa07f
deps: remove dependence on ini and add env
Hweinstock Feb 3, 2025
bf3af8f
merge: resolve package-lock conflicts
Hweinstock Feb 10, 2025
20552b8
refactor: simplify conditional args
Hweinstock Feb 11, 2025
21b8b31
docs: add section with links for shared credentials file
Hweinstock Feb 11, 2025
4befa8c
fix: avoid importing from index.ts
Hweinstock Feb 11, 2025
c3ac3d8
Merge branch 'master' into auth/loadedConfigFile
Hweinstock Feb 11, 2025
7a58a30
Merge branch 'master' into auth/loadedConfigFile
Hweinstock Feb 11, 2025
0242b59
merge: fix conflicts with upstream
Hweinstock Feb 11, 2025
a54d8c0
merge: bring in auth changes
Hweinstock Feb 11, 2025
4a847f5
merge: resolve conflicts
Hweinstock Feb 13, 2025
3039bdb
refactor: undo unnecessary import change
Hweinstock Feb 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
merge: undo merge
Hweinstock committed Jan 31, 2025
commit 4cd53bdad1d893172dbe35eb5e59412c6fb65012
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"type": "Bug Fix",
"description": "Amazon Q: Fix code upload error when using /dev or /doc on Remote SSH"
}
14 changes: 6 additions & 8 deletions packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts
Original file line number Diff line number Diff line change
@@ -11,13 +11,11 @@ import {
maxRepoSizeBytes,
} from 'aws-core-vscode/amazonqFeatureDev'
import { assertTelemetry, getWorkspaceFolder, TestFolder } from 'aws-core-vscode/test'
import { fs, AmazonqCreateUpload } from 'aws-core-vscode/shared'
import { fs, AmazonqCreateUpload, ZipStream } from 'aws-core-vscode/shared'
import { MetricName, Span } from 'aws-core-vscode/telemetry'
import sinon from 'sinon'
import { CodeWhispererSettings } from 'aws-core-vscode/codewhisperer'

import AdmZip from 'adm-zip'

const testDevfilePrepareRepo = async (devfileEnabled: boolean) => {
const files: Record<string, string> = {
'file.md': 'test content',
@@ -38,8 +36,8 @@ const testDevfilePrepareRepo = async (devfileEnabled: boolean) => {
}

const expectedFiles = !devfileEnabled
? ['./file.md', './.gitignore']
: ['./devfile.yaml', './file.md', './.gitignore', './abc.jar', 'data/logo.ico']
? ['file.md', '.gitignore']
: ['devfile.yaml', 'file.md', '.gitignore', 'abc.jar', 'data/logo.ico']

const workspace = getWorkspaceFolder(folder.path)
sinon
@@ -71,8 +69,8 @@ const testPrepareRepoData = async (
}

// Unzip the buffer and compare the entry names
const zip = new AdmZip(result.zipFileBuffer)
const actualZipEntries = zip.getEntries().map((entry) => entry.entryName)
const zipEntries = await ZipStream.unzip(result.zipFileBuffer)
const actualZipEntries = zipEntries.map((entry) => entry.filename)
actualZipEntries.sort((a, b) => a.localeCompare(b))
assert.deepStrictEqual(actualZipEntries, expectedFiles)
}
@@ -89,7 +87,7 @@ describe('file utils', () => {
await folder.write('file2.md', 'test content')
const workspace = getWorkspaceFolder(folder.path)

await testPrepareRepoData(workspace, ['./file1.md', './file2.md'])
await testPrepareRepoData(workspace, ['file1.md', 'file2.md'])
})

it('prepareRepoData ignores denied file extensions', async function () {
19 changes: 9 additions & 10 deletions packages/core/src/amazonqFeatureDev/util/files.ts
Original file line number Diff line number Diff line change
@@ -7,11 +7,9 @@ import * as vscode from 'vscode'
import * as path from 'path'
import { collectFiles } from '../../shared/utilities/workspaceUtils'

import AdmZip from 'adm-zip'
import { ContentLengthError, PrepareRepoFailedError } from '../errors'
import { getLogger } from '../../shared/logger/logger'
import { maxFileSizeBytes } from '../limits'
import { createHash } from 'crypto'
import { CurrentWsFolders } from '../types'
import { hasCode, ToolkitError } from '../../shared/errors'
import { AmazonqCreateUpload, Span, telemetry as amznTelemetry } from '../../shared/telemetry/telemetry'
@@ -20,8 +18,7 @@ import { maxRepoSizeBytes } from '../constants'
import { isCodeFile } from '../../shared/filetypes'
import { fs } from '../../shared'
import { CodeWhispererSettings } from '../../codewhisperer'

const getSha256 = (file: Buffer) => createHash('sha256').update(file).digest('base64')
import { ZipStream } from '../../shared/utilities/zipStream'

export async function checkForDevFile(root: string) {
const devFilePath = root + '/devfile.yaml'
@@ -37,7 +34,7 @@ export async function prepareRepoData(
workspaceFolders: CurrentWsFolders,
telemetry: TelemetryHelper,
span: Span<AmazonqCreateUpload>,
zip: AdmZip = new AdmZip()
zip: ZipStream = new ZipStream()
) {
try {
const autoBuildSetting = CodeWhispererSettings.instance.getAutoBuildSetting()
@@ -77,11 +74,12 @@ export async function prepareRepoData(
continue
}
totalBytes += fileSize

const zipFolderPath = path.dirname(file.zipFilePath)
// Paths in zip should be POSIX compliant regardless of OS
// Reference: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
const posixPath = file.zipFilePath.split(path.sep).join(path.posix.sep)

try {
zip.addLocalFile(file.fileUri.fsPath, zipFolderPath)
zip.writeFile(file.fileUri.fsPath, posixPath)
} catch (error) {
if (error instanceof Error && error.message.includes('File not found')) {
// No-op: Skip if file was deleted or does not exist
@@ -111,11 +109,12 @@ export async function prepareRepoData(

telemetry.setRepositorySize(totalBytes)
span.record({ amazonqRepositorySize: totalBytes })
const zipResult = await zip.finalize()

const zipFileBuffer = zip.toBuffer()
const zipFileBuffer = zipResult.streamBuffer.getContents() || Buffer.from('')
return {
zipFileBuffer,
zipFileChecksum: getSha256(zipFileBuffer),
zipFileChecksum: zipResult.hash,
}
} catch (error) {
getLogger().debug(`featureDev: Failed to prepare repo: ${error}`)
1 change: 1 addition & 0 deletions packages/core/src/shared/index.ts
Original file line number Diff line number Diff line change
@@ -48,6 +48,7 @@ export * from './localizedText'
export * as env from './vscode/env'
export * from './vscode/commands2'
export * from './utilities/pathUtils'
export * from './utilities/zipStream'
export * from './errors'
export * as messages from './utilities/messages'
export * as errors from './errors'
15 changes: 14 additions & 1 deletion packages/core/src/shared/utilities/zipStream.ts
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@ import { WritableStreamBuffer } from 'stream-buffers'
import crypto from 'crypto'
import { readFileAsString } from '../filesystemUtilities'
// Use require instead of import since this package doesn't support commonjs
const { ZipWriter, TextReader } = require('@zip.js/zip.js')
const { ZipWriter, TextReader, ZipReader, Uint8ArrayReader } = require('@zip.js/zip.js')
import { getLogger } from '../logger/logger'

export interface ZipStreamResult {
@@ -15,6 +15,10 @@ export interface ZipStreamResult {
streamBuffer: WritableStreamBuffer
}

export type ZipReaderResult = {
filename: string
}

export type ZipStreamProps = {
hashAlgorithm: 'md5' | 'sha256'
maxNumberOfFileStreams: number
@@ -150,4 +154,13 @@ export class ZipStream {
streamBuffer: this._streamBuffer,
}
}

public static async unzip(zipBuffer: Buffer): Promise<ZipReaderResult[]> {
const reader = new ZipReader(new Uint8ArrayReader(new Uint8Array(zipBuffer)))
try {
return await reader.getEntries()
} finally {
await reader.close()
}
}
}
11 changes: 11 additions & 0 deletions packages/core/src/test/shared/utilities/zipStream.test.ts
Original file line number Diff line number Diff line change
@@ -60,4 +60,15 @@ describe('zipStream', function () {
assert.strictEqual(result.hash, expectedMd5)
assert.strictEqual(result.sizeInBytes, (await fs.stat(zipPath)).size)
})

it('should unzip from a buffer', async function () {
const zipStream = new ZipStream()
await zipStream.writeString('foo bar', 'file.txt')
const result = await zipStream.finalize()

const zipBuffer = result.streamBuffer.getContents()
assert.ok(zipBuffer)
const zipEntries = await ZipStream.unzip(zipBuffer)
assert.strictEqual(zipEntries[0].filename, 'file.txt')
})
})
5 changes: 2 additions & 3 deletions packages/core/src/testInteg/perf/prepareRepoData.test.ts
Original file line number Diff line number Diff line change
@@ -30,7 +30,7 @@ function performanceTestWrapper(numFiles: number, fileSize: number) {
getEqualOSTestOptions({
userCpuUsage: 200,
systemCpuUsage: 35,
heapTotal: 4,
heapTotal: 20,
}),
`handles ${numFiles} files of size ${fileSize} bytes`,
function () {
@@ -63,8 +63,7 @@ function verifyResult(setup: setupResult, result: resultType, telemetry: Telemet
assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true)
assert.strictEqual(telemetry.repositorySize, expectedSize)
assert.strictEqual(result.zipFileChecksum.length, 44)

assert.ok(getFsCallsUpperBound(setup.fsSpy) <= setup.numFiles * 4, 'total system calls should be under 4 per file')
assert.ok(getFsCallsUpperBound(setup.fsSpy) <= setup.numFiles * 8, 'total system calls should be under 8 per file')
}

describe('prepareRepoData', function () {