From 0b8959c58e4753390640e8330a84505ea6d040ef Mon Sep 17 00:00:00 2001 From: Trae Yelovich Date: Wed, 18 Oct 2023 16:18:29 -0400 Subject: [PATCH] feat(uss/fs): copy/paste, write contents to LPAR, (wip) conflict management Signed-off-by: Trae Yelovich --- .../src/profiles/ZoweExplorerApi.ts | 7 +- .../src/profiles/ZoweExplorerZosmfApi.ts | 10 +- .../zowe-explorer/src/uss/FileStructure.ts | 4 +- .../zowe-explorer/src/uss/UssFSProvider.ts | 126 ++++++++++++++++-- packages/zowe-explorer/src/uss/ZoweUSSNode.ts | 70 ++-------- packages/zowe-explorer/src/uss/actions.ts | 9 +- 6 files changed, 146 insertions(+), 80 deletions(-) diff --git a/packages/zowe-explorer-api/src/profiles/ZoweExplorerApi.ts b/packages/zowe-explorer-api/src/profiles/ZoweExplorerApi.ts index 26cbbc4094..b1bb10026e 100644 --- a/packages/zowe-explorer-api/src/profiles/ZoweExplorerApi.ts +++ b/packages/zowe-explorer-api/src/profiles/ZoweExplorerApi.ts @@ -124,7 +124,7 @@ export namespace ZoweExplorerApi { * @param {string} ussFilePath * @param {zowe.IDownloadOptions} options */ - getContents(ussFilePath: string, options: zowe.IDownloadOptions): Promise; + getContents(ussFilePath: string, options: zowe.IDownloadSingleOptions): Promise; /** * Uploads the file at the given path. Use for Save. @@ -149,6 +149,11 @@ export namespace ZoweExplorerApi { returnEtag?: boolean ): Promise; + /** + * Uploads a given buffer as the contents of a file on USS. + */ + uploadBufferAsFile?(buffer: Buffer, filePath: string, options?: zowe.IUploadOptions): Promise; + /** * Uploads the file at the given path. Use for Save. * diff --git a/packages/zowe-explorer-api/src/profiles/ZoweExplorerZosmfApi.ts b/packages/zowe-explorer-api/src/profiles/ZoweExplorerZosmfApi.ts index 9ab385b4d3..476d1486a0 100644 --- a/packages/zowe-explorer-api/src/profiles/ZoweExplorerZosmfApi.ts +++ b/packages/zowe-explorer-api/src/profiles/ZoweExplorerZosmfApi.ts @@ -113,14 +113,20 @@ export class ZosmfUssApi extends ZosmfApiCommon implements ZoweExplorerApi.IUss return zowe.Utilities.isFileTagBinOrAscii(this.getSession(), ussFilePath); } - public getContents(inputFilePath: string, options: zowe.IGetOptions | zowe.IDownloadOptions): Promise { - return zowe.Get.USSFile(this.getSession(), inputFilePath, options); + public async getContents(inputFilePath: string, options: zowe.IDownloadSingleOptions): Promise { + return zowe.Download.ussFile(this.getSession(), inputFilePath, { + ...options, + }); } public copy(outputPath: string, options?: Omit): Promise { return zowe.Utilities.putUSSPayload(this.getSession(), outputPath, { ...(options ?? {}), request: "copy" }); } + public uploadBufferAsFile(buffer: Buffer, filePath: string, options?: zowe.IUploadOptions): Promise { + return zowe.Upload.bufferToUssFile(this.getSession(), filePath, buffer, options); + } + /** * API method to wrap to the newer `putContent`. * @deprecated diff --git a/packages/zowe-explorer/src/uss/FileStructure.ts b/packages/zowe-explorer/src/uss/FileStructure.ts index 92e3e6a46e..81d3425e5b 100644 --- a/packages/zowe-explorer/src/uss/FileStructure.ts +++ b/packages/zowe-explorer/src/uss/FileStructure.ts @@ -10,6 +10,7 @@ */ import { ZoweLogger } from "../utils/LoggerUtils"; +import * as vscode from "vscode"; /** * File types within the USS tree structure @@ -20,8 +21,7 @@ export enum UssFileType { } export interface UssFileTree { - // The path of the file on the local file system, if it exists - localPath?: string; + localUri?: vscode.Uri; // The path of the file/directory as defined in USS ussPath: string; diff --git a/packages/zowe-explorer/src/uss/UssFSProvider.ts b/packages/zowe-explorer/src/uss/UssFSProvider.ts index 79927639d3..3ad865773b 100644 --- a/packages/zowe-explorer/src/uss/UssFSProvider.ts +++ b/packages/zowe-explorer/src/uss/UssFSProvider.ts @@ -15,6 +15,8 @@ import * as path from "path"; import * as vscode from "vscode"; import { Profiles } from "../Profiles"; import { ZoweExplorerApiRegister } from "../ZoweExplorerApiRegister"; +import { UssFileTree, UssFileType } from "./FileStructure"; +import { Duplex } from "stream"; export type FileEntryMetadata = { profile: imperative.IProfileLoaded; @@ -25,18 +27,21 @@ export interface UssEntry { name: string; metadata: FileEntryMetadata; type: vscode.FileType; + wasAccessed: boolean; } export class UssFile implements UssEntry, vscode.FileStat { public name: string; public metadata: FileEntryMetadata; public type: vscode.FileType; + public wasAccessed: boolean; public ctime: number; public mtime: number; public size: number; public binary: boolean; public data?: Uint8Array; + public etag?: string; public attributes: FileAttributes; public constructor(name: string) { @@ -46,6 +51,7 @@ export class UssFile implements UssEntry, vscode.FileStat { this.size = 0; this.name = name; this.binary = false; + this.wasAccessed = false; } } @@ -53,12 +59,12 @@ export class UssDirectory implements UssEntry, vscode.FileStat { public name: string; public metadata: FileEntryMetadata; public type: vscode.FileType; + public wasAccessed: boolean; public ctime: number; public mtime: number; public size: number; public entries: Map; - public wasAccessed: boolean; public constructor(name: string) { this.type = vscode.FileType.Directory; @@ -196,24 +202,47 @@ export class UssFSProvider implements vscode.FileSystemProvider { throw vscode.FileSystemError.FileNotFound("Session does not exist for this file."); } - if (!file.data) { + if (!file.wasAccessed) { + // we need to fetch the contents from the mainframe since the file hasn't been accessed yet + + class BufferBuilder extends Duplex { + private chunks: Uint8Array[]; + + public constructor() { + super(); + this.chunks = []; + } + + public _write(chunk: any, encoding: BufferEncoding, callback: (error?: Error) => void): void { + this.chunks.push(chunk); + callback(); + } + + public _read(size: number): void { + const concatBuf = Buffer.concat(this.chunks); + this.push(concatBuf); + this.push(null); + } + } + + const bufBuilder = new BufferBuilder(); const filePath = uri.path.substring(startPathPos); const resp = await ZoweExplorerApiRegister.getUssApi(loadedProfile).getContents(filePath, { returnEtag: true, encoding: loadedProfile.profile?.encoding, responseTimeout: loadedProfile.profile?.responseTimeout, + stream: bufBuilder, }); - if (!(resp instanceof Buffer)) { - throw vscode.FileSystemError.FileNotFound(); - } - file.data = resp; + file.data = bufBuilder.read(); + file.etag = resp.apiResponse.etag; + file.wasAccessed = true; } return file.data; } - public writeFile(uri: vscode.Uri, content: Uint8Array, options: { create: boolean; overwrite: boolean }): void { + public async writeFile(uri: vscode.Uri, content: Uint8Array, options: { create: boolean; overwrite: boolean }): Promise { /* 1. Parse URI to get file path for API endpoint 2. Make API call after assigning data to File object @@ -239,11 +268,31 @@ export class UssFSProvider implements vscode.FileSystemProvider { : this._getInfoFromUri(uri); entry = new UssFile(basename); entry.metadata = profInfo; + entry.data = content; parent.entries.set(basename, entry); this._fireSoon({ type: vscode.FileChangeType.Created, uri }); } else { - // TODO: we might want to use a callback or promise that returns data rather than saving the full contents in memory - entry.data = content.length > 0 ? content : null; + if (entry.wasAccessed) { + // entry was already accessed, this is an update to the existing file + // eslint-disable-next-line no-useless-catch + try { + await ZoweExplorerApiRegister.getUssApi(parent.metadata.profile).uploadBufferAsFile( + Buffer.from(content), + entry.metadata.ussPath, + { etag: entry.etag } + ); + } catch (err) { + // TODO: conflict management + // if (err.message.includes("Rest API failure with HTTP(S) status 412")) { + // Gui.errorMessage("There is a newer version of this file on the mainframe. Compare with remote contents or overwrite?"); + // } + throw err; + } + entry.data = content; + } else { + entry.data = content; + } + // if the entry hasn't been accessed yet, we don't need to call the API since we are just creating the file } entry.mtime = Date.now(); entry.size = content.byteLength; @@ -331,6 +380,65 @@ export class UssFSProvider implements vscode.FileSystemProvider { this._fireSoon({ type: vscode.FileChangeType.Changed, uri: dirname }, { uri, type: vscode.FileChangeType.Deleted }); } + public async copyEx( + source: vscode.Uri, + destination: vscode.Uri, + options: { readonly overwrite: boolean; readonly tree: UssFileTree } + ): Promise { + const destInfo = this._getInfoFromUri(destination); + const sourceInfo = this._getInfoFromUri(source); + const api = ZoweExplorerApiRegister.getUssApi(destInfo.profile); + + const hasCopyApi = api.copy != null; + + const apiResponse = await api.fileList(destInfo.ussPath); + const fileList = apiResponse.apiResponse?.items; + + // Check root path for conflicts before pasting nodes in this path + let fileName = path.basename(sourceInfo.ussPath); + if (fileList?.find((file) => file.name === fileName) != null) { + // If file names match, build the copy suffix + let dupCount = 1; + const extension = path.extname(fileName); + const baseNameForFile = path.parse(fileName)?.name; + let dupName = `${baseNameForFile} (${dupCount})${extension}`; + while (fileList.find((file) => file.name === dupName) != null) { + dupCount++; + dupName = `${baseNameForFile} (${dupCount})${extension}`; + } + fileName = dupName; + } + const outputPath = `${destInfo.ussPath}/${fileName}`; + + if (hasCopyApi && sourceInfo.profile.profile === destInfo.profile.profile) { + await api.copy(outputPath, { + from: sourceInfo.ussPath, + recursive: options.tree.type === UssFileType.Directory, + overwrite: options.overwrite ?? true, + }); + } else if (options.tree.type === UssFileType.Directory) { + // Not all APIs respect the recursive option, so it's best to + // recurse within this operation to avoid missing files/folders + await api.create(outputPath, "directory"); + if (options.tree.children) { + for (const child of options.tree.children) { + await this.copyEx(child.localUri, vscode.Uri.parse(`uss:${outputPath}`), { ...options, tree: child }); + } + } + } else { + const fileEntry = this._lookup(source, true); + if (fileEntry == null) { + return; + } + + if (!fileEntry.wasAccessed) { + // must fetch contents of file first before pasting in new path + const fileContents = await vscode.workspace.fs.readFile(source); + await api.uploadBufferAsFile(Buffer.from(fileContents), outputPath); + } + } + } + public createDirectory(uri: vscode.Uri): void { /* 1. Parse URI to get desired directory path diff --git a/packages/zowe-explorer/src/uss/ZoweUSSNode.ts b/packages/zowe-explorer/src/uss/ZoweUSSNode.ts index 172a887bf0..bd6b3aeaf4 100644 --- a/packages/zowe-explorer/src/uss/ZoweUSSNode.ts +++ b/packages/zowe-explorer/src/uss/ZoweUSSNode.ts @@ -25,7 +25,7 @@ import { closeOpenedTextFile } from "../utils/workspace"; import * as nls from "vscode-nls"; import { UssFileTree, UssFileType, UssFileUtils } from "./FileStructure"; import { ZoweLogger } from "../utils/LoggerUtils"; -import { UssFile } from "./UssFSProvider"; +import { UssFile, UssFSProvider } from "./UssFSProvider"; import { USSTree } from "./USSTree"; // Set up localization @@ -670,71 +670,19 @@ export class ZoweUSSNode extends ZoweTreeNode implements IZoweUSSTreeNode { */ public async paste( sessionName: string, - rootPath: string, + destUri: vscode.Uri, uss: { tree: UssFileTree; api: ZoweExplorerApi.IUss; options?: IUploadOptions } ): Promise { ZoweLogger.trace("ZoweUSSNode.paste called."); const hasCopyApi = uss.api.copy != null; - const hasPutContentApi = uss.api.putContent != null; - if (!uss.api.fileList || (!hasCopyApi && !hasPutContentApi)) { - throw new Error(localize("paste.missingApis", "Required API functions for pasting (fileList, copy and/or putContent) were not found.")); - } - - const apiResponse = await uss.api.fileList(rootPath); - const fileList = apiResponse.apiResponse?.items; - - // Check root path for conflicts before pasting nodes in this path - let fileName = uss.tree.baseName; - if (fileList?.find((file) => file.name === fileName) != null) { - // If file names match, build the copy suffix - let dupCount = 1; - const extension = path.extname(uss.tree.baseName); - const baseNameForFile = path.parse(uss.tree.baseName)?.name; - let dupName = `${baseNameForFile} (${dupCount})${extension}`; - while (fileList.find((file) => file.name === dupName) != null) { - dupCount++; - dupName = `${baseNameForFile} (${dupCount})${extension}`; - } - fileName = dupName; + if (!uss.api.fileList || !hasCopyApi) { + throw new Error(localize("paste.missingApis", "Required API functions for pasting (fileList and copy) were not found.")); } - const outputPath = `${rootPath}/${fileName}`; - if (hasCopyApi && UssFileUtils.toSameSession(uss.tree, sessionName)) { - await uss.api.copy(outputPath, { - from: uss.tree.ussPath, - recursive: uss.tree.type === UssFileType.Directory, - }); - } else { - const existsLocally = fs.existsSync(uss.tree.localPath); - switch (uss.tree.type) { - case UssFileType.Directory: - if (!existsLocally) { - // We will need to build the file structure locally, to pull files down if needed - fs.mkdirSync(uss.tree.localPath, { recursive: true }); - } - // Not all APIs respect the recursive option, so it's best to - // recurse within this operation to avoid missing files/folders - await uss.api.create(outputPath, "directory"); - if (uss.tree.children) { - for (const child of uss.tree.children) { - await this.paste(sessionName, outputPath, { api: uss.api, tree: child, options: uss.options }); - } - } - break; - case UssFileType.File: - if (!existsLocally) { - await uss.api.getContents(uss.tree.ussPath, { - file: uss.tree.localPath, - binary: uss.tree.binary, - returnEtag: true, - encoding: this.profile.profile?.encoding, - responseTimeout: this.profile.profile?.responseTimeout, - }); - } - await uss.api.putContent(uss.tree.localPath, outputPath, uss.options); - break; - } - } + await UssFSProvider.instance.copyEx(uss.tree.localUri, destUri, { + overwrite: true, + tree: uss.tree, + }); } /** @@ -766,7 +714,7 @@ export class ZoweUSSNode extends ZoweTreeNode implements IZoweUSSTreeNode { }; for (const subnode of fileTreeToPaste.children) { - await this.paste(sessionName, remotePath, { api, tree: subnode, options }); + await this.paste(sessionName, vscode.Uri.parse(`uss:/${this.profile.name}${this.fullPath}`), { api, tree: subnode, options }); } } catch (error) { await errorHandling(error, this.label.toString(), localize("copyUssFile.error", "Error uploading files")); diff --git a/packages/zowe-explorer/src/uss/actions.ts b/packages/zowe-explorer/src/uss/actions.ts index a34fb0f60d..8585d9d215 100644 --- a/packages/zowe-explorer/src/uss/actions.ts +++ b/packages/zowe-explorer/src/uss/actions.ts @@ -357,7 +357,7 @@ export async function buildFileStructure(node: IZoweUSSTreeNode): Promise, nod title: localize("ZoweUssNode.copyUpload.progress", "Pasting files..."), }, async () => { - await (selectedNode.pasteUssTree ? selectedNode.pasteUssTree() : selectedNode.copyUssFile()); + await selectedNode.pasteUssTree(); } ); - const nodeToRefresh = node?.contextValue != null && contextually.isUssSession(node) ? selectedNode : selectedNode.getParent(); - ussFileProvider.refreshElement(nodeToRefresh); + ussFileProvider.refreshElement(selectedNode); }