Merge remote-tracking branch 'origin/next' into remove-unnecessary-ex…

…ports

Signed-off-by: Andrew W. Harn <[email protected]>

packages/cli/CHANGELOG.md

@@ -97,7 +97,6 @@ LTS Breaking: Removed the following previously deprecated items: [#1981](https:/
   - Removed zosmfProfile from `ZosFilesBase.handler.ts`
   - Removed statCmdFlag as an export from Shell.ts
 
-
 ## `8.0.0-next.202401262128`
 
 - Enhancement: Adding `--binary` and `--encoding` options to `zosfiles edit`
@@ -132,6 +131,10 @@ LTS Breaking: Removed the following previously deprecated items: [#1981](https:/
 
 - Major: First major version bump for V3
 
+## `7.25.1`
+
+- BugFix: Updated `braces` dependency for technical currency. [#2157](https://github.com/zowe/zowe-cli/pull/2157)
+
 ## `7.25.0`
 
 - Enhancement: Added the ability to set JCL reader properties for `--jobRecordLength`, `--jobRecordFormat` and `--jobEncoding` on the `zowe jobs submit local-file` and `zowe jobs submit stdin` commands. [#2139](https://github.com/zowe/zowe-cli/pull/2139)

packages/core/CHANGELOG.md

@@ -10,7 +10,6 @@ All notable changes to the Zowe core SDK package will be documented in this file
 
 - Enhancement: Modified Services.ts to have commented properties end with commas. [#1049](https://github.com/zowe/zowe-cli/issues/1049)
 
-
 ## `8.0.0-next.202403041352`
 
 - BugFix: Updated engine to Node 18.12.0. [#2074](https://github.com/zowe/zowe-cli/pull/2074)
@@ -40,6 +39,10 @@ All notable changes to the Zowe core SDK package will be documented in this file
 
 - Major: First major version bump for V3
 
+## `7.25.2`
+
+- Enhancement: Modified Services.ts to have commented properties end with commas. [#1049](https://github.com/zowe/zowe-cli/issues/1049)
+
 ## `7.25.0`
 
 - Enhancement: Added `X_IBM_INTRDR_FILE_ENCODING` header to `ZosmfHeaders` [#2139](https://github.com/zowe/zowe-cli/pull/2139)
@@ -59,7 +62,7 @@ All notable changes to the Zowe core SDK package will be documented in this file
 ## `7.17.0`
 
 - Enhancement: Set properties for z/OSMF REST errors for use in a more user-friendly format with the ZOWE_V3_ERR_FORMAT environment variable. [zowe-cli#935](https://github.com/zowe/zowe-cli/issues/935)
--
+
 ## `7.16.5`
 
 - BugFix: Fixed confusing error message "Token is not valid or expired" when APIML token is used to connect direct-to-service with `ZosmfRestClient`. [Imperative #978](https://github.com/zowe/imperative/issues/978)

packages/imperative/CHANGELOG.md

@@ -3,6 +3,9 @@
 All notable changes to the Imperative package will be documented in this file.
 
 ## Recent Changes
+- Enhancement: Added `BufferBuilder` utility class to provide convenient way of downloading to a stream that can be read as a buffer. [#2167](https://github.com/zowe/zowe-cli/pull/2167)
+- BugFix: Fixed error in REST client that when using stream could cause small data sets to download with incomplete contents. [#744](https://github.com/zowe/zowe-cli/issues/744)
+- BugFix: Updated `micromatch` dependency for technical currency. [#2167](https://github.com/zowe/zowe-cli/pull/2167)
 - LTS Breaking: Modified the @zowe/imperative SDK [#2083](https://github.com/zowe/zowe-cli/issues/2083)
   - Removed the following exported classes:
     - AbstractAuthHandler
@@ -378,6 +381,20 @@ All notable changes to the Imperative package will be documented in this file.
 
 - Major: First major version bump for V3
 
+## `5.24.0`
+
+- Enhancement: Added `BufferBuilder` utility class to provide convenient way of downloading to a stream that can be read as a buffer. [#2167](https://github.com/zowe/zowe-cli/pull/2167)
+- BugFix: Fixed error in REST client that when using stream could cause small data sets to download with incomplete contents. [#744](https://github.com/zowe/zowe-cli/issues/744)
+- BugFix: Updated `micromatch` dependency for technical currency. [#2167](https://github.com/zowe/zowe-cli/pull/2167)
+
+## `5.23.4`
+
+- BugFix: Updated `braces` dependency for technical currency. [#2157](https://github.com/zowe/zowe-cli/pull/2157)
+
+## `5.23.3`
+
+- BugFix: Modified error text in SyntaxValidator.invalidOptionError. [#2138](https://github.com/zowe/zowe-cli/issues/2138)
+
 ## `5.23.2`
 
 - BugFix: Updated error text for invalid command options so that allowable values are displayed as strings instead of regular expressions when possible. [#1863](https://github.com/zowe/zowe-cli/issues/1863)

packages/imperative/src/rest/__tests__/client/AbstractRestClient.unit.test.ts

@@ -464,7 +464,7 @@ describe("AbstractRestClient tests", () => {
         const fakeResponseStream: any = {
             write: jest.fn(),
             on: jest.fn(),
-            end: jest.fn(),
+            end: jest.fn((cb: any) => cb()),
             writableFinished: true
         };
         const fakeRequestStream: any = {

packages/imperative/src/rest/src/client/AbstractRestClient.ts

@@ -789,10 +789,6 @@ export abstract class AbstractRestClient {
             this.mTask.percentComplete = TaskProgress.ONE_HUNDRED_PERCENT;
             this.mTask.stageName = TaskStage.COMPLETE;
         }
-        if (this.mResponseStream != null) {
-            this.log.debug("Ending response stream");
-            this.mResponseStream.end();
-        }
         if (this.mContentEncoding != null && this.mData.length > 0) {
             this.log.debug("Decompressing encoded response");
             try {
@@ -801,22 +797,25 @@ export abstract class AbstractRestClient {
                 this.mReject(err);
             }
         }
-        if (this.requestFailure) {
-            // Reject the promise with an error
-            const httpStatus = this.response == null ? undefined : this.response.statusCode;
-            this.mReject(this.populateError({
-                msg: "Rest API failure with HTTP(S) status " + httpStatus,
-                causeErrors: this.dataString,
-                source: "http"
-            }));
-        } else if (this.mResponseStream != null && this.mResponseStream.writableEnded) {
-            // This will correct any instances where the finished event does not get emitted
-            // even though the stream processing has ended.
-            this.mResolve(this.dataString);
-        } else if (this.mResponseStream != null && !this.mResponseStream.writableFinished) {
-            this.mResponseStream.on("finish", () => this.mResolve(this.dataString));
+
+        const requestEnd = () => {
+            if (this.requestFailure) {
+                // Reject the promise with an error
+                const httpStatus = this.response == null ? undefined : this.response.statusCode;
+                this.mReject(this.populateError({
+                    msg: "Rest API failure with HTTP(S) status " + httpStatus,
+                    causeErrors: this.dataString,
+                    source: "http"
+                }));
+            } else {
+                this.mResolve(this.dataString);
+            }
+        };
+        if (this.mResponseStream != null) {
+            this.log.debug("Ending response stream");
+            this.mResponseStream.end(requestEnd);
         } else {
-            this.mResolve(this.dataString);
+            requestEnd();
         }
     }
 

packages/imperative/src/utilities/__tests__/BufferBuilder.unit.test.ts

@@ -0,0 +1,30 @@
+/*
+* This program and the accompanying materials are made available under the terms of the
+* Eclipse Public License v2.0 which accompanies this distribution, and is available at
+* https://www.eclipse.org/legal/epl-v20.html
+*
+* SPDX-License-Identifier: EPL-2.0
+*
+* Copyright Contributors to the Zowe Project.
+*
+*/
+
+import { BufferBuilder } from "../src/BufferBuilder";
+
+describe("BufferBuilder", () => {
+    it("calls the given callback on write", () => {
+        const bufBuilder = new BufferBuilder();
+        const callbackMock = jest.fn();
+        bufBuilder._write(new Uint8Array([1, 2, 3]), "binary", callbackMock);
+        expect(callbackMock).toHaveBeenCalled();
+    });
+
+    it("calls 'push' on read", () => {
+        const bufBuilder = new BufferBuilder();
+        const callbackMock = jest.fn();
+        const pushMock = jest.spyOn(bufBuilder, "push");
+        bufBuilder._write(new Uint8Array([1, 2, 3]), "binary", callbackMock);
+        bufBuilder._read(3);
+        expect(pushMock).toHaveBeenCalledTimes(2);
+    });
+});

packages/imperative/src/utilities/index.ts

@@ -11,6 +11,7 @@
 
 export * from "./src/doc/IDaemonRequest";
 export * from "./src/doc/IDaemonResponse";
+export * from "./src/BufferBuilder";
 export * from "./src/DaemonRequest";
 export * from "./src/ExecUtils";
 export * from "./src/ImperativeConfig";

packages/imperative/src/utilities/src/BufferBuilder.ts

@@ -0,0 +1,32 @@
+/*
+* This program and the accompanying materials are made available under the terms of the
+* Eclipse Public License v2.0 which accompanies this distribution, and is available at
+* https://www.eclipse.org/legal/epl-v20.html
+*
+* SPDX-License-Identifier: EPL-2.0
+*
+* Copyright Contributors to the Zowe Project.
+*
+*/
+
+import { Duplex } from "stream";
+
+export class BufferBuilder extends Duplex {
+    private chunks: Uint8Array[];
+
+    public constructor() {
+        super();
+        this.chunks = [];
+    }
+
+    public _write(chunk: any, _encoding: BufferEncoding, callback: (error?: Error) => void): void {
+        this.chunks.push(chunk);
+        callback();
+    }
+
+    public _read(_size: number): void {
+        const concatBuf = Buffer.concat(this.chunks);
+        this.push(concatBuf);
+        this.push(null);
+    }
+}

packages/zosfiles/CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to the Zowe z/OS files SDK package will be documented in thi
   - Deprecated the following interfaces:
     - IOptionsFullResponse - use `IOptionsFullResponse` from `@zowe/imperative`.
     - IRestClientResponse - use `IRestClientResponse` from `@zowe/imperative`.
+- BugFix: Fixed error where `Get.dataSet` and `Get.USSFile` methods could silently fail when downloading large data sets or files. [#2167](https://github.com/zowe/zowe-cli/pull/2167)
 
 ## `8.0.0-next.202405202020`
 
@@ -58,6 +59,14 @@ All notable changes to the Zowe z/OS files SDK package will be documented in thi
 
 - Major: First major version bump for V3
 
+## `7.26.1`
+
+- BugFix: Fixed `Get.dataSet` and `Get.USSFile` methods so that they return an empty buffer instead of null for empty files. [#2173](https://github.com/zowe/zowe-cli/pull/2173)
+
+## `7.26.0`
+
+- BugFix: Fixed error where `Get.dataSet` and `Get.USSFile` methods could silently fail when downloading large data sets or files. [#2167](https://github.com/zowe/zowe-cli/pull/2167)
+
 ## `7.24.0`
 
 - BugFix: Fixed error that could occur when listing data set members that contain control characters in the name. [#2104](https://github.com/zowe/zowe-cli/pull/2104)

packages/zosfiles/__tests__/__unit__/methods/get/Get.unit.test.ts

@@ -109,6 +109,32 @@ describe("z/OS Files - View", () => {
             }));
         });
 
+        it("should get data set content when empty", async () => {
+            let response;
+            let caughtError;
+            zosmfExpectSpy.mockImplementationOnce(async (_session, options) => {
+                options.responseStream?.end();
+                return {};
+            });
+
+            try {
+                response = await Get.dataSet(dummySession, dsname);
+            } catch (e) {
+                caughtError = e;
+            }
+
+            const endpoint = posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, dsname);
+
+            expect(caughtError).toBeUndefined();
+            expect(response).toEqual(Buffer.alloc(0));
+
+            expect(zosmfExpectSpy).toHaveBeenCalledTimes(1);
+            expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, expect.objectContaining({
+                reqHeaders: [ZosmfHeaders.ACCEPT_ENCODING, ZosmfHeaders.TEXT_PLAIN],
+                resource: endpoint
+            }));
+        });
+
         it("should get data set content in binary mode", async () => {
             let response;
             let caughtError;
@@ -369,6 +395,32 @@ describe("z/OS Files - View", () => {
             }));
         });
 
+        it("should get uss file content when empty", async () => {
+            let response;
+            let caughtError;
+            zosmfExpectSpy.mockImplementationOnce(async (_session, options) => {
+                options.responseStream?.end();
+                return {};
+            });
+
+            try {
+                response = await Get.USSFile(dummySession, ussfile);
+            } catch (e) {
+                caughtError = e;
+            }
+
+            const endpoint = posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, ussfile);
+
+            expect(caughtError).toBeUndefined();
+            expect(response).toEqual(Buffer.alloc(0));
+
+            expect(zosmfExpectSpy).toHaveBeenCalledTimes(1);
+            expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, expect.objectContaining({
+                reqHeaders: [ZosmfHeaders.ACCEPT_ENCODING, ZosmfHeaders.TEXT_PLAIN],
+                resource: endpoint
+            }));
+        });
+
         it("should get uss file content in binary mode", async () => {
             let response;
             let caughtError;

packages/zosfiles/src/methods/get/Get.ts

@@ -9,9 +9,7 @@
 *
 */
 
-import { PassThrough } from "stream";
-import { buffer } from 'stream/consumers';
-import { AbstractSession, ImperativeExpect } from "@zowe/imperative";
+import { AbstractSession, BufferBuilder, ImperativeExpect } from "@zowe/imperative";
 import { ZosFilesMessages } from "../../constants/ZosFiles.messages";
 import { Download } from "../download/Download";
 import { IGetOptions } from "./doc/IGetOptions";
@@ -37,12 +35,12 @@ export class Get {
         ImperativeExpect.toNotBeNullOrUndefined(dataSetName, ZosFilesMessages.missingDatasetName.message);
         ImperativeExpect.toNotBeEqual(dataSetName, "", ZosFilesMessages.missingDatasetName.message);
 
-        const responseStream = new PassThrough();
+        const responseStream = new BufferBuilder();
         await Download.dataSet(session, dataSetName, {
             ...options,
             stream: responseStream
         });
-        return buffer(responseStream);
+        return responseStream.read() ?? Buffer.alloc(0);
     }
 
     /**
@@ -61,11 +59,11 @@ export class Get {
         ImperativeExpect.toNotBeEqual(USSFileName, "", ZosFilesMessages.missingUSSFileName.message);
         ImperativeExpect.toNotBeEqual(options.record, true, ZosFilesMessages.unsupportedDataType.message); // This should never exist for USS files
 
-        const responseStream = new PassThrough();
+        const responseStream = new BufferBuilder();
         await Download.ussFile(session, USSFileName, {
             ...options,
             stream: responseStream
         });
-        return buffer(responseStream);
+        return responseStream.read() ?? Buffer.alloc(0);
     }
 }

packages/zosjobs/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to the Zowe z/OS jobs SDK package will be documented in this file.
 
+## Recent Changes
+
+- BugFix: Fixed error in `DownloadJobs.downloadSpoolContentCommon` method when encoding parameter is not specified. [#2173](https://github.com/zowe/zowe-cli/pull/2173)
+
 ## `8.0.0-next.202403131702`
 
 - BugFix: Removing stack trace for zosjobs errors. Added constant to JobsMessages.ts for error handling. [#2078](https://github.com/zowe/zowe-cli/pull/2078)
@@ -27,6 +31,10 @@ All notable changes to the Zowe z/OS jobs SDK package will be documented in this
 
 - Major: First major version bump for V3
 
+## `7.26.1`
+
+- BugFix: Fixed error in `DownloadJobs.downloadSpoolContentCommon` method when encoding parameter is not specified. [#2173](https://github.com/zowe/zowe-cli/pull/2173)
+
 ## `7.25.0`
 
 - Enhancement: Added the ability to set `internalReaderFileEncoding` on the `submitJcl`, `submitJclString`, `submitJclCommon`, `submitJclNotify`, and `submitJclNotifyCommon` Jobs APIs [#2139](https://github.com/zowe/zowe-cli/pull/2139)

packages/zosjobs/__tests__/__unit__/DownloadJobs.unit.test.ts

@@ -103,6 +103,10 @@ describe("DownloadJobs", () => {
 
         describe("downloadAllSpoolContentCommon", () => {
             it("should allow users to call downloadAllSpoolContentCommon with correct parameters", async () => {
+                let uri: string = "";
+                ZosmfRestClient.getStreamed = jest.fn(async (session: AbstractSession, resource: string, reqHeaders?: any[]): Promise<any> => {
+                    uri = resource;
+                });
                 const allSpoolParms: IDownloadAllSpoolContentParms = {
                     jobid: fakeJobID,
                     jobname: fakeJobName,
@@ -116,6 +120,7 @@ describe("DownloadJobs", () => {
 
                 expect(GetJobs.getSpoolFiles).toHaveBeenCalled();
                 expect(IO.createDirsSyncFromFilePath).toHaveBeenCalledWith(expectedFile);
+                expect(uri).not.toContain("fileEncoding");
             });
 
             it("should allow users to call downloadAllSpoolContentCommon with correct parameters and binary mode", async () => {