From 4bfd4f86311e1aec50f9f8c660570c1b4cc7ad09 Mon Sep 17 00:00:00 2001 From: prakashsvmx Date: Sat, 14 Dec 2024 10:03:56 +0530 Subject: [PATCH] refactor listobjects v1 api to ts --- package-lock.json | 4 +- src/internal/client.ts | 133 +++++++++++++++++++++++++++++++++- src/internal/type.ts | 77 ++++++++++++++++++++ src/internal/xml-parser.ts | 133 ++++++++++++++++++++++++++++------ src/minio.d.ts | 2 - src/minio.js | 141 ------------------------------------- src/transformers.js | 5 -- src/xml-parsers.js | 108 +--------------------------- tests/unit/test.js | 2 +- 9 files changed, 324 insertions(+), 281 deletions(-) diff --git a/package-lock.json b/package-lock.json index ab27646c..c23c1bc7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "minio", - "version": "8.0.2", + "version": "8.0.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "minio", - "version": "8.0.2", + "version": "8.0.3", "license": "Apache-2.0", "dependencies": { "async": "^3.2.4", diff --git a/src/internal/client.ts b/src/internal/client.ts index 637f82e3..6e47f98e 100644 --- a/src/internal/client.ts +++ b/src/internal/client.ts @@ -88,6 +88,9 @@ import type { ItemBucketMetadata, LifecycleConfig, LifeCycleConfigParam, + ListObjectQueryOpts, + ListObjectQueryRes, + ObjectInfo, ObjectLockConfigParam, ObjectLockInfo, ObjectMetaData, @@ -115,13 +118,14 @@ import type { UploadPartConfig, } from './type.ts' import type { ListMultipartResult, UploadedPart } from './xml-parser.ts' -import * as xmlParsers from './xml-parser.ts' import { parseCompleteMultipart, parseInitiateMultipart, + parseListObjects, parseObjectLegalHoldConfig, parseSelectObjectContentResponse, } from './xml-parser.ts' +import * as xmlParsers from './xml-parser.ts' const xml = new xml2js.Builder({ renderOpts: { pretty: false }, headless: true }) @@ -3005,4 +3009,131 @@ export class TypedClient { throw err } } + // list a batch of objects + async listObjectsQuery(bucketName: string, prefix?: string, marker?: string, listQueryOpts?: ListObjectQueryOpts) { + if (!isValidBucketName(bucketName)) { + throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName) + } + if (!isString(prefix)) { + throw new TypeError('prefix should be of type "string"') + } + if (!isString(marker)) { + throw new TypeError('marker should be of type "string"') + } + + if (listQueryOpts && !isObject(listQueryOpts)) { + throw new TypeError('listQueryOpts should be of type "object"') + } + let { Delimiter, MaxKeys, IncludeVersion } = listQueryOpts as ListObjectQueryOpts + + if (!isString(Delimiter)) { + throw new TypeError('Delimiter should be of type "string"') + } + if (!isNumber(MaxKeys)) { + throw new TypeError('MaxKeys should be of type "number"') + } + + const queries = [] + // escape every value in query string, except maxKeys + queries.push(`prefix=${uriEscape(prefix)}`) + queries.push(`delimiter=${uriEscape(Delimiter)}`) + queries.push(`encoding-type=url`) + + if (IncludeVersion) { + queries.push(`versions`) + } + + if (marker) { + marker = uriEscape(marker) + if (IncludeVersion) { + queries.push(`key-marker=${marker}`) + } else { + queries.push(`marker=${marker}`) + } + } + + // no need to escape maxKeys + if (MaxKeys) { + if (MaxKeys >= 1000) { + MaxKeys = 1000 + } + queries.push(`max-keys=${MaxKeys}`) + } + queries.sort() + let query = '' + if (queries.length > 0) { + query = `${queries.join('&')}` + } + + const method = 'GET' + const res = await this.makeRequestAsync({ method, bucketName, query }) + const body = await readAsString(res) + const listQryList = parseListObjects(body) + return listQryList + } + + listObjects( + bucketName: string, + prefix?: string, + recursive?: boolean, + listOpts?: ListObjectQueryOpts | undefined, + ): BucketStream { + if (prefix === undefined) { + prefix = '' + } + if (recursive === undefined) { + recursive = false + } + if (!isValidBucketName(bucketName)) { + throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName) + } + if (!isValidPrefix(prefix)) { + throw new errors.InvalidPrefixError(`Invalid prefix : ${prefix}`) + } + if (!isString(prefix)) { + throw new TypeError('prefix should be of type "string"') + } + if (!isBoolean(recursive)) { + throw new TypeError('recursive should be of type "boolean"') + } + if (listOpts && !isObject(listOpts)) { + throw new TypeError('listOpts should be of type "object"') + } + let marker: string | undefined = '' + const listQueryOpts = { + Delimiter: recursive ? '' : '/', // if recursive is false set delimiter to '/' + MaxKeys: 1000, + IncludeVersion: listOpts?.IncludeVersion, + } + let objects: ObjectInfo[] = [] + let ended = false + const readStream: stream.Readable = new stream.Readable({ objectMode: true }) + readStream._read = async () => { + // push one object per _read() + if (objects.length) { + readStream.push(objects.shift()) + return + } + if (ended) { + return readStream.push(null) + } + + try { + const result: ListObjectQueryRes = await this.listObjectsQuery(bucketName, prefix, marker, listQueryOpts) + if (result.isTruncated) { + marker = result.nextMarker || result.versionIdMarker + } else { + ended = true + } + if (result.objects) { + objects = result.objects + } + // @ts-ignore + readStream._read() + } catch (err) { + readStream.emit('error', err) + } + } + return readStream + } } diff --git a/src/internal/type.ts b/src/internal/type.ts index a2446a32..e5ab5804 100644 --- a/src/internal/type.ts +++ b/src/internal/type.ts @@ -465,3 +465,80 @@ export type UploadPartConfig = { } export type PreSignRequestParams = { [key: string]: string } + +/** List object api types **/ + +// Common types +export type CommonPrefix = { + Prefix: string +} + +export type Owner = { + ID: string + DisplayName: string +} + +export type Metadata = { + Items: MetadataItem[] +} + +export type ObjectInfo = { + key?: string + name?: string + lastModified?: Date // time string of format "2006-01-02T15:04:05.000Z" + etag?: string + owner?: Owner + storageClass?: string + userMetadata?: Metadata + userTags?: string + prefix?: string + size?: number +} + +export type ListObjectQueryRes = { + isTruncated?: boolean + nextMarker?: string + versionIdMarker?: string + objects?: ObjectInfo[] +} + +export type ListObjectQueryOpts = { + Delimiter?: string + MaxKeys?: number + IncludeVersion?: boolean +} +/** List object api types **/ + +// New Attempt + +export type ObjectVersionEntry = { + IsLatest?: string + VersionId?: string +} + +export type ObjectRowEntry = ObjectVersionEntry & { + Key: string + LastModified?: Date | undefined + ETag?: string + Size?: string + Owner?: Owner + StorageClass?: string +} + +export interface ListBucketResultV1 { + Name?: string + Prefix?: string + ContinuationToken?: string + KeyCount?: string + Marker?: string + MaxKeys?: string + Delimiter?: string + IsTruncated?: boolean + Contents: ObjectRowEntry[] + NextKeyMarker?: string + CommonPrefixes: CommonPrefix[] + Version?: ObjectRowEntry[] + DeleteMarker?: ObjectRowEntry[] + VersionIdMarker?: string + NextVersionIdMarker?: string +} diff --git a/src/internal/xml-parser.ts b/src/internal/xml-parser.ts index 3ca83948..c5f57ead 100644 --- a/src/internal/xml-parser.ts +++ b/src/internal/xml-parser.ts @@ -6,13 +6,17 @@ import { XMLParser } from 'fast-xml-parser' import * as errors from '../errors.ts' import { SelectResults } from '../helpers.ts' -import { isObject, parseXml, readableStream, sanitizeETag, sanitizeObjectKey, toArray } from './helper.ts' +import { isObject, parseXml, readableStream, sanitizeETag, sanitizeObjectKey, sanitizeSize, toArray } from './helper.ts' import { readAsString } from './response.ts' import type { BucketItemFromList, BucketItemWithMetadata, + CommonPrefix, CopyObjectResultV1, + ListBucketResultV1, + ObjectInfo, ObjectLockInfo, + ObjectRowEntry, ReplicationConfig, Tags, } from './type.ts' @@ -26,6 +30,13 @@ export function parseBucketRegion(xml: string): string { const fxp = new XMLParser() +const fxpWithoutNumParser = new XMLParser({ + // @ts-ignore + numberParseOptions: { + skipLike: /./, + }, +}) + // Parse XML and return information as Javascript types // parse error XML response export function parseError(xml: string, headerInfo: Record) { @@ -159,23 +170,6 @@ export function parseListObjectsV2WithMetadata(xml: string) { return result } -export type Multipart = { - uploads: Array<{ - key: string - uploadId: string - initiator: unknown - owner: unknown - storageClass: unknown - initiated: unknown - }> - prefixes: { - prefix: string - }[] - isTruncated: boolean - nextKeyMarker: undefined - nextUploadIdMarker: undefined -} - export type UploadedPart = { part: number lastModified?: Date @@ -630,8 +624,103 @@ export function parseCopyObject(xml: string): CopyObjectResultV1 { return result } -export function uploadPartParser(xml: string) { - const xmlObj = parseXml(xml) - const respEl = xmlObj.CopyPartResult - return respEl + +const formatObjInfo = (content: ObjectRowEntry, opts: { IsDeleteMarker?: boolean } = {}) => { + const { Key, LastModified, ETag, Size, VersionId, IsLatest } = content + + if (!isObject(opts)) { + opts = {} + } + + const name = sanitizeObjectKey(toArray(Key)[0] || '') + const lastModified = LastModified ? new Date(toArray(LastModified)[0] || '') : undefined + const etag = sanitizeETag(toArray(ETag)[0] || '') + const size = sanitizeSize(Size || '') + + return { + name, + lastModified, + etag, + size, + versionId: VersionId, + isLatest: IsLatest, + isDeleteMarker: opts.IsDeleteMarker ? opts.IsDeleteMarker : false, + } +} + +// parse XML response for list objects in a bucket +export function parseListObjects(xml: string) { + const result: { objects: ObjectInfo[]; isTruncated?: boolean; nextMarker?: string; versionIdMarker?: string } = { + objects: [], + isTruncated: false, + nextMarker: undefined, + versionIdMarker: undefined, + } + let isTruncated = false + let nextMarker, nextVersionKeyMarker + const xmlobj = fxpWithoutNumParser.parse(xml) + + const parseCommonPrefixesEntity = (commonPrefixEntry: CommonPrefix[]) => { + if (commonPrefixEntry) { + toArray(commonPrefixEntry).forEach((commonPrefix) => { + result.objects.push({ prefix: sanitizeObjectKey(toArray(commonPrefix.Prefix)[0] || ''), size: 0 }) + }) + } + } + + const listBucketResult: ListBucketResultV1 = xmlobj.ListBucketResult + const listVersionsResult: ListBucketResultV1 = xmlobj.ListVersionsResult + + if (listBucketResult) { + if (listBucketResult.IsTruncated) { + isTruncated = listBucketResult.IsTruncated + } + if (listBucketResult.Contents) { + toArray(listBucketResult.Contents).forEach((content) => { + const name = sanitizeObjectKey(toArray(content.Key)[0] || '') + const lastModified = new Date(toArray(content.LastModified)[0] || '') + const etag = sanitizeETag(toArray(content.ETag)[0] || '') + const size = sanitizeSize(content.Size || '') + result.objects.push({ name, lastModified, etag, size }) + }) + } + + if (listBucketResult.Marker) { + nextMarker = listBucketResult.Marker + } else if (isTruncated && result.objects.length > 0) { + nextMarker = result.objects[result.objects.length - 1]?.name + } + parseCommonPrefixesEntity(listBucketResult.CommonPrefixes) + } + + if (listVersionsResult) { + if (listVersionsResult.IsTruncated) { + isTruncated = listVersionsResult.IsTruncated + } + + if (listVersionsResult.Version) { + toArray(listVersionsResult.Version).forEach((content) => { + result.objects.push(formatObjInfo(content)) + }) + } + if (listVersionsResult.DeleteMarker) { + toArray(listVersionsResult.DeleteMarker).forEach((content) => { + result.objects.push(formatObjInfo(content, { IsDeleteMarker: true })) + }) + } + + if (listVersionsResult.NextKeyMarker) { + nextVersionKeyMarker = listVersionsResult.NextKeyMarker + } + if (listVersionsResult.NextVersionIdMarker) { + result.versionIdMarker = listVersionsResult.NextVersionIdMarker + } + parseCommonPrefixesEntity(listVersionsResult.CommonPrefixes) + } + + result.isTruncated = isTruncated + if (isTruncated) { + result.nextMarker = nextVersionKeyMarker || nextMarker + } + return result } diff --git a/src/minio.d.ts b/src/minio.d.ts index cf3937ae..f6c4d30b 100644 --- a/src/minio.d.ts +++ b/src/minio.d.ts @@ -137,8 +137,6 @@ export interface SourceObjectStats { // Exports from library export class Client extends TypedClient { - listObjects(bucketName: string, prefix?: string, recursive?: boolean): BucketStream - listObjectsV2(bucketName: string, prefix?: string, recursive?: boolean, startAfter?: string): BucketStream // Bucket Policy & Notification operations diff --git a/src/minio.js b/src/minio.js index ac51d156..a6b37263 100644 --- a/src/minio.js +++ b/src/minio.js @@ -49,147 +49,6 @@ export class Client extends TypedClient { // * `appName` _string_ - Application name. // * `appVersion` _string_ - Application version. - // list a batch of objects - listObjectsQuery(bucketName, prefix, marker, listQueryOpts = {}) { - if (!isValidBucketName(bucketName)) { - throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName) - } - if (!isString(prefix)) { - throw new TypeError('prefix should be of type "string"') - } - if (!isString(marker)) { - throw new TypeError('marker should be of type "string"') - } - let { Delimiter, MaxKeys, IncludeVersion } = listQueryOpts - - if (!isObject(listQueryOpts)) { - throw new TypeError('listQueryOpts should be of type "object"') - } - - if (!isString(Delimiter)) { - throw new TypeError('Delimiter should be of type "string"') - } - if (!isNumber(MaxKeys)) { - throw new TypeError('MaxKeys should be of type "number"') - } - - const queries = [] - // escape every value in query string, except maxKeys - queries.push(`prefix=${uriEscape(prefix)}`) - queries.push(`delimiter=${uriEscape(Delimiter)}`) - queries.push(`encoding-type=url`) - - if (IncludeVersion) { - queries.push(`versions`) - } - - if (marker) { - marker = uriEscape(marker) - if (IncludeVersion) { - queries.push(`key-marker=${marker}`) - } else { - queries.push(`marker=${marker}`) - } - } - - // no need to escape maxKeys - if (MaxKeys) { - if (MaxKeys >= 1000) { - MaxKeys = 1000 - } - queries.push(`max-keys=${MaxKeys}`) - } - queries.sort() - var query = '' - if (queries.length > 0) { - query = `${queries.join('&')}` - } - - var method = 'GET' - var transformer = transformers.getListObjectsTransformer() - this.makeRequest({ method, bucketName, query }, '', [200], '', true, (e, response) => { - if (e) { - return transformer.emit('error', e) - } - pipesetup(response, transformer) - }) - return transformer - } - - // List the objects in the bucket. - // - // __Arguments__ - // * `bucketName` _string_: name of the bucket - // * `prefix` _string_: the prefix of the objects that should be listed (optional, default `''`) - // * `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. (optional, default `false`) - // * `listOpts _object_: query params to list object with below keys - // * listOpts.MaxKeys _int_ maximum number of keys to return - // * listOpts.IncludeVersion _bool_ true|false to include versions. - // __Return Value__ - // * `stream` _Stream_: stream emitting the objects in the bucket, the object is of the format: - // * `obj.name` _string_: name of the object - // * `obj.prefix` _string_: name of the object prefix - // * `obj.size` _number_: size of the object - // * `obj.etag` _string_: etag of the object - // * `obj.lastModified` _Date_: modified time stamp - // * `obj.isDeleteMarker` _boolean_: true if it is a delete marker - // * `obj.versionId` _string_: versionId of the object - listObjects(bucketName, prefix, recursive, listOpts = {}) { - if (prefix === undefined) { - prefix = '' - } - if (recursive === undefined) { - recursive = false - } - if (!isValidBucketName(bucketName)) { - throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName) - } - if (!isValidPrefix(prefix)) { - throw new errors.InvalidPrefixError(`Invalid prefix : ${prefix}`) - } - if (!isString(prefix)) { - throw new TypeError('prefix should be of type "string"') - } - if (!isBoolean(recursive)) { - throw new TypeError('recursive should be of type "boolean"') - } - if (!isObject(listOpts)) { - throw new TypeError('listOpts should be of type "object"') - } - var marker = '' - const listQueryOpts = { - Delimiter: recursive ? '' : '/', // if recursive is false set delimiter to '/' - MaxKeys: 1000, - IncludeVersion: listOpts.IncludeVersion, - } - var objects = [] - var ended = false - var readStream = Stream.Readable({ objectMode: true }) - readStream._read = () => { - // push one object per _read() - if (objects.length) { - readStream.push(objects.shift()) - return - } - if (ended) { - return readStream.push(null) - } - // if there are no objects to push do query for the next batch of objects - this.listObjectsQuery(bucketName, prefix, marker, listQueryOpts) - .on('error', (e) => readStream.emit('error', e)) - .on('data', (result) => { - if (result.isTruncated) { - marker = result.nextMarker || result.versionIdMarker - } else { - ended = true - } - objects = result.objects - readStream._read() - }) - } - return readStream - } - // listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. diff --git a/src/transformers.js b/src/transformers.js index 452e75e3..2bebb243 100644 --- a/src/transformers.js +++ b/src/transformers.js @@ -95,11 +95,6 @@ export function getHashSummer(enableSHA256) { // Following functions return a stream object that parses XML // and emits suitable Javascript objects. -// Parses listObjects response. -export function getListObjectsTransformer() { - return getConcater(xmlParsers.parseListObjects) -} - // Parses listObjects response. export function getListObjectsV2Transformer() { return getConcater(xmlParsers.parseListObjectsV2) diff --git a/src/xml-parsers.js b/src/xml-parsers.js index b6110a1f..48800051 100644 --- a/src/xml-parsers.js +++ b/src/xml-parsers.js @@ -14,16 +14,8 @@ * limitations under the License. */ -import { XMLParser } from 'fast-xml-parser' - import * as errors from './errors.ts' -import { isObject, parseXml, sanitizeETag, sanitizeObjectKey, sanitizeSize, toArray } from './internal/helper.ts' - -const fxpWithoutNumParser = new XMLParser({ - numberParseOptions: { - skipLike: /./, - }, -}) +import { parseXml, sanitizeETag, sanitizeObjectKey, toArray } from './internal/helper.ts' // parse XML response for bucket notification export function parseBucketNotification(xml) { @@ -98,104 +90,6 @@ export function parseBucketNotification(xml) { return result } -const formatObjInfo = (content, opts = {}) => { - let { Key, LastModified, ETag, Size, VersionId, IsLatest } = content - - if (!isObject(opts)) { - opts = {} - } - - const name = sanitizeObjectKey(toArray(Key)[0]) - const lastModified = new Date(toArray(LastModified)[0]) - const etag = sanitizeETag(toArray(ETag)[0]) - const size = sanitizeSize(Size) - - return { - name, - lastModified, - etag, - size, - versionId: VersionId, - isLatest: IsLatest, - isDeleteMarker: opts.IsDeleteMarker ? opts.IsDeleteMarker : false, - } -} - -// parse XML response for list objects in a bucket -export function parseListObjects(xml) { - var result = { - objects: [], - isTruncated: false, - } - let isTruncated = false - let nextMarker, nextVersionKeyMarker - const xmlobj = fxpWithoutNumParser.parse(xml) - - const parseCommonPrefixesEntity = (responseEntity) => { - if (responseEntity) { - toArray(responseEntity).forEach((commonPrefix) => { - result.objects.push({ prefix: sanitizeObjectKey(toArray(commonPrefix.Prefix)[0]), size: 0 }) - }) - } - } - - const listBucketResult = xmlobj.ListBucketResult - const listVersionsResult = xmlobj.ListVersionsResult - - if (listBucketResult) { - if (listBucketResult.IsTruncated) { - isTruncated = listBucketResult.IsTruncated - } - if (listBucketResult.Contents) { - toArray(listBucketResult.Contents).forEach((content) => { - const name = sanitizeObjectKey(toArray(content.Key)[0]) - const lastModified = new Date(toArray(content.LastModified)[0]) - const etag = sanitizeETag(toArray(content.ETag)[0]) - const size = sanitizeSize(content.Size) - result.objects.push({ name, lastModified, etag, size }) - }) - } - - if (listBucketResult.NextMarker) { - nextMarker = listBucketResult.NextMarker - } else if (isTruncated && result.objects.length > 0) { - nextMarker = result.objects[result.objects.length - 1].name - } - parseCommonPrefixesEntity(listBucketResult.CommonPrefixes) - } - - if (listVersionsResult) { - if (listVersionsResult.IsTruncated) { - isTruncated = listVersionsResult.IsTruncated - } - - if (listVersionsResult.Version) { - toArray(listVersionsResult.Version).forEach((content) => { - result.objects.push(formatObjInfo(content)) - }) - } - if (listVersionsResult.DeleteMarker) { - toArray(listVersionsResult.DeleteMarker).forEach((content) => { - result.objects.push(formatObjInfo(content, { IsDeleteMarker: true })) - }) - } - - if (listVersionsResult.NextKeyMarker) { - nextVersionKeyMarker = listVersionsResult.NextKeyMarker - } - if (listVersionsResult.NextVersionIdMarker) { - result.versionIdMarker = listVersionsResult.NextVersionIdMarker - } - parseCommonPrefixesEntity(listVersionsResult.CommonPrefixes) - } - - result.isTruncated = isTruncated - if (isTruncated) { - result.nextMarker = nextVersionKeyMarker || nextMarker - } - return result -} - // parse XML response for list objects v2 in a bucket export function parseListObjectsV2(xml) { var result = { diff --git a/tests/unit/test.js b/tests/unit/test.js index 33b9fae1..72e559b8 100644 --- a/tests/unit/test.js +++ b/tests/unit/test.js @@ -32,8 +32,8 @@ import { partsRequired, } from '../../src/internal/helper.ts' import { joinHostPort } from '../../src/internal/join-host-port.ts' +import { parseListObjects } from '../../src/internal/xml-parser.ts' import * as Minio from '../../src/minio.js' -import { parseListObjects } from '../../src/xml-parsers.js' const Package = { version: 'development' }