Skip to content

Commit

Permalink
Merge pull request #2026 from codefori/feature/object_list_perf
Browse files Browse the repository at this point in the history 
Initial work to remove joins from object list
  • Loading branch information
@worksofliam
worksofliam authored May 6, 2024
2 parents 6858b7c + 47a509a commit aeaac2d
Show file tree
Hide file tree
Showing 9 changed files with 180 additions and 73 deletions.
40 changes: 25 additions & 15 deletions src/api/IBMiContent.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import tmp from 'tmp';
import util from 'util';
import { window } from 'vscode';
import { ObjectTypes } from '../filesystems/qsys/Objects';
import { CommandResult, IBMiError, IBMiMember, IBMiObject, IFSFile, QsysPath } from '../typings';
import { AttrOperands, CommandResult, IBMiError, IBMiMember, IBMiObject, IFSFile, QsysPath } from '../typings';
import { ConnectionConfiguration } from './Configuration';
import { FilterType, parseFilter, singleGenericName } from './Filter';
import { default as IBMi } from './IBMi';
Expand Down Expand Up @@ -530,12 +530,8 @@ export default class IBMiContent {
` 'PF' as ATTRIBUTE,`,
` t.TABLE_TEXT as TEXT,`,
` 1 as IS_SOURCE,`,
` p.NUMBER_PARTITIONS as NB_MBR,`,
` t.ROW_LENGTH as SOURCE_LENGTH,`,
` c.CCSID as CCSID`,
` t.ROW_LENGTH as SOURCE_LENGTH`,
`from QSYS2.SYSTABLES as t`,
` join QSYS2.SYSTABLESTAT as p on ( t.SYSTEM_TABLE_SCHEMA, t.SYSTEM_TABLE_NAME ) = ( p.SYSTEM_TABLE_SCHEMA, p.SYSTEM_TABLE_NAME )`,
` join QSYS2.SYSCOLUMNS as c on ( t.SYSTEM_TABLE_SCHEMA, t.SYSTEM_TABLE_NAME, 3 ) = ( c.SYSTEM_TABLE_SCHEMA, c.SYSTEM_TABLE_NAME, c.ORDINAL_POSITION )`,
`where t.table_schema = '${library}' and t.file_type = 'S'${objectNameLike()}`,
];
} else if (!withSourceFiles) {
Expand Down Expand Up @@ -565,12 +561,8 @@ export default class IBMiContent {
` 'PF' as ATTRIBUTE,`,
` t.TABLE_TEXT as TEXT,`,
` 1 as IS_SOURCE,`,
` p.NUMBER_PARTITIONS as NB_MBR,`,
` t.ROW_LENGTH as SOURCE_LENGTH,`,
` c.CCSID as CCSID`,
` t.ROW_LENGTH as SOURCE_LENGTH`,
` from QSYS2.SYSTABLES as t`,
` join QSYS2.SYSTABLESTAT as p on ( t.SYSTEM_TABLE_SCHEMA, t.SYSTEM_TABLE_NAME ) = ( p.SYSTEM_TABLE_SCHEMA, p.SYSTEM_TABLE_NAME )`,
` join QSYS2.SYSCOLUMNS as c on ( t.SYSTEM_TABLE_SCHEMA, t.SYSTEM_TABLE_NAME, 3 ) = ( c.SYSTEM_TABLE_SCHEMA, c.SYSTEM_TABLE_NAME, c.ORDINAL_POSITION )`,
` where t.table_schema = '${library}' and t.file_type = 'S'${objectNameLike()}`,
`), OBJD as (`,
` select `,
Expand All @@ -592,9 +584,7 @@ export default class IBMiContent {
` o.ATTRIBUTE,`,
` o.TEXT,`,
` case when s.IS_SOURCE is not null then s.IS_SOURCE else o.IS_SOURCE end as IS_SOURCE,`,
` s.NB_MBR,`,
` s.SOURCE_LENGTH,`,
` s.CCSID,`,
` o.SIZE,`,
` o.CREATED,`,
` o.CHANGED,`,
Expand All @@ -612,10 +602,8 @@ export default class IBMiContent {
type: String(object.TYPE),
attribute: String(object.ATTRIBUTE),
text: String(object.TEXT || ""),
memberCount: object.NB_MBR !== undefined ? Number(object.NB_MBR) : undefined,
sourceFile: Boolean(object.IS_SOURCE),
sourceLength: object.SOURCE_LENGTH !== undefined ? Number(object.SOURCE_LENGTH) : undefined,
CCSID: object.CCSID !== undefined ? Number(object.CCSID) : undefined,
size: Number(object.SIZE),
created: new Date(Number(object.CREATED)),
changed: new Date(Number(object.CHANGED)),
Expand Down Expand Up @@ -979,4 +967,26 @@ export default class IBMiContent {

return cl;
}

async getAttributes(path: string | (QsysPath & { member?: string }), ...operands: AttrOperands[]) {
const target = (path = typeof path === 'string' ? path : Tools.qualifyPath(path.library, path.name, path.member, path.asp));
const result = await this.ibmi.sendCommand({ command: `${this.ibmi.remoteFeatures.attr} -p ${target} ${operands.join(" ")}` });
if (result.code === 0) {
return result.stdout
.split('\n')
.map(line => line.split('='))
.reduce((attributes, [key, value]) => {
attributes[key] = value;
return attributes;
}, {} as Record<string, string>)
}
}

async countMembers(path: QsysPath) {
return this.countFiles(Tools.qualifyPath(path.library, path.name, undefined, path.asp))
}

async countFiles(directory: string) {
return Number((await this.ibmi.sendCommand({ command: `ls | wc -l`, directory })).stdout.trim());
}
}
13 changes: 7 additions & 6 deletions src/api/Tools.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,18 +164,19 @@ export namespace Tools {
}

/**
* Build the IFS path string to a member
* Build the IFS path string to an object or member
* @param library
* @param object
* @param member
* @param member Optional
* @param iasp Optional: an iASP name
*/
export function qualifyPath(library: string, object: string, member: string, iasp?: string, sanitise?: boolean) {
export function qualifyPath(library: string, object: string, member?: string, iasp?: string, sanitise?: boolean) {
const libraryPath = library === `QSYS` ? `QSYS.LIB` : `QSYS.LIB/${Tools.sanitizeLibraryNames([library]).join(``)}.LIB`;
const memberPath = `${object}.FILE/${member}.MBR`
const memberSubpath = sanitise ? Tools.escapePath(memberPath) : memberPath;
const filePath = `${object}.FILE`;
const memberPath = member ? `/${member}.MBR` : '';
const subPath = `${filePath}${memberPath}`;

const result = (iasp && iasp.length > 0 ? `/${iasp}` : ``) + `/${libraryPath}/${memberSubpath}`;
const result = (iasp && iasp.length > 0 ? `/${iasp}` : ``) + `/${libraryPath}/${sanitise ? subPath : Tools.escapePath(subPath)}`;
return result;
}

Expand Down
18 changes: 9 additions & 9 deletions src/api/local/deployTools.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { Tools } from '../Tools';
import { LocalLanguageActions } from './LocalLanguageActions';
import { Deployment } from './deployment';

type ServerFileChanges = {uploads: Uri[], relativeRemoteDeletes: string[]};
type ServerFileChanges = { uploads: Uri[], relativeRemoteDeletes: string[] };

export namespace DeployTools {
export async function launchActionsSetup(workspaceFolder?: WorkspaceFolder) {
Expand Down Expand Up @@ -39,7 +39,7 @@ export namespace DeployTools {
}
}

export function getRemoteDeployDirectory(workspaceFolder: WorkspaceFolder): string|undefined {
export function getRemoteDeployDirectory(workspaceFolder: WorkspaceFolder): string | undefined {
const storage = instance.getStorage();
const existingPaths = storage?.getDeployment();
return existingPaths ? existingPaths[workspaceFolder.uri.fsPath] : undefined;
Expand All @@ -52,7 +52,7 @@ export namespace DeployTools {
* @param method if no method is provided, a prompt will be shown to pick the deployment method.
* @returns the index of the deployed workspace or `undefined` if the deployment failed
*/
export async function launchDeploy(workspaceIndex?: number, method?: DeploymentMethod): Promise<{remoteDirectory: string, workspaceId: number} | undefined> {
export async function launchDeploy(workspaceIndex?: number, method?: DeploymentMethod): Promise<{ remoteDirectory: string, workspaceId: number } | undefined> {
const folder = await Deployment.getWorkspaceFolder(workspaceIndex);
if (folder) {
const remotePath = getRemoteDeployDirectory(folder);
Expand Down Expand Up @@ -80,8 +80,8 @@ export namespace DeployTools {

if (methods.find((element) => element.method === defaultDeploymentMethod)) { // default deploy method is usable
method = defaultDeploymentMethod
} else {

} else {
if (defaultDeploymentMethod as string !== '') {
vscode.window.showWarningMessage('Default deployment method is set but not usable in your environment.')
}
Expand Down Expand Up @@ -151,7 +151,7 @@ export namespace DeployTools {
break;

case "compare":
const {uploads, relativeRemoteDeletes} = await getDeployCompareFiles(parameters, progress)
const { uploads, relativeRemoteDeletes } = await getDeployCompareFiles(parameters, progress)
files.push(...uploads);
deletes.push(...relativeRemoteDeletes);
break;
Expand Down Expand Up @@ -249,11 +249,11 @@ export namespace DeployTools {

export async function getDeployCompareFiles(parameters: DeploymentParameters, progress?: vscode.Progress<{ message?: string }>): Promise<ServerFileChanges> {
if (Deployment.getConnection().remoteFeatures.md5sum) {
const isEmpty = (await Deployment.getConnection().sendCommand({ directory: parameters.remotePath, command: `ls | wc -l` })).stdout === "0";
const isEmpty = await Deployment.getContent().countFiles(parameters.remotePath) === 0;
if (isEmpty) {
Deployment.deploymentLog.appendLine("Remote directory is empty; switching to 'deploy all'");
const allFiles = await getDeployAllFiles(parameters);
return {uploads: allFiles, relativeRemoteDeletes: []};
return { uploads: allFiles, relativeRemoteDeletes: [] };
}
else {
Deployment.deploymentLog.appendLine("Starting MD5 synchronization transfer");
Expand Down Expand Up @@ -339,7 +339,7 @@ export namespace DeployTools {
// get the .deployignore file or .gitignore file from workspace with priority to .deployignore
const ignoreFile = [
...await vscode.workspace.findFiles(new vscode.RelativePattern(workspaceFolder, `.deployignore`), ``, 1),
...await vscode.workspace.findFiles(new vscode.RelativePattern(workspaceFolder, `.gitignore`), ``, 1)
...await vscode.workspace.findFiles(new vscode.RelativePattern(workspaceFolder, `.gitignore`), ``, 1)
].at(0);

if (ignoreFile) {
Expand Down
8 changes: 8 additions & 0 deletions src/api/local/deployment.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,14 @@ export namespace Deployment {
return connection;
}

export function getContent() {
const content = instance.getContent();
if (!content) {
throw new Error("Please connect to an IBM i");
}
return content;
}

export async function createRemoteDirectory(remotePath: string) {
return await getConnection().sendCommand({
command: `mkdir -p "${remotePath}"`
Expand Down
15 changes: 7 additions & 8 deletions src/testing/action.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
import assert from "assert";
import { existsSync } from "fs";
import { TestSuite } from ".";
import { instance } from "../instantiate";
import vscode from "vscode";
import { File, Folder, createFolder } from "./deployTools";
import { TestSuite } from ".";
import { CompileTools } from "../api/CompileTools";
import { Tools } from "../api/Tools";
import { LocalLanguageActions } from "../api/local/LocalLanguageActions";
import { DeployTools } from "../api/local/deployTools";
import { getEnvConfig } from "../api/local/env";
import { getMemberUri, getUriFromPath } from "../filesystems/qsys/QSysFs";
import { Action } from "../typings";
import { CompileTools } from "../api/CompileTools";
import { instance } from "../instantiate";
import { Action, IBMiObject } from "../typings";
import { File, Folder, createFolder } from "./deployTools";

export const helloWorldProject: Folder = {
name: `DeleteMe_${Tools.makeid()}`,
Expand Down Expand Up @@ -155,9 +155,8 @@ async function testHelloWorldProgram(uri: vscode.Uri, action: Action, library: s
type: '*PGM',
text: '',
attribute: 'RPGLE',
sourceFile: false,
memberCount: undefined,
}));
sourceFile: false
} as IBMiObject));

const connection = instance.getConnection();
await connection?.runCommand({ command: `DLTOBJ OBJ(${library}/HELLO) OBJTYPE(*PGM)` });
Expand Down
22 changes: 8 additions & 14 deletions src/testing/connection.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,31 +272,25 @@ export const ConnectionSuite: TestSuite = {
}
},
{
name: `Test withTempDirectory`, test: async () => {
name: `Test withTempDirectory and countFiles`, test: async () => {
const connection = instance.getConnection()!;
let temp;
const countFiles = async (dir: string) => {
const countResult = await connection.sendCommand({ command: `find ${dir} -type f | wc -l` });
assert.strictEqual(countResult.code, 0);
return Number(countResult.stdout);
};
const content = instance.getContent()!;
let temp;

await connection.withTempDirectory(async tempDir => {
temp = tempDir;
//Directory must exist
assert.strictEqual((await connection.sendCommand({ command: `[ -d ${tempDir} ]` })).code, 0);

//Directory must be empty
assert.strictEqual(await countFiles(tempDir), 0);
assert.strictEqual(await content.countFiles(tempDir), 0);

const toCreate = 10;
for (let i = 0; i < toCreate; i++) {
assert.strictEqual((await connection.sendCommand({ command: `echo "Test ${i}" >> ${tempDir}/file${i}` })).code, 0);
}

const newCountResult = await connection.sendCommand({ command: `ls -l ${tempDir} | wc -l` });
assert.strictEqual(newCountResult.code, 0);
assert.strictEqual(await countFiles(tempDir), toCreate);
assert.strictEqual(await content.countFiles(tempDir), toCreate);
});

if (temp) {
Expand All @@ -309,8 +303,8 @@ export const ConnectionSuite: TestSuite = {
name: `Test upperCaseName`, test: async () => {
const connection = instance.getConnection()!;
const variantsBackup = connection.variantChars.local;
try{

try {
const checkVariants = () => connection.variantChars.local !== connection.variantChars.local.toLocaleUpperCase();
//CCSID 297 variants
connection.variantChars.local = '£à$';
Expand All @@ -326,7 +320,7 @@ export const ConnectionSuite: TestSuite = {
assert.strictEqual(connection.upperCaseName("@TesT#ye$"), "@TEST#YE$");
assert.strictEqual(connection.upperCaseName("test_cAsE"), "TEST_CASE");
}
finally{
finally {
connection.variantChars.local = variantsBackup;
}
}
Expand Down
Loading

0 comments on commit aeaac2d

Please sign in to comment.