Skip to content

Commit

Permalink
Merge pull request #1993 from codefori/fix/parser_locale_fixes
Browse files Browse the repository at this point in the history 
Improved support for DBCS output
  • Loading branch information
@worksofliam
worksofliam committed Apr 22, 2024
2 parents 87ce474 + be2a818 commit 0fe9190
Show file tree
Hide file tree
Showing 2 changed files with 197 additions and 2 deletions.
29 changes: 27 additions & 2 deletions src/api/Tools.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,36 @@ export namespace Tools {
figuredLengths = true;
} else {
let row: DB2Row = {};
let slideBytesBy = 0;

headers.forEach(header => {
const strValue = line.substring(header.from, header.from + header.length).trimEnd();
const fromPos = header.from - slideBytesBy;
let strValue = line.substring(fromPos, fromPos + header.length);

let realValue: string | number | null = strValue;
/* For each DBCS character, add 1
Since we are reading characters as UTF8 here, we assume any UTF8 character made up of more than 2 bytes is DBCS
https://stackoverflow.com/a/14495321/4763757
Look at a list of Unicode blocks and their code point ranges, e.g.
the browsable http://www.fileformat.info/info/unicode/block/index.htm or
the official http://www.unicode.org/Public/UNIDATA/Blocks.txt :
Anything up to U+007F takes 1 byte: Basic Latin
Then up to U+07FF it takes 2 bytes: Greek, Arabic, Cyrillic, Hebrew, etc
Then up to U+FFFF it takes 3 bytes: Chinese, Japanese, Korean, Devanagari, etc
Beyond that it takes 4 bytes
*/

const extendedBytes = strValue.split(``).map(c => Buffer.byteLength(c) < 3 ? 0 : 1).reduce((a: number, b: number) => a + b, 0);

slideBytesBy += extendedBytes;
if (extendedBytes > 0) {
strValue = strValue.substring(0, strValue.length - extendedBytes);
}

let realValue: string | number | null = strValue.trimEnd();

// is value a number?
if (strValue.startsWith(` `)) {
Expand Down
170 changes: 170 additions & 0 deletions src/testing/tools.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,176 @@ export const ToolsSuite: TestSuite = {
statement = Tools.fixSQL("@COMMAND LIB(QTEMP/*ALL) TEXT('Hello!');\nSelect * From QTEMP.MYTABLE -- This is mytable");
assert.deepStrictEqual(statement, "Call QSYS2.QCMDEXC('COMMAND LIB(QTEMP/*ALL) TEXT(''Hello!'')');\nSelect * From QTEMP.MYTABLE \n-- This is mytable");
}
},
{
name: `EN result set test`, test: async () => {
const lines = [
`DB2>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
``,
`CUSNUM LSTNAM INIT STREET CITY STATE ZIPCOD CDTLMT CHGCOD BALDUE CDTDUE `,
`-------- -------- ----- ------------- ------ ------ ------- ------- ------- -------- --------`,
` 938472 Henning G K 4859 Elm Ave Dallas TX 75217 5000 3 37.00 0.00`,
``,
` 1 RECORD(S) SELECTED.`,
]

const rows = Tools.db2Parse(lines.join(`\n`));

assert.strictEqual(rows.length, 1);

assert.strictEqual(rows[0].CUSNUM, 938472);
assert.strictEqual(rows[0].LSTNAM, `Henning`);
assert.strictEqual(rows[0].INIT, `G K`);
assert.strictEqual(rows[0].STREET, `4859 Elm Ave`);
assert.strictEqual(rows[0].CITY, `Dallas`);
assert.strictEqual(rows[0].STATE, `TX`);
assert.strictEqual(rows[0].ZIPCOD, 75217);
assert.strictEqual(rows[0].CDTLMT, 5000);
assert.strictEqual(rows[0].CHGCOD, 3);
assert.strictEqual(rows[0].BALDUE, 37);
assert.strictEqual(rows[0].CDTDUE, 0);
}
},
{
name: `FR result set test`, test: async () => {
const lines = [
`DB2>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
``,
`ALPHA NUMÉRIQUE état unité MAJUSCULES minuscules aperçu hélas où? `,
`---------- ----------- ---------- ---------- ---------- ---------- ---------- ---------- -------`,
`Valeur1 1 Français mètre ÀÉÈÇÙ àéèçù déterminé oui? LÀ-BAS!`,
``,
` 1 RECORD(S) SELECTED.`,
]

const rows = Tools.db2Parse(lines.join(`\n`));

assert.strictEqual(rows.length, 1);

assert.strictEqual(rows[0].ALPHA, `Valeur1`);
assert.strictEqual(rows[0].NUMÉRIQUE, 1);
assert.strictEqual(rows[0].état, `Français`);
assert.strictEqual(rows[0].unité, `mètre`);
assert.strictEqual(rows[0].MAJUSCULES, `ÀÉÈÇÙ`);
assert.strictEqual(rows[0].minuscules, `àéèçù`);
assert.strictEqual(rows[0].aperçu, `déterminé`);
assert.strictEqual(rows[0].hélas, `oui?`);
assert.strictEqual(rows[0]["où?"], `LÀ-BAS!`);
}
},
{
name: `DA result set test`, test: async () => {
const lines = [
`DB2>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
``,
`COL1A COL2N COL3Æ COL4Ø COL5Å ÆCOL6 ØCOL7 ÅCOL8 ÆCOL9 ØCOL10 ÅCOL11 `,
`---------- ----------- ---------- ---------- ---------- ---------- ---------- ---------- ---------- ---------- ----------`,
`Val1 1 Val3æÆ Val4øØ Val5åÅ æÆVal6 øØVal7 åÅVal8 ValæÆ9 ValøØ10 ValåÅ11 `,
``,
` 1 RECORD(S) SELECTED.`,
]

const rows = Tools.db2Parse(lines.join(`\n`));

assert.strictEqual(rows.length, 1);

assert.strictEqual(rows[0].COL1A, `Val1`);
assert.strictEqual(rows[0].COL2N, 1);
assert.strictEqual(rows[0].COL3Æ, `Val3æÆ`);
assert.strictEqual(rows[0].COL4Ø, `Val4øØ`);
assert.strictEqual(rows[0].COL5Å, `Val5åÅ`);
assert.strictEqual(rows[0].ÆCOL6, `æÆVal6`);
assert.strictEqual(rows[0].ØCOL7, `øØVal7`);
assert.strictEqual(rows[0].ÅCOL8, `åÅVal8`);
assert.strictEqual(rows[0].ÆCOL9, `ValæÆ9`);
assert.strictEqual(rows[0].ØCOL10, `ValøØ10`);
assert.strictEqual(rows[0].ÅCOL11, `ValåÅ11`);
}
},
{
name: `JP result set test (A)`, test: async () => {
const lines = [
`DB2>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
``,
``,
`LIBRARY RECORD_LENGTH ASP SOURCE_FILE NAME TYPE TEXT LINES CREATED CHANGED `,
`---------- -------------------- ------ ------------ ---------- ---------- ------------------------------------------------------------------------------------------------------------------------------ -------------------- -------------------- --------------------`,
`SNDLIB 112 0 QRPGLESRC SNDDEFD DSPF 送信定義一覧／追加・修正 124 1712670631000 1712683676000`,
`SNDLIB 112 0 QRPGLESRC SNDDEFR RPGLE 送信定義一覧／追加・修正 386 1712683661000 1712683692000`,
``,
` 2 RECORD(S) SELECTED.`,
];

const rows = Tools.db2Parse(lines.join(`\n`));

assert.strictEqual(rows.length, 2);

assert.strictEqual(rows[0].LIBRARY, `SNDLIB`);
assert.strictEqual(rows[0].RECORD_LENGTH, 112);
assert.strictEqual(rows[0].ASP, 0);
assert.strictEqual(rows[0].SOURCE_FILE, `QRPGLESRC`);
assert.strictEqual(rows[0].NAME, `SNDDEFD`);
assert.strictEqual(rows[0].TYPE, `DSPF`);
assert.strictEqual(rows[0].TEXT, `送信定義一覧／追加・修正`);
assert.strictEqual(rows[0].LINES, 124);
assert.strictEqual(rows[0].CREATED, 1712670631000);
assert.strictEqual(rows[0].CHANGED, 1712683676000);
}
},
{
name: `JP result set test (B)`, test: async () => {
const lines = [
`DB2>`,
` ?>`,
` ?>`,
` ?>`,
` ?>`,
``,
``,
`LIBRARY RECORD_LENGTH ASP SOURCE_FILE NAME TYPE TEXT LINES CREATED CHANGED`,
`--------- -------------------- ------ ------------ ---------- ---------- ------------------------------------------------------------------------------------------------------------------------------ -------------------- -------------------- --------------------`,
`SNDLIB 112 0 QRPGLESRC TESTEDTW RPGLE 日付と時刻を先行０付きで表示-> 8桁では無理？ 9 1713451802000 1713453741000`,
``,
` 1 RECORD(S) SELECTED.`,
];

const rows = Tools.db2Parse(lines.join(`\n`));

assert.strictEqual(rows.length, 1);

assert.strictEqual(rows[0].LIBRARY, `SNDLIB`);
assert.strictEqual(rows[0].RECORD_LENGTH, 112);
assert.strictEqual(rows[0].ASP, 0);
assert.strictEqual(rows[0].SOURCE_FILE, `QRPGLESRC`);
assert.strictEqual(rows[0].NAME, `TESTEDTW`);
assert.strictEqual(rows[0].TYPE, `RPGLE`);
assert.strictEqual(rows[0].TEXT, `日付と時刻を先行０付きで表示-> 8桁では無理？`);
assert.strictEqual(rows[0].LINES, 9);
assert.strictEqual(rows[0].CREATED, 1713451802000);
assert.strictEqual(rows[0].CHANGED, 1713453741000);
}
}
]
};

0 comments on commit 0fe9190

Please sign in to comment.