Remove SEU colour support, add cleanup support

package.json

@@ -378,6 +378,11 @@
 								"type": "boolean",
 								"default": false,
 								"description": "Tells the debug service to send more data to the client. Only useful for debugging issues in the service."
+							},
+							"autoFixInvalidCharacters": {
+								"type": "boolean",
+								"default": false,
+								"description": "Automatically fix invalid characters in source members when opened."
 							}
 						}
 					}
@@ -416,11 +421,6 @@
 					"default": true,
 					"description": "If enabled, will log spool files from command executed. You can find it under Output for 'IBM i Compile Log'."
 				},
-				"code-for-ibmi.showSeuColors": {
-					"type": "boolean",
-					"default": false,
-					"description": "If enabled, will colourise lines like SEU. Only supports source members. Requires restart if changed."
-				},
 				"code-for-ibmi.clearOutputEveryTime": {
 					"type": "boolean",
 					"default": true,

src/api/Configuration.ts

@@ -134,6 +134,7 @@ export namespace ConnectionConfiguration {
     protectedPaths: string[];
     showHiddenFiles: boolean;
     lastDownloadLocation:string;
+    autoFixInvalidCharacters: boolean;
     [name: string]: any;
   }  
 
@@ -213,7 +214,8 @@ export namespace ConnectionConfiguration {
       defaultDeploymentMethod: parameters.defaultDeploymentMethod || ``,
       protectedPaths: (parameters.protectedPaths || []),
       showHiddenFiles: (parameters.showHiddenFiles === true || parameters.showHiddenFiles === undefined),
-      lastDownloadLocation: (parameters.lastDownloadLocation || os.homedir())
+      lastDownloadLocation: (parameters.lastDownloadLocation || os.homedir()),
+      autoFixInvalidCharacters: (parameters.autoFixInvalidCharacters === true)
     }
   }
 

src/filesystems/qsys/extendedContent.ts

@@ -10,10 +10,6 @@ const writeFileAsync = util.promisify(fs.writeFile);
 
 const DEFAULT_RECORD_LENGTH = 80;
 
-// Translate x'25' to x'2F' and back, or x'25' will become x'0A' (linefeed)!
-const SEU_GREEN_UL_RI = `x'25'`;
-const SEU_GREEN_UL_RI_temp = `x'2F'`;
-
 export class ExtendedIBMiContent {
   constructor(readonly sourceDateHandler: SourceDateHandler) {
 
@@ -36,7 +32,6 @@ export class ExtendedIBMiContent {
       spf = connection.upperCaseName(spf);
       mbr = connection.upperCaseName(mbr);
 
-      const sourceColourSupport = GlobalConfiguration.get<boolean>(`showSeuColors`);
       const tempLib = config.tempLibrary;
       const alias = getAliasName(lib, spf, mbr);
       const aliasPath = `${tempLib}.${alias}`;
@@ -50,15 +45,9 @@ export class ExtendedIBMiContent {
         this.sourceDateHandler.recordLengths.set(alias, recordLength);
       }
 
-      let rows;
-      if (sourceColourSupport)
-        rows = await content.runSQL(
-          `select srcdat, rtrim(translate(srcdta, ${SEU_GREEN_UL_RI_temp}, ${SEU_GREEN_UL_RI})) as srcdta from ${aliasPath}`
-        );
-      else
-        rows = await content.runSQL(
-          `select srcdat, srcdta from ${aliasPath}`
-        );
+      let rows = await content.runSQL(
+        `select srcdat, srcdta from ${aliasPath}`
+      );
 
       if (rows.length === 0) {
         rows.push({
@@ -130,7 +119,6 @@ export class ExtendedIBMiContent {
       const client = connection.client;
       const tempRmt = connection.getTempRemote(lib + spf + mbr);
       if (tempRmt) {
-        const sourceColourSupport = GlobalConfiguration.get<boolean>(`showSeuColors`);
         const tmpobj = await tmpFile();
 
         const sourceData = body.split(`\n`);
@@ -146,16 +134,9 @@ export class ExtendedIBMiContent {
             sourceData[i] = sourceData[i].substring(0, recordLength);
           }
 
-          // We only want to do the translate when source colours at enabled.
-          // For large sources, translate adds a bunch of time to the saving process.
-          if (sourceColourSupport)
-            rows.push(
-              `(${sequence}, ${sourceDates[i] ? sourceDates[i].padEnd(6, `0`) : `0`}, translate('${escapeString(sourceData[i])}', ${SEU_GREEN_UL_RI}, ${SEU_GREEN_UL_RI_temp}))`,
-            );
-          else
-            rows.push(
-              `(${sequence}, ${sourceDates[i] ? sourceDates[i].padEnd(6, `0`) : `0`}, '${escapeString(sourceData[i])}')`,
-            );
+          rows.push(
+            `(${sequence}, ${sourceDates[i] ? sourceDates[i].padEnd(6, `0`) : `0`}, '${escapeString(sourceData[i])}')`,
+          );
 
         }
 

src/instantiate.ts

@@ -12,11 +12,11 @@ import { refreshDiagnosticsFromServer } from './api/errors/diagnostics';
 import { setupGitEventHandler } from './api/local/git';
 import { GetMemberInfo } from './components/getMemberInfo';
 import { QSysFS, getUriFromPath, parseFSOptions } from "./filesystems/qsys/QSysFs";
-import { SEUColorProvider } from "./languages/general/SEUColorProvider";
 import { t } from './locale';
 import { Action, BrowserItem, DeploymentMethod, MemberItem, OpenEditableOptions, WithPath } from "./typings";
 import { ActionsUI } from './webviews/actions';
 import { VariablesUI } from "./webviews/variables";
+import { initialiseColourChecker } from './languages/colour';
 
 export let instance: Instance;
 
@@ -787,10 +787,8 @@ export async function loadAllofExtension(context: vscode.ExtensionContext) {
     })
   );
 
-  // Color provider
-  if (GlobalConfiguration.get<boolean>(`showSeuColors`)) {
-    SEUColorProvider.intitialize(context);
-  }
+  // Colour fixer
+  initialiseColourChecker(context);
 
   // Register git events based on workspace folders
   if (vscode.workspace.workspaceFolders) {

src/languages/colour.ts

@@ -0,0 +1,82 @@
+import { ExtensionContext, Range, window, workspace, WorkspaceEdit } from "vscode";
+import { instance } from "../instantiate";
+import { t } from "../locale";
+
+const NEW_LINE_NUMBERS = [10, 13];
+
+export function initialiseColourChecker(context: ExtensionContext) {
+  context.subscriptions.push(
+    workspace.onDidOpenTextDocument(async (document) => {
+      if (document.uri.scheme === `member` && !document.isClosed) {
+        const content = document.getText();
+        let doWork = hasInvalidCharacters(content);
+
+        if (doWork) {
+          const shouldFix = await askUserToStart();
+
+          if (shouldFix) {
+            const fixedContent = replaceInvalidCharacters(content);
+            const edit = new WorkspaceEdit();
+            edit.replace(document.uri, new Range(0, 0, document.lineCount, 0), fixedContent);
+            workspace.applyEdit(edit);
+          }
+        }
+      }
+    })
+  )
+}
+
+export function hasInvalidCharacters(content: string) {
+  for (let i = 0; i < content.length; i++) {
+    if (shouldReplaceCharCode(content.charCodeAt(i))) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function shouldReplaceCharCode(charCode: number) {
+  if ((charCode < 32 && !NEW_LINE_NUMBERS.includes(charCode)) || (charCode >= 128 && charCode <= 157)) {
+    return true;
+  }
+  return false;
+}
+
+async function askUserToStart() {
+  const config = instance.getConfig()
+
+  if (config?.autoFixInvalidCharacters) {
+    return true;
+  }
+
+  const always = t(`Always`);
+  const no = t(`No`);
+
+  const chosen = await window.showInformationMessage(
+    t(`seuColours.warning`), 
+    t(`Yes`), always, no);
+
+  if (chosen === no) {
+    return false;
+  }
+
+  if (chosen === always && config) {
+    config.autoFixInvalidCharacters = true;
+    await instance.setConfig(config);
+  }
+
+  return true;
+}
+
+export function replaceInvalidCharacters(content: string) {
+  const chars = content.split(``);
+
+  // return content.replace(/[\x00-\x1F]/g, ``); // This almost works, but we want to keep line feed / carriage return
+  for (let i = 0; i < content.length; i++) {
+    if (shouldReplaceCharCode(content.charCodeAt(i))) {
+      chars[i] = ` `;
+    }
+  }
+
+  return chars.join(``);
+}