Remove SEU colour support

package.json

@@ -379,6 +379,11 @@
 								"type": "boolean",
 								"default": false,
 								"description": "Tells the debug service to send more data to the client. Only useful for debugging issues in the service."
+							},
+							"autoFixInvalidCharacters": {
+								"type": "boolean",
+								"default": false,
+								"description": "Automatically fix invalid characters in source members when opened."
 							}
 						}
 					}
@@ -417,11 +422,6 @@
 					"default": true,
 					"description": "If enabled, will log spool files from command executed. You can find it under Output for 'IBM i Compile Log'."
 				},
-				"code-for-ibmi.showSeuColors": {
-					"type": "boolean",
-					"default": false,
-					"description": "If enabled, will colourise lines like SEU. Only supports source members. Requires restart if changed."
-				},
 				"code-for-ibmi.clearOutputEveryTime": {
 					"type": "boolean",
 					"default": true,

src/api/Configuration.ts

@@ -134,7 +134,8 @@ export namespace ConnectionConfiguration {
     defaultDeploymentMethod: DeploymentMethod | '';
     protectedPaths: string[];
     showHiddenFiles: boolean;
-    lastDownloadLocation: string;
+    lastDownloadLocation:string;
+    autoFixInvalidCharacters: boolean;
     [name: string]: any;
   }
 
@@ -214,7 +215,8 @@ export namespace ConnectionConfiguration {
       defaultDeploymentMethod: parameters.defaultDeploymentMethod || ``,
       protectedPaths: (parameters.protectedPaths || []),
       showHiddenFiles: (parameters.showHiddenFiles === true || parameters.showHiddenFiles === undefined),
-      lastDownloadLocation: (parameters.lastDownloadLocation || os.homedir())
+      lastDownloadLocation: (parameters.lastDownloadLocation || os.homedir()),
+      autoFixInvalidCharacters: (parameters.autoFixInvalidCharacters === true)
     }
   }
 

src/filesystems/qsys/extendedContent.ts

@@ -11,9 +11,8 @@ const writeFileAsync = util.promisify(fs.writeFile);
 
 const DEFAULT_RECORD_LENGTH = 80;
 
-// Translate x'25' to x'2F' and back, or x'25' will become x'0A' (linefeed)!
+// Translate x'25' to a space forever, or x'25' will become x'0A' (linefeed)!
 const SEU_GREEN_UL_RI = `x'25'`;
-const SEU_GREEN_UL_RI_temp = `x'2F'`;
 
 export class ExtendedIBMiContent {
   constructor(readonly sourceDateHandler: SourceDateHandler) {
@@ -49,17 +48,10 @@ export class ExtendedIBMiContent {
         this.sourceDateHandler.recordLengths.set(alias, recordLength);
       }
 
-      let rows;
-      if (sourceColourSupport)
-        rows = await connection.runSQL(
-          `select srcdat, rtrim(translate(srcdta, ${SEU_GREEN_UL_RI_temp}, ${SEU_GREEN_UL_RI})) as srcdta from ${aliasPath}`,
-          {forceSafe: true}
-        );
-      else
-        rows = await connection.runSQL(
-          `select srcdat, srcdta from ${aliasPath}`,
-          {forceSafe: true}
-        );
+      let rows = await connection.runSQL(
+        `select srcdat, rtrim(translate(srcdta, ' ', ${SEU_GREEN_UL_RI})) as srcdta from ${aliasPath}`,
+        {forceSafe: true}
+      );
 
       if (rows.length === 0) {
         rows.push({
@@ -133,7 +125,6 @@ export class ExtendedIBMiContent {
       const { library, file, name } = connection.parserMemberPath(uri.path);
       const tempRmt = connection.getTempRemote(library + file + name);
       if (tempRmt) {
-        const sourceColourSupport = GlobalConfiguration.get<boolean>(`showSeuColors`);
         const tmpobj = await tmpFile();
 
         const sourceData = body.split(`\n`);
@@ -150,16 +141,9 @@ export class ExtendedIBMiContent {
             sourceData[i] = sourceData[i].substring(0, recordLength);
           }
 
-          // We only want to do the translate when source colours at enabled.
-          // For large sources, translate adds a bunch of time to the saving process.
-          if (sourceColourSupport)
-            rows.push(
-              `(${sequence}, ${sourceDates[i] ? sourceDates[i].padEnd(6, `0`) : `0`}, translate('${escapeString(sourceData[i])}', ${SEU_GREEN_UL_RI}, ${SEU_GREEN_UL_RI_temp}))`,
-            );
-          else
-            rows.push(
-              `(${sequence}, ${sourceDates[i] ? sourceDates[i].padEnd(6, `0`) : `0`}, '${escapeString(sourceData[i])}')`,
-            );
+          rows.push(
+            `(${sequence}, ${sourceDates[i] ? sourceDates[i].padEnd(6, `0`) : `0`}, '${escapeString(sourceData[i])}')`,
+          );
 
         }
 

src/instantiate.ts

@@ -12,10 +12,10 @@ import { refreshDiagnosticsFromServer } from './api/errors/diagnostics';
 import { setupGitEventHandler } from './api/local/git';
 import { GetMemberInfo } from './components/getMemberInfo';
 import { QSysFS, getUriFromPath, parseFSOptions } from "./filesystems/qsys/QSysFs";
-import { SEUColorProvider } from "./languages/general/SEUColorProvider";
 import { Action, BrowserItem, DeploymentMethod, MemberItem, OpenEditableOptions, WithPath } from "./typings";
 import { ActionsUI } from './webviews/actions';
 import { VariablesUI } from "./webviews/variables";
+import { initialiseColourChecker } from './languages/colour';
 
 export let instance: Instance;
 
@@ -791,10 +791,8 @@ export async function loadAllofExtension(context: vscode.ExtensionContext) {
     })
   );
 
-  // Color provider
-  if (GlobalConfiguration.get<boolean>(`showSeuColors`)) {
-    SEUColorProvider.intitialize(context);
-  }
+  // Colour fixer
+  initialiseColourChecker(context);
 
   // Register git events based on workspace folders
   if (vscode.workspace.workspaceFolders) {

src/languages/colour.ts

@@ -0,0 +1,81 @@
+import { ExtensionContext, l10n, Range, window, workspace, WorkspaceEdit } from "vscode";
+import { instance } from "../instantiate";
+
+const NEW_LINE_NUMBERS = [10, 13];
+
+export function initialiseColourChecker(context: ExtensionContext) {
+  context.subscriptions.push(
+    workspace.onDidOpenTextDocument(async (document) => {
+      if (document.uri.scheme === `member` && !document.isClosed) {
+        const content = document.getText();
+        let doWork = hasInvalidCharacters(content);
+
+        if (doWork) {
+          const shouldFix = await askUserToStart();
+
+          if (shouldFix) {
+            const fixedContent = replaceInvalidCharacters(content);
+            const edit = new WorkspaceEdit();
+            edit.replace(document.uri, new Range(0, 0, document.lineCount, 0), fixedContent);
+            workspace.applyEdit(edit);
+          }
+        }
+      }
+    })
+  )
+}
+
+export function hasInvalidCharacters(content: string) {
+  for (let i = 0; i < content.length; i++) {
+    if (shouldReplaceCharCode(content.charCodeAt(i))) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function shouldReplaceCharCode(charCode: number) {
+  if ((charCode < 32 && !NEW_LINE_NUMBERS.includes(charCode)) || (charCode >= 128 && charCode <= 157)) {
+    return true;
+  }
+  return false;
+}
+
+async function askUserToStart() {
+  const config = instance.getConfig()
+
+  if (config?.autoFixInvalidCharacters) {
+    return true;
+  }
+
+  const always = l10n.t(`Always`);
+  const no = l10n.t(`No`);
+
+  const chosen = await window.showInformationMessage(
+    l10n.t("This member contains invalid characters. Would you like to clean it up?"), 
+    l10n.t(`Yes`), always, no);
+
+  if (chosen === no) {
+    return false;
+  }
+
+  if (chosen === always && config) {
+    config.autoFixInvalidCharacters = true;
+    await instance.setConfig(config);
+  }
+
+  return true;
+}
+
+export function replaceInvalidCharacters(content: string) {
+  const chars = content.split(``);
+
+  // return content.replace(/[\x00-\x1F]/g, ``); // This almost works, but we want to keep line feed / carriage return
+  for (let i = 0; i < content.length; i++) {
+    if (shouldReplaceCharCode(content.charCodeAt(i))) {
+      chars[i] = ` `;
+    }
+  }
+
+  return chars.join(``);
+}