From c1f4467c8d03ae8647c7e33d9e6a9c2c8406fa0c Mon Sep 17 00:00:00 2001
From: Sonam Gupta <71771131+sonamgupta21@users.noreply.github.com>
Date: Fri, 13 Sep 2024 12:58:13 +0530
Subject: [PATCH] UI: Handle Null Columns (#1611)

* Fix Query Builder For Null Values

* Added the hide-null0column-checkbox

* Checkbox Style

* CSS

* Lint Fix

* Null columns updates with scroll

* Lint Fix

* Available Fields

* Lint Fix

---------

Signed-off-by: Sonam Gupta <71771131+sonamgupta21@users.noreply.github.com>
Co-authored-by: Kunal Nawale <knawale@sigscalr.io>
---
 eslint.config.mjs           |   4 +-
 static/css/siglens.css      |  54 ++++++++++++++
 static/index.html           |   3 +
 static/js/common.js         |   6 ++
 static/js/event-handlers.js | 145 ++++++++++++++++++++++++++++++++++--
 static/js/query-builder.js  |   8 +-
 static/js/search.js         |   4 +
 7 files changed, 213 insertions(+), 11 deletions(-)

diff --git a/eslint.config.mjs b/eslint.config.mjs
index f7ead0544..48e7a486f 100644
--- a/eslint.config.mjs
+++ b/eslint.config.mjs
@@ -198,8 +198,8 @@ export default [
         "toggleClearButtonVisibility": true,
         "updateQueryModeUI": true,
         "queryMode": true,
-
-
+        "allColumns": true,
+        "columnsWithNonNullValues":true
       },
     },
     rules: {
diff --git a/static/css/siglens.css b/static/css/siglens.css
index 41ab7bd5e..bd69e5ec4 100644
--- a/static/css/siglens.css
+++ b/static/css/siglens.css
@@ -2488,7 +2488,61 @@ input[type="submit" i] {
 
 /* end save query dialog  */
 
+#hide-null-column-box{
+    display: flex;
+    align-items: center;
+}
+
+#hide-null-columns-checkbox{
+    color: var(--select-unselect-header-text-color);
+    font-size: 14px;
+    position: relative;
+    margin-right: 6px;
+    display: flex;
+    align-items: center;
+    width: 18px;
+}
 
+#hide-null-columns-checkbox[type="checkbox"] {
+    width: 16px;
+    height: 16px;
+    accent-color: var(--purple-1);
+} 
+
+#hide-null-columns-checkbox[type="checkbox"] {
+    appearance: none;
+    -webkit-appearance: none;
+    display: flex;
+    align-content: center;
+    justify-content: center;
+    padding: 0.1rem;
+    background-color: white; 
+    border: 1.5px solid var(--purple-1);
+    border-radius: 2px;
+} 
+
+#hide-null-columns-checkbox[type="checkbox"]:not(:checked)::before {
+    content: '';
+}
+
+#hide-null-columns-checkbox[type="checkbox"]:checked {
+    background-color: var(--purple-1);
+}
+
+#hide-null-columns-checkbox[type="checkbox"]:checked::before {
+    content: '\2713'; /* Unicode for checkmark */
+    color: white;
+    font-size: 13px;
+    position: absolute;
+    left: 2px;
+    top: -3px;
+    transform: scale(1);
+}
+
+#hide-null-columns-checkbox[type="checkbox"]:hover {
+    cursor: pointer;
+}
+      
 /* datepicker */
 
 #date-picker-btn {
diff --git a/static/index.html b/static/index.html
index 4356328f3..58285fa49 100644
--- a/static/index.html
+++ b/static/index.html
@@ -296,6 +296,9 @@
                         </div>
                     </div>
                     <div id="views-container">
+                        <label id="hide-null-column-box">
+                            <input type="checkbox" id="hide-null-columns-checkbox" checked="" class="active">Hide Null Columns
+                        </label>
                         <div class="btn-group" role="group" id="log-view">
                             <button class="btn log-opt-single-btn" id="log-opt-single-btn">
                                 <span class="single-line-icon" ></span>
diff --git a/static/js/common.js b/static/js/common.js
index 3b271272a..10e19300a 100644
--- a/static/js/common.js
+++ b/static/js/common.js
@@ -66,6 +66,9 @@ let isQueryBuilderSearch = false;
 let sortByTimestampAtDefault = true;
 let defaultDashboardIds = ['10329b95-47a8-48df-8b1d-0a0a01ec6c42', 'a28f485c-4747-4024-bb6b-d230f101f852', 'bd74f11e-26c8-4827-bf65-c0b464e1f2a4', '53cb3dde-fd78-4253-808c-18e4077ef0f1'];
 let initialSearchData = {};
+let columnsWithNonNullValues = new Set();
+let columnsWithNullValues = new Set();
+let allColumns = new Set();
 let isMetricsScreen = false;
 
 let aggGridOptions = {
@@ -168,6 +171,9 @@ function resetDashboard() {
     resetAvailableFields();
     $('#LogResultsGrid').html('');
     $('#measureAggGrid').html('');
+    columnsWithNonNullValues.clear();
+    columnsWithNullValues.clear();
+    allColumns.clear();
     gridDiv = null;
     eGridDiv = null;
 }
diff --git a/static/js/event-handlers.js b/static/js/event-handlers.js
index 8c38e725d..233981b84 100644
--- a/static/js/event-handlers.js
+++ b/static/js/event-handlers.js
@@ -29,6 +29,7 @@ function setupEventHandlers() {
     $('#views-container #available-fields .select-unselect-header').on('click', '.select-unselect-checkbox', toggleAllAvailableFieldsHandler);
     $('#views-container #available-fields .select-unselect-header').on('click', '.select-unselect-checkmark', toggleAllAvailableFieldsHandler);
     $('#available-fields .fields').on('click', '.available-fields-dropdown-item', availableFieldsSelectHandler);
+    $('#hide-null-columns-checkbox').on('change', handleHideNullColumnsCheckbox);
 
     $('#corner-popup').on('click', '.corner-btn-close', hideCornerPopupError);
 
@@ -499,16 +500,29 @@ function availableFieldsSelectHandler(evt, isCloseIcon = false) {
     if (colName !== 'timestamp') {
         // toggle the column visibility
         $(`.toggle-${encColName}`).toggleClass('active');
-        if ($(`.toggle-${encColName}`).hasClass('active')) {
+        const isSelected = $(`.toggle-${encColName}`).hasClass('active');
+
+        if (isSelected) {
             // Update the selectedFieldsList everytime a field is selected
-            selectedFieldsList.push(colName);
+            if (!selectedFieldsList.includes(colName)) {
+                selectedFieldsList.push(colName);
+            }
         } else {
             // Everytime the field is unselected, remove it from selectedFieldsList
-            for (let i = 0; i < selectedFieldsList.length; i++) {
-                if (selectedFieldsList[i] === colName) {
-                    selectedFieldsList.splice(i, 1);
-                    i--;
-                }
+            selectedFieldsList = selectedFieldsList.filter((field) => field !== colName);
+        }
+
+        // Check if the selected/unselected column is a null column
+        //eslint-disable-next-line no-undef
+        const nullColumns = Array.from(allColumns).filter((column) => columnsWithNullValues?.has(column) && !columnsWithNonNullValues?.has(column));
+        const isNullColumn = nullColumns.includes(colName);
+
+        if (isNullColumn) {
+            const $checkbox = $('#hide-null-columns-checkbox');
+            if (isSelected) {
+                $checkbox.prop('checked', false);
+            } else {
+                $checkbox.prop('checked', true);
             }
         }
     }
@@ -571,6 +585,10 @@ function toggleAllAvailableFieldsHandler(_evt) {
     processTableViewOption();
     let el = $('#available-fields .select-unselect-header');
     let isChecked = el.find('.select-unselect-checkmark');
+    const nullColumnCheckbox = $('#hide-null-columns-checkbox');
+    //eslint-disable-next-line no-undef
+    const nullColumns = Array.from(allColumns).filter((column) => columnsWithNullValues.has(column) && !columnsWithNonNullValues.has(column));
+
     if (isChecked.length === 0) {
         if (theme === 'light') {
             el.append(`<img class="select-unselect-checkmark" src="assets/available-fields-check-light.svg">`);
@@ -584,6 +602,11 @@ function toggleAllAvailableFieldsHandler(_evt) {
             gridOptions.columnApi.setColumnVisible(colName, true);
         });
         selectedFieldsList = tempFieldList;
+
+        // Uncheck the null column checkbox if there are any null columns
+        if (nullColumns.length > 0) {
+            nullColumnCheckbox.prop('checked', false);
+        }
     } else {
         let cmClass = el.find('.select-unselect-checkmark');
         cmClass.remove();
@@ -593,6 +616,7 @@ function toggleAllAvailableFieldsHandler(_evt) {
             gridOptions.columnApi.setColumnVisible(colName, false);
         });
         selectedFieldsList = [];
+        nullColumnCheckbox.prop('checked', true);
     }
     updatedSelFieldList = true;
     // Always hide the logs column
@@ -766,3 +790,110 @@ function saveqInputHandler(evt) {
     evt.preventDefault();
     $(this).addClass('active');
 }
+//eslint-disable-next-line no-unused-vars
+function updateNullColumnsTracking(records) {
+    if (!records || records.length === 0) return;
+
+    records.forEach((record) => {
+        Object.keys(record).forEach((column) => {
+            allColumns.add(column);
+            if (record[column] !== null && record[column] !== undefined && record[column] !== '') {
+                //eslint-disable-next-line no-undef
+                columnsWithNonNullValues.add(column);
+            } else {
+                //eslint-disable-next-line no-undef
+                columnsWithNullValues.add(column);
+            }
+        });
+    });
+}
+//eslint-disable-next-line no-unused-vars
+function finalizeNullColumnsHiding() {
+    //eslint-disable-next-line no-undef
+    const nullColumns = Array.from(allColumns).filter((column) => columnsWithNullValues.has(column) && !columnsWithNonNullValues.has(column));
+    const checkbox = $('#hide-null-columns-checkbox');
+    const checkboxParent = $('#hide-null-column-box');
+
+    if (nullColumns.length === 0) {
+        // No null columns, hide checkbox
+        checkboxParent.hide();
+        updateColumnsVisibility(false, []); // Show all columns
+        return;
+    }
+
+    checkboxParent.show();
+    // Update column visibility if the checkbox is checked
+    const hideNullColumns = checkbox.is(':checked');
+    updateColumnsVisibility(hideNullColumns, nullColumns);
+}
+
+function handleHideNullColumnsCheckbox(event) {
+    const hideNullColumns = event.target.checked;
+    updateColumnsVisibility(hideNullColumns);
+}
+
+function updateColumnsVisibility(hideNullColumns, nullColumns = null) {
+    const columnDefs = gridOptions.columnApi?.getColumns().map((col) => ({ field: col.getColId() }));
+    let updatedSelectedFieldsList = [...selectedFieldsList]; // Use selectedFieldsList instead of availColNames
+
+    if (!nullColumns) {
+        //eslint-disable-next-line no-undef
+        nullColumns = Array.from(allColumns).filter((column) => columnsWithNullValues.has(column) && !columnsWithNonNullValues.has(column));
+    }
+
+    columnDefs?.forEach((colDef) => {
+        const colField = colDef.field;
+        if (colField !== 'timestamp' && colField !== 'logs') {
+            const isSelected = selectedFieldsList.includes(colField);
+            const isNullColumn = nullColumns.includes(colField);
+
+            let shouldBeVisible = isSelected;
+
+            if (hideNullColumns && isNullColumn && isSelected) {
+                shouldBeVisible = false;
+                updatedSelectedFieldsList = updatedSelectedFieldsList.filter((field) => field !== colField);
+            }
+
+            gridOptions.columnApi.setColumnVisible(colField, shouldBeVisible);
+
+            if (shouldBeVisible) {
+                $(`.toggle-${string2Hex(colField)}`).addClass('active');
+            } else {
+                $(`.toggle-${string2Hex(colField)}`).removeClass('active');
+            }
+        }
+    });
+    updateAvailableFieldsUI(updatedSelectedFieldsList);
+    gridOptions.api?.sizeColumnsToFit();
+}
+
+function updateAvailableFieldsUI(updatedSelectedFieldsList) {
+    let visibleColumns = 0;
+    let totalColumns = availColNames.length;
+    if (updatedSelectedFieldsList && updatedSelectedFieldsList.length > 0) {
+        availColNames.forEach((colName) => {
+            if (updatedSelectedFieldsList.includes(colName)) {
+                visibleColumns++;
+                $(`.toggle-${string2Hex(colName)}`).addClass('active');
+            } else {
+                $(`.toggle-${string2Hex(colName)}`).removeClass('active');
+            }
+        });
+
+        let el = $('#available-fields .select-unselect-header');
+
+        // Update the toggle-all checkbox
+        if (visibleColumns === totalColumns - 2) {
+            // Excluding timestamp and logs
+            if (theme === 'light') {
+                el.find('.select-unselect-checkmark').remove();
+                el.append(`<img class="select-unselect-checkmark" src="assets/available-fields-check-light.svg">`);
+            } else {
+                el.find('.select-unselect-checkmark').remove();
+                el.append(`<img class="select-unselect-checkmark" src="assets/index-selection-check.svg">`);
+            }
+        } else {
+            el.find('.select-unselect-checkmark').remove();
+        }
+    }
+}
diff --git a/static/js/query-builder.js b/static/js/query-builder.js
index 560bf8919..d89ffff7a 100644
--- a/static/js/query-builder.js
+++ b/static/js/query-builder.js
@@ -573,8 +573,12 @@ function getValuesofColumn(chooseColumn) {
         if (res && res.hits && res.hits.records) {
             for (let i = 0; i < res.hits.records.length; i++) {
                 let cur = res.hits.records[i][chooseColumn];
-                if (typeof cur == 'string') valuesOfColumn.add(cur);
-                else valuesOfColumn.add(cur.toString());
+
+                // Check if cur is not null or undefined before processing
+                if (cur !== null && cur !== undefined) {
+                    if (typeof cur == 'string') valuesOfColumn.add(cur);
+                    else valuesOfColumn.add(cur.toString());
+                }
             }
         }
         let arr = Array.from(valuesOfColumn);
diff --git a/static/js/search.js b/static/js/search.js
index eb9a1a897..1ad02a2cb 100644
--- a/static/js/search.js
+++ b/static/js/search.js
@@ -118,6 +118,8 @@ function doSearch(data) {
                     }
                     resetDataTable(firstQUpdate);
                     processQueryUpdate(jsonEvent, eventType, totalEventsSearched, timeToFirstByte, totalHits);
+                    //eslint-disable-next-line no-undef
+                    updateNullColumnsTracking(jsonEvent.hits.records);
                     console.timeEnd('QUERY_UPDATE');
                     firstQUpdate = false;
                     break;
@@ -131,6 +133,8 @@ function doSearch(data) {
                     canScrollMore = jsonEvent.can_scroll_more;
                     scrollFrom = jsonEvent.total_rrc_count;
                     processCompleteUpdate(jsonEvent, eventType, totalEventsSearched, timeToFirstByte, eqRel);
+                    //eslint-disable-next-line no-undef
+                    finalizeNullColumnsHiding();
                     console.timeEnd('COMPLETE');
                     socket.close(1000);
                     break;