Merge pull request #3 from ANSSI-FR/deny-tags

Add the ability to deny tags in flow filtering
ANSSI-FR · Jul 12, 2024 · 3c316f6 · 3c316f6
2 parents 7d8c43f + 92fa4db
commit 3c316f6
Show file tree

Hide file tree

Showing 4 changed files with 96 additions and 35 deletions.
diff --git a/webapp/main.py b/webapp/main.py
@@ -42,31 +42,41 @@ async def api_flow_list(request):
     services = request.query_params.getlist("service")
     app_proto = request.query_params.get("app_proto")
     search = request.query_params.get("search")
-    tags = request.query_params.getlist("tag")
+    tags_require = request.query_params.getlist("tag_require")
+    tags_deny = request.query_params.getlist("tag_deny")
     if not ts_to.isnumeric():
         raise HTTPException(400)
 
     # Query flows and associated tags using filters
     query = """
         WITH fsrvs AS (SELECT value FROM json_each(?1)),
-          ftags AS (SELECT value FROM json_each(?2)),
-          fsearchfid AS (SELECT value FROM json_each(?5))
+          ftags_req AS (SELECT value FROM json_each(?2)),
+          ftags_deny AS (SELECT value FROM json_each(?3)),
+          fsearchfid AS (SELECT value FROM json_each(?6))
         SELECT id, ts_start, ts_end, dest_ipport, app_proto,
           (SELECT GROUP_CONCAT(tag) FROM alert WHERE flow_id = flow.id) AS tags
-        FROM flow WHERE ts_start <= ?3 AND (?4 IS NULL OR app_proto = ?4)
+        FROM flow WHERE ts_start <= ?4 AND (?5 IS NULL OR app_proto = ?5)
     """
     if services == ["!"]:
         # Filter flows related to no services
         query += "AND NOT (src_ipport IN fsrvs OR dest_ipport IN fsrvs)"
         services = sum(CTF_CONFIG["services"].values(), [])
     elif services:
         query += "AND (src_ipport IN fsrvs OR dest_ipport IN fsrvs)"
-    if tags:
+    if tags_deny:
+        # No alert with at least a denied tag exists for this flow
+        query += """
+            AND NOT EXISTS (
+                SELECT 1 FROM alert
+                WHERE flow_id == flow.id AND alert.tag IN ftags_deny
+            )
+        """
+    if tags_require:
         # Relational division to get all flow_id matching all chosen tags
         query += """
             AND flow.id IN (
-                SELECT flow_id FROM alert WHERE tag IN ftags GROUP BY flow_id
-                HAVING COUNT(*) = (SELECT COUNT(*) FROM ftags)
+                SELECT flow_id FROM alert WHERE tag IN ftags_req GROUP BY flow_id
+                HAVING COUNT(*) = (SELECT COUNT(*) FROM ftags_req)
             )
         """
     search_fid = []
@@ -84,7 +94,8 @@ async def api_flow_list(request):
         query,
         (
             json.dumps(services),
-            json.dumps(tags),
+            json.dumps(tags_require),
+            json.dumps(tags_deny),
             int(ts_to) * 1000,
             app_proto,
             json.dumps(search_fid),

diff --git a/webapp/static/js/api.js b/webapp/static/js/api.js
@@ -20,9 +20,10 @@ export default class Api {
    * @param {Array} services Keep only flows matching these IP address and ports
    * @param {String} appProto Keep only flows matching this app-layer protocol
    * @param {String} search Search for this glob pattern in flows payloads
-   * @param {Array} tags Keep only flows matching these tags
+   * @param {Array} tagsRequire Keep only flows matching these tags
+   * @param {Array} tagsDeny Deny flows matching these tags
    */
-  async listFlows (timestampFrom, timestampTo, services, appProto, search, tags) {
+  async listFlows (timestampFrom, timestampTo, services, appProto, search, tagsRequire, tagsDeny) {
     const url = new URL(`${location.origin}${location.pathname}api/flow`)
     if (typeof timestampFrom === 'number') {
       url.searchParams.append('from', timestampFrom)
@@ -39,8 +40,11 @@ export default class Api {
     if (search) {
       url.searchParams.append('search', search)
     }
-    tags?.forEach((t) => {
-      url.searchParams.append('tag', t)
+    tagsRequire?.forEach((t) => {
+      url.searchParams.append('tag_require', t)
+    })
+    tagsDeny?.forEach((t) => {
+      url.searchParams.append('tag_deny', t)
     })
     const response = await fetch(url.href, {})
     if (!response.ok) {

diff --git a/webapp/static/js/flowlist.js b/webapp/static/js/flowlist.js
@@ -158,18 +158,38 @@ class FlowList {
       const tag = e.target.closest('a')?.dataset.tag
       if (tag) {
         const url = new URL(document.location)
-        const activeTags = url.searchParams.getAll('tag')
-        if (activeTags.includes(tag)) {
-          // Remove tag
-          url.searchParams.delete('tag')
-          activeTags.forEach(t => {
+        const requiredTags = url.searchParams.getAll('tag_require')
+        const deniedTags = url.searchParams.getAll('tag_deny')
+        if (requiredTags.includes(tag)) {
+          // Remove tag from required tags
+          url.searchParams.delete('tag_require')
+          requiredTags.forEach(t => {
             if (t !== tag) {
-              url.searchParams.append('tag', t)
+              url.searchParams.append('tag_require', t)
             }
           })
+          // If shift is pressed, then add to denied tags
+          if (e.shiftKey) {
+            url.searchParams.append('tag_deny', tag)
+          }
+        } else if (deniedTags.includes(tag)) {
+          // Remove tag from denied tags
+          url.searchParams.delete('tag_deny')
+          deniedTags.forEach(t => {
+            if (t !== tag) {
+              url.searchParams.append('tag_deny', t)
+            }
+          })
+          // If shift is pressed, then add to required tags
+          if (e.shiftKey) {
+            url.searchParams.append('tag_require', tag)
+          }
+        } else if (e.shiftKey) {
+          // Add tag to denied tags
+          url.searchParams.append('tag_deny', tag)
         } else {
-          // Add tag
-          url.searchParams.append('tag', tag)
+          // Add tag to required tags
+          url.searchParams.append('tag_require', tag)
         }
         window.history.pushState(null, '', url.href)
         this.update()
@@ -262,25 +282,33 @@ class FlowList {
    */
   updateTagFilter (tags) {
     // Empty dropdown content
-    const tagFilterDropdown = document.getElementById('filter-tag')
-    while (tagFilterDropdown.lastChild) {
-      tagFilterDropdown.removeChild(tagFilterDropdown.lastChild)
-    }
+    ['filter-tag-available', 'filter-tag-require', 'filter-tag-deny'].forEach(id => {
+      const el = document.getElementById(id)
+      el.parentElement.classList.add('d-none')
+      while (el.lastChild) {
+        el.removeChild(el.lastChild)
+      }
+    })
 
+    // Create tags and append to corresponding section of dropdown
+    const url = new URL(document.location)
+    const requiredTags = url.searchParams.getAll('tag_require')
+    const deniedTags = url.searchParams.getAll('tag_deny')
     tags.forEach(t => {
-      // Create tag and append to dropdown
       const { tag, color } = t
-      const url = new URL(document.location)
-      const activeTags = url.searchParams.getAll('tag')
       const badge = this.tagBadge(tag, color)
-      badge.classList.add('border', 'border-2')
-      badge.classList.toggle('border-purple', activeTags.includes(tag))
-      badge.classList.toggle('text-bg-purple', activeTags.includes(tag))
       const link = document.createElement('a')
       link.href = '#'
       link.dataset.tag = tag
       link.appendChild(badge)
-      tagFilterDropdown.appendChild(link)
+      let destElement = document.getElementById('filter-tag-available')
+      if (requiredTags.includes(tag)) {
+        destElement = document.getElementById('filter-tag-require')
+      } else if (deniedTags.includes(tag)) {
+        destElement = document.getElementById('filter-tag-deny')
+      }
+      destElement.appendChild(link)
+      destElement.parentElement.classList.remove('d-none')
     })
   }
 
@@ -381,14 +409,16 @@ class FlowList {
     const services = url.searchParams.getAll('service')
     const filterAppProto = url.searchParams.get('app_proto')
     const filterSearch = url.searchParams.get('search')
-    const filterTags = url.searchParams.getAll('tag')
+    const filterTagsRequire = url.searchParams.getAll('tag_require')
+    const filterTagsDeny = url.searchParams.getAll('tag_deny')
     const { flows, appProto, tags } = await this.apiClient.listFlows(
       fromTs ? Number(fromTs) : null,
       toTs ? Number(toTs) : null,
       services,
       filterAppProto,
       filterSearch,
-      filterTags
+      filterTagsRequire,
+      filterTagsDeny
     )
 
     // Update search input
@@ -402,7 +432,7 @@ class FlowList {
     this.updateActiveFlow()
 
     // Update filter dropdown visual indicator
-    document.querySelector('#dropdown-filter > button').classList.toggle('text-bg-purple', toTs || filterTags.length || filterAppProto || filterSearch)
+    document.querySelector('#dropdown-filter > button').classList.toggle('text-bg-purple', toTs || filterTagsRequire.length || filterTagsDeny.length || filterAppProto || filterSearch)
 
     // Update service filter select state
     document.getElementById('services-select').value = services.join(',')

diff --git a/webapp/templates/index.html.jinja2 b/webapp/templates/index.html.jinja2
@@ -85,7 +85,23 @@
               </span>
               <input type="text" class="form-control" placeholder="glob, e.g. 'ex?mple'" id="filter-search">
             </div>
-            <div id="filter-tag"></div>
+            <div id="filter-tag">
+              <div class="card mb-2 bg-secondary-subtle rounded-0">
+                <header class="card-header d-flex justify-content-between py-1 px-2 small">Available tags</header>
+                <div class="card-body mb-0 p-2" id="filter-tag-available"></div>
+              </div>
+              <div class="card mb-2 bg-secondary-subtle rounded-0 border-success">
+                <header class="card-header d-flex justify-content-between py-1 px-2 small">Required tags</header>
+                <div class="card-body mb-0 p-2" id="filter-tag-require"></div>
+              </div>
+              <div class="card mb-2 bg-secondary-subtle rounded-0 border-danger">
+                <header class="card-header d-flex justify-content-between py-1 px-2 small">Denied tags</header>
+                <div class="card-body mb-0 p-2" id="filter-tag-deny"></div>
+              </div>
+              <p class="my-1">
+                <small class="fw-light fst-italic">Hold <kbd>Shift</kbd> to deny tag.</small>
+              </p>
+            </div>
           </div>
         </div>
       </div>