Manual and clustermap

backend/src/app.py

@@ -16,6 +16,7 @@
 from save_document import pack_document, unpack_document
 from app_state import create_app_state
 from validations import validate_fasta
+import cluster
 import platform
 from Bio import SeqIO
 import psutil
@@ -497,8 +498,22 @@ def load_data_and_stats(self, doc_id: str):
         cols_path = os.path.join(cols_dir, f"{cols_file_base}_cols.csv")
 
         if os.path.exists(cols_path):
-            identity_scores = read_csv(cols_path, skiprows=1).values.tolist()
+            cols_data = read_csv(cols_path, skiprows=1).values.tolist()
+
+            id_map = {}
+            identity_scores = []
+
+            for row in cols_data:
+                a, b = row[:2]
+                if a not in id_map:
+                    id_map[a] = len(id_map)
+                if b not in id_map:
+                    id_map[b] = len(id_map)
+                identity_scores.append([id_map[a], id_map[b]] + list(row[2:]))
+
+            ids = list(id_map.keys())
         else:
+           ids = []
            identity_scores = []
 
         df = read_csv(
@@ -514,10 +529,10 @@ def load_data_and_stats(self, doc_id: str):
         max_val = int(nanmax(data_no_diag))
 
         # TODO might be able to make one tick text object for both to use?
-        return data, tick_text, min_val, max_val, identity_scores, stats_df
+        return data, tick_text, min_val, max_val, ids, identity_scores, stats_df
 
     def get_data(self, doc_id: str):
-        data, tick_text, min_val, max_val, identity_scores, stats_df = (
+        data, tick_text, min_val, max_val, ids, identity_scores, stats_df = (
             self.load_data_and_stats(doc_id)
         )
         heat_data = DataFrame(data, index=tick_text)
@@ -526,11 +541,24 @@ def get_data(self, doc_id: str):
         data_to_dump = dict(
             metadata=dict(minVal=min_val, maxVal=max_val),
             data=([tick_text] + parsedData),
+            ids=ids,
             identity_scores=identity_scores,
             full_stats=stats_df.values.tolist()
         )
         return json.dumps(data_to_dump)
 
+    def generate_cluster_data(self, doc_id: str, threshold_one: int, threshold_two: int = 0):
+        doc = get_document(doc_id)
+        if doc is None:
+            raise Exception(f"Could not find document: {doc_id}")
+        matrix_path = get_matrix_path(doc)
+
+        df = cluster.export(matrix_path, threshold_one, threshold_two, False)
+        df = df.rename(columns={str(df.columns[0]): 'id', str(df.columns[1]): 'group'})
+        if len(df.columns) > 2 and df.columns[2] is not None:
+            df = df.rename(columns={str(df.columns[2]): 'subgroup'})
+        return df.to_dict(orient="records")
+
     def new_doc(self):
         id = make_doc_id()
         new_document(id)
@@ -570,6 +598,7 @@ def save_doc_settings(self, args: dict):
             args["id"],
             dataView=args["dataView"],
             heatmap=args["heatmap"],
+            clustermap=args["clustermap"],
             distribution=args["distribution"],
         )
         doc = get_document(args["id"])

backend/src/cluster.py

@@ -1,98 +1,127 @@
 import os
 import numpy as np
-from scipy.sparse import csr_matrix
-from scipy.sparse.csgraph import connected_components
 import pandas as pd
 from collections import defaultdict
+import networkx as nx
 
-
-def process_groups(threshold, data, index):
-    # create adjacensy matrix to id which cells are related by the threshold marking as binary with (1) for related or (0) for not meeting the threshold
-    adjacency_matrix = (data >= threshold).astype(int)
-    # create sparse matrix(absent 0s) for memeory efficiancy
-    sparse_matrix = csr_matrix(adjacency_matrix)
-    # identify connected components
-    _, labels = connected_components(
-        csgraph=sparse_matrix, directed=False, return_labels=True
-    )
-    groups_dict = defaultdict(list)
-    for i, label in enumerate(labels):
-        groups_dict[label].append(index[i])
-    groups = {}
-    for indx, clade in enumerate(labels):
-        groups.update({indx: clade})
-    return groups_dict
-
+## switching from scipy to networkx. takes two threshold inputs now
+def process_groups(data, index, threshold_1, threshold_2=0):
+    # check for a threshold 2
+    if threshold_2 is None or threshold_2 == 0:
+        # set all values in the matrix that meet threshold 1 to 1 and all lower or NaN values to 0
+        adjacency_matrix = (~np.isnan(data) & (data >= threshold_1)).astype(int)
+        # Create a graph from the adjacency matrix
+        G1 = nx.from_numpy_array(adjacency_matrix, parallel_edges=False, create_using=None)
+
+        # empty dict to store the groups
+        groups_dict = defaultdict(list)
+
+        # look for connected components in graph G1
+        for i, component in enumerate(nx.connected_components(G1)):
+            for node_idx in component:
+                groups_dict[i].append(index[node_idx])
+
+        return groups_dict
+    else:
+        # Create two adjacency matrices, one for each threshold ~ is the bitwise 'not' operator
+        adjacency_1 = (~np.isnan(data) & (data >= threshold_1)).astype(int)
+        adjacency_2 = (~np.isnan(data) & (data >= threshold_2)).astype(int)
+
+        # convert adjacency matrices to networkx graphs
+        G1 = nx.from_numpy_array(adjacency_1)
+        G2 = nx.from_numpy_array(adjacency_2)
+
+        # find primary clusters with threshold_1
+        groups_dict_1 = defaultdict(list)
+        for i, component in enumerate(nx.connected_components(G1)):
+            for node_idx in component:
+                groups_dict_1[i].append(index[node_idx])
+
+        # find subclusters with threshold_2 
+        groups_dict_2 = defaultdict(list)
+        for i, component in enumerate(nx.connected_components(G2)):
+            for node_idx in component:
+                groups_dict_2[i].append(index[node_idx])
+
+        # return both cluster sets
+        return groups_dict_1, groups_dict_2
 
 def cluster_by_identity(clusters, nodes):
     output = []
     reverse_clusters = {}
-
-    # Create reverse lookup dictionary were values in value list are extracted to key and groups are assigned to value
+    
+    # create lookup table from sequence ID to primary cluster ID
     for group, values in clusters.items():
         for value in values:
             reverse_clusters[value] = group
-
-    # Initialize subgroup counters
+    
+    # initialize counters for subgroups within each primary cluster
     subgroup_counters = {group: 1 for group in clusters.keys()}
-
-    # Iterate through nodes to determine the subgroup_number within each group_number
+    
+    # assign subgroups within each primary cluster
     for _, node_list in nodes.items():
         if node_list:
+            # get first node to determine which primary cluster this belongs to
             first_value = node_list[0]
             if first_value in reverse_clusters:
+                # get primary cluster ID (add 1 for human-readable indexing)
                 group_number = reverse_clusters[first_value] + 1
+                # get next available subgroup number for this primary cluster
                 subgroup_number = subgroup_counters[reverse_clusters[first_value]]
+
+                # process all nodes in this subcluster
                 for value in node_list:
+                    # only include if node belongs to the same primary cluster
                     if value in reverse_clusters:
                         output.append((value, group_number, subgroup_number))
+
+                # increment subgroup counter for this primary cluster
                 subgroup_counters[reverse_clusters[first_value]] += 1
-
+    
     return output
 
 
-def export(matrix_path, threshold_1=79, threshold_2=0):
+def export(matrix_path, threshold_1=79, threshold_2=0, save_csv=True):
     output_dir = os.path.dirname(matrix_path)
     file_name = os.path.basename(matrix_path)
     file_base, _ = os.path.splitext(file_name)
     file_name = file_base.replace("_mat", "")
     output_file = os.path.join(output_dir, file_name + "_cluster.csv")
-
-    # https://stackoverflow.com/a/57824142
-    # SDT1 matrix CSVs do not have padding for columns
+
     with open(matrix_path, "r") as temp_f:
         col_count = [len(l.split(",")) for l in temp_f.readlines()]
         column_names = [i for i in range(0, max(col_count))]
-
+    
     df = pd.read_csv(
         matrix_path, delimiter=",", index_col=0, header=None, names=column_names
     )
-    # extract index
+
     index = df.index.tolist()
-    # convert df to np array
     data = df.to_numpy()
-    # format values
     data = np.round(data, 2)
-    # maintain order of threshold processing
+
     if threshold_2 != 0 and threshold_1 >= threshold_2:
         threshold_1, threshold_2 = threshold_2, threshold_1
-    # handle instances of no threshold_2
+
     if threshold_2 is None or threshold_2 == 0:
-        output = process_groups(threshold_1, data, index)
+        output = process_groups(data, index, threshold_1)
         flattened_output = [
             (item, key + 1) for key, sublist in output.items() for item in sublist
         ]
-        df = pd.DataFrame(flattened_output)
-        df.columns = ["ID", "Group 1 - Theshold: " + str(threshold_1)]
-
+        df_result = pd.DataFrame(flattened_output)
+        df_result.columns = ["SeqID", "Group - Threshold: " + str(threshold_1)]
     else:
-        clusters = process_groups(threshold_1, data, index)
-        nodes = process_groups(threshold_2, data, index)
+        clusters, nodes = process_groups(data, index, threshold_1, threshold_2)
         output = cluster_by_identity(clusters, nodes)
-        df = pd.DataFrame(output)
-        df.columns = [
+        df_result = pd.DataFrame(output)
+        df_result.columns = [
             "ID",
-            "Group 1 - Theshold: " + str(threshold_1),
-            "Group 2 - Theshold: " + str(threshold_2),
+            "Group - Threshold: " + str(threshold_1),
+            "Subgroup - Threshold: " + str(threshold_2),
         ]
-    df.to_csv(output_file, index=False)
+
+    if save_csv:
+        df_result.to_csv(output_file, index=False)
+
+    return df_result
+

backend/src/document_state.py

@@ -23,6 +23,7 @@
         "validation_error_id",
         "compute_stats",
         "heatmap",
+        "clustermap",
         "distribution"
     ],
 )
@@ -34,9 +35,7 @@
     vmin=65,
     cellspace=1,
     annotation=False,
-    annotation_font_size=10,
     annotation_rounding=0,
-    annotation_alpha="0",
     showscale=True,
     titleFont="Sans Serif",
     showTitles=False,
@@ -48,14 +47,30 @@
     cbar_aspect=2.5,
     cbar_pad=10,
     axis_labels=False,
-    axlabel_xrotation=270,
-    axlabel_xfontsize=12,
-    axlabel_yrotation=360,
-    axlabel_yfontsize=12,
+    axlabel_xrotation=0,
+    axlabel_fontsize=12,
+    axlabel_yrotation=0,
     cutoff_1=95,
     cutoff_2=75
 )
 
+default_clustermap_state = dict(
+    threshold_one=85,
+    threshold_two=0,
+    annotation=False,
+    titleFont="Sans Serif",
+    showTitles=False,
+    title="",
+    subtitle="",
+    xtitle="",
+    ytitle="",
+    axis_labels=False,
+    axlabel_xrotation=0,
+    axlabel_fontsize=12,
+    axlabel_yrotation=0,
+    cellspace=1,
+)
+
 visualization_defaults = dict(
   plotTitle="Distribution of Percent Identities",
   lineColor="hsl(9, 100%, 64%)",
@@ -163,6 +178,7 @@ def create_document_state(
     validation_error_id=None,
     compute_stats=None,
     heatmap=default_heatmap_state,
+    clustermap=default_clustermap_state,
     distribution=default_distribution_state
 ):
     if filetype == "application/vnd.sdt" and tempdir_path:
@@ -191,6 +207,7 @@ def create_document_state(
         validation_error_id=validation_error_id,
         compute_stats=compute_stats,
         heatmap=heatmap,
+        clustermap=clustermap,
         distribution=distribution
     )
 
@@ -210,6 +227,7 @@ def save_doc_settings(doc_state: DocState):
         settings = {
             "dataView": doc_state.dataView,
             "heatmap": doc_state.heatmap,
+            "clustermap": doc_state.clustermap,
             "distribution": doc_state.distribution
         }
         json.dump(settings, f, indent=2)

backend/src/export_data.py

@@ -6,7 +6,7 @@
 import cluster
 from constants import data_file_suffixes
 
-image_types = ["heatmap", "histogram", "violin"]
+image_types = ["heatmap", "clustermap", "histogram", "violin"]
 
 def find_source_files(state: DocState, prefix, suffixes):
     with os.scandir(state.tempdir_path) as entries:
@@ -44,7 +44,6 @@ def prepare_export_data(export_path: str, matrix_path: str, doc: DocState, args:
 
     if args["output_cluster"] == True:
         suffixes.append("_cluster")
-        # TODO: it's not intuitive that an export happens in here, move it outside?
         cluster.export(
             matrix_path,
             args["cluster_threshold_one"],
@@ -62,8 +61,8 @@ def prepare_export_data(export_path: str, matrix_path: str, doc: DocState, args:
     )
 
     image_filenames = {
-        img_type: f"{base_filename}_{img_type}.{image_format}"
-        for img_type in image_types
+        image_type: f"{base_filename}_{image_type}.{image_format}"
+        for image_type in image_types
     }
 
     image_destinations = {
@@ -86,9 +85,9 @@ def do_export_data(export_path, image_destinations, image_format, doc, prefix, s
         shutil.copy2(entry.path, temp_destination_path)
         os.replace(temp_destination_path, destination_path)
 
-    for img_type in image_types:
+    for image_type in image_types:
         save_image_from_api(
-            data=args[f"{img_type}_image_data"],
+            data=args[f"{image_type}_image_data"],
             format=image_format,
-            destination=image_destinations[img_type],
+            destination=image_destinations[image_type],
         )

bun.lock

@@ -11,6 +11,7 @@
         "react": "^18.2.0",
         "react-aria-components": "^1.5.0",
         "react-dom": "^18.2.0",
+        "react-icons": "^5.5.0",
         "react-plotly.js": "^2.6.0",
         "tinycolor2": "^1.6.0",
         "zod": "^3.23.8",
@@ -1020,6 +1021,8 @@
 
     "react-dom": ["[email protected]", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.0" }, "peerDependencies": { "react": "^18.2.0" } }, "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g=="],
 
+    "react-icons": ["[email protected]", "", { "peerDependencies": { "react": "*" } }, "sha512-MEFcXdkP3dLo8uumGI5xN3lDFNsRtrjbOEKDLD7yv76v4wpnEq2Lt2qeHaQOr34I/wPN3s3+N08WkQ+CW37Xiw=="],
+
     "react-is": ["[email protected]", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="],
 
     "react-plotly.js": ["[email protected]", "", { "dependencies": { "prop-types": "^15.8.1" }, "peerDependencies": { "plotly.js": ">1.34.0", "react": ">0.13.0" } }, "sha512-g93xcyhAVCSt9kV1svqG1clAEdL6k3U+jjuSzfTV7owaSU9Go6Ph8bl25J+jKfKvIGAEYpe4qj++WHJuc9IaeA=="],