Merge pull request #33 from pgleeson/main

Improved resources linking
openworm · Dec 4, 2024 · 9065704 · 9065704
2 parents b5e5985 + 047d178
commit 9065704
Show file tree

Hide file tree

Showing 72 changed files with 10,003 additions and 2,656 deletions.
diff --git a/.gitignore b/.gitignore
@@ -684,3 +684,4 @@ __pycache__
 /docs/Brittin*_data*.md
 /docs/assets/Brittin2021*
 /docs/*_Brittin2021_data*.md
+/clean.sh
diff --git a/cect/CellInfo.py b/cect/CellInfo.py
@@ -11,6 +11,7 @@
 from cect.Cells import get_primary_classification
 from cect.Cells import get_standard_color
 from cect.Cells import is_male_specific_cell
+from cect.Cells import is_bilateral_left
 
 
 from cect import print_
@@ -251,6 +252,7 @@ def generate_cell_info_pages(connectomes):
         reference_mono = "Bentley2016_MA"
         reference_pep = "RipollSanchezShortRange"
         reference_func = "Randi2023"
+        reference_cont = "Brittin2021"
         max_conn_cells = 5
         conns_from_cs = "???"
         conns_to_cs = "???"
@@ -260,6 +262,8 @@ def generate_cell_info_pages(connectomes):
         conns_to_pep = "???"
         conns_from_func = "???"
         conns_to_func = "???"
+        conns_from_cont = "???"
+        conns_to_cont = "???"
         conns_gj = "???"
 
         tables_md = ""
@@ -298,6 +302,8 @@ def generate_cell_info_pages(connectomes):
                         conns_to_pep = _get_top_list(conns, max_conn_cells)
                     if cds_name == reference_func:
                         conns_to_func = _get_top_list(conns, max_conn_cells)
+                    if cds_name == reference_cont:
+                        conns_to_cont = _get_top_list(conns, max_conn_cells)
 
                     for c in conns:
                         cc = get_cell_internal_link(
@@ -345,6 +351,8 @@ def generate_cell_info_pages(connectomes):
                             conns_from_pep = _get_top_list(conns, max_conn_cells)
                         if cds_name == reference_func:
                             conns_from_func = _get_top_list(conns, max_conn_cells)
+                        if cds_name == reference_cont:
+                            conns_from_cont = _get_top_list(conns, max_conn_cells)
 
                         for c in conns:
                             cc = get_cell_internal_link(
@@ -370,22 +378,24 @@ def generate_cell_info_pages(connectomes):
 
 ### Summary of connections
 
-<p class="subtext">Top {max_conn_cells} connections of specified types to/from this cell (based on {get_dataset_link(reference_cs)}, {get_dataset_link(reference_mono)}, {get_dataset_link(reference_pep)} & {get_dataset_link(reference_func)})</p>
+<p class="subtext">Top {max_conn_cells} connections of specified types to/from this cell (based on {get_dataset_link(reference_cs)}, {get_dataset_link(reference_mono)}, {get_dataset_link(reference_pep)}, {get_dataset_link(reference_func)} & {get_dataset_link(reference_cont)})</p>
 
 <table style="width:700px">
 <tr>
     <td><b><a href="#chemical-synaptic-connections-to-{cell.lower()}">Chemical</a></b></td>
     <td style="width:40%">{conns_to_cs}</td>
     <td style="width:5%" style="vertical-align:bottom;text-align:center;">\u2198</td>
-    <td rowspan="4" style="vertical-align:middle;text-align:center;"><b>{cell_link}</b></td>
+    <td rowspan="5" style="vertical-align:middle;text-align:center;"><b>{cell_link}</b></td>
     <td style="width:5%" style="vertical-align:bottom;text-align:center;">\u2197</td>
     <td style="width:40%">{conns_from_cs}</td>
 </tr><tr>
     <td><b><a href="#monoaminergic-connections-to-{cell.lower()}">Monoaminergic</a></b></td><td>{conns_to_mono}</td><td align="middle">→</td><td align="middle">→</td><td>{conns_from_mono}</td>
 </tr><tr>
     <td><b><a href="#peptidergic-connections-to-{cell.lower()}">Peptidergic</a></b></td>  <td>{conns_to_pep}</td><td align="middle">→</td><td align="middle">→</td><td>{conns_from_pep}</td>
 </tr><tr>
-    <td><b><a href="#functional-connections-to-{cell.lower()}">Functional</a></b></td>   <td>{conns_to_func}</td><td align="middle">\u2197</td><td align="middle">\u2198</td><td>{conns_from_func}</td>
+    <td><b><a href="#functional-connections-to-{cell.lower()}">Functional</a></b></td>   <td>{conns_to_func}</td><td align="middle">→</td><td align="middle">→</td><td>{conns_from_func}</td>
+</tr><tr>
+    <td><b><a href="#membrane-contacts-to-{cell.lower()}">Contactome</a></b></td>   <td>{conns_to_cont}</td><td align="middle">\u2197</td><td align="middle">\u2198</td><td>{conns_from_cont}</td>
 </tr><tr>
     <td>&nbsp;</td> <td colspan="5" align="middle">\u2195</td> 
 </tr><tr>
@@ -412,7 +422,142 @@ def generate_cell_info_pages(connectomes):
 if __name__ == "__main__":
     import sys
 
-    if "-i" in sys.argv:
+    if "-pca" in sys.argv:
+        # from cect.Cells import PREFERRED_HERM_NEURON_NAMES
+
+        from cect.White_whole import get_instance
+
+        cds_src = get_instance()
+        """
+        from cect.RipollSanchezMidRangeReader import get_instance
+        cds_src = get_instance()"""
+        """ 
+        from cect.Cook2019HermReader import get_instance
+        from cect.WormNeuroAtlasFuncReader import get_instance
+        from cect.TestDataReader import get_instance
+
+        from cect.ConnectomeView import PHARYNX_VIEW as view
+        from cect.ConnectomeView import RAW_VIEW as view
+        """
+        from cect.ConnectomeView import NEURONS_VIEW as view
+
+        cds_src = get_instance()
+        cds = cds_src.get_connectome_view(view)
+        """
+        for cell in ['I3']:
+            print(cds.nodes)
+            print(cds.connections.keys())
+            index = cds.nodes.index(cell)
+            print('Conns from %s (index: %i): %s'%(cell,index,cds.get_connections_from(cell, syntype)))
+            matrix = cds.connections[syntype]
+            print(matrix[index])
+            print('Conns to %s (index: %i): %s'%(cell,index,cds.get_connections_to(cell, syntype)))
+            print(matrix.T[index])"""
+
+        data = {}
+
+        for syntype in cds.connections.keys():
+            matrix = cds.connections[syntype]
+            if matrix.max() != matrix.min():
+                print("Adding data matrix of type: %s" % syntype)
+
+                for cell in cds.nodes:
+                    if cell not in data:
+                        data[cell] = []
+                    index = cds.nodes.index(cell)
+                    for v in matrix[index] + matrix.T[index]:
+                        scale = False
+                        data[cell].append((1 if v != 0 else 0) if scale else v)
+
+        df = pd.DataFrame(data)
+
+        print("Data being used: %s\n%s" % (df.shape, df))
+
+        from sklearn.decomposition import PCA
+
+        pca = PCA(n_components=3)
+        pcs = pca.fit_transform(df.T)
+
+        # print(pca.components_)
+        print(pcs)
+
+        xs = []
+        ys = []
+        zs = []
+        texts = []
+        colors = []
+        from cect.Cells import get_standard_color
+
+        positions = {}
+        for i in range(len(cds.nodes)):
+            a = pcs[i]
+            cell = cds.nodes[i]
+            print("%i) Plotting %s at %s" % (i, cell, a))
+            xs.append(a[0])
+            ys.append(a[1])
+            zs.append(a[2])
+            texts.append(cell)
+            colors.append(get_standard_color(cell))
+            positions[cell] = (a[0], a[1], a[2])
+
+        add_text = False
+
+        import plotly.graph_objects as go
+
+        scat = go.Scatter3d(
+            x=xs,
+            y=ys,
+            z=zs,
+            text=texts,
+            mode="markers+text" if add_text else "markers",
+            marker=dict(
+                size=10,
+                color=colors,
+                line_width=1,
+            ),
+        )
+
+        scat.marker.color = colors
+
+        edge_traces = []
+
+        for cell in cds.nodes:
+            if is_bilateral_left(cell):
+                right = cell[:-1] + "R"
+                a = positions[cell]
+                if right in positions:
+                    b = positions[right]
+                    print("Connecting %s->%s: %s->%s" % (cell, right, a, b))
+                    # Add edges to the figure
+                    edge_trace = go.Scatter3d(
+                        x=[a[0], b[0]],
+                        y=[a[1], b[1]],
+                        z=[a[2], b[2]],
+                        mode="lines",
+                        # marker=dict(symbol="arrow",size=weight * 3,angleref="previous",     ),
+                        line=dict(
+                            color=get_standard_color(cell),
+                            width=5,
+                        ),
+                        hoverinfo="none",
+                    )
+                    edge_traces.append(edge_trace)
+
+        fig = go.Figure(
+            data=[scat] + edge_traces,
+            layout=go.Layout(
+                showlegend=False,
+                hovermode="closest",
+                margin=dict(b=20, l=5, r=5, t=40),
+                xaxis=dict(showgrid=False, zeroline=False),
+                yaxis=dict(showgrid=False, zeroline=False),
+                width=1600,
+                height=800,
+            ),
+        )
+        fig.show()
+
+    elif "-i" in sys.argv:
         from cect.Cells import ALL_PREFERRED_NEURON_NAMES
         from cect.Cells import PREFERRED_MUSCLE_NAMES
         from cect.Cells import ALL_NON_NEURON_MUSCLE_CELLS

diff --git a/cect/ConnectomeDataset.py b/cect/ConnectomeDataset.py
@@ -1062,12 +1062,13 @@ def connection_number_plot(self, synclass):  # Todo: get better name
     print(pprint.pprint(nx.node_link_data(G)))
 
     # from cect.ConnectomeView import NEURONS_VIEW as view
-    from cect.ConnectomeView import RAW_VIEW as view
+    # from cect.ConnectomeView import RAW_VIEW as view
     # from cect.ConnectomeView import ESCAPE_VIEW as view
 
     # from cect.ConnectomeView import SOCIAL_VIEW as view
     # from cect.ConnectomeView import SOCIAL_VIEW as view
     # from cect.ConnectomeView import COOK_FIG3_VIEW as view
+    from cect.ConnectomeView import PEP_HUBS_VIEW as view
 
     # from cect.White_whole import get_instance
     from cect.BrittinDataReader import get_instance

diff --git a/cect/ConnectomeView.py b/cect/ConnectomeView.py
@@ -34,6 +34,7 @@
 from cect.Cells import get_standard_color
 
 from cect.ConnectomeReader import DEFAULT_COLORMAP
+from cect.RipollSanchezDataReader import load_hub_info
 
 import copy
 
@@ -291,6 +292,44 @@ def get_index_of_cell(self, cell):
 
     ESCAPE_VIEW.node_sets.append(ns)
 
+PEP_HUBS_VIEW = View(
+    "PeptidergicHubs",
+    "Peptidergic Hubs",
+    "Peptidergic hubs as outlined in in [Ripoll-Sánchez et al. 2023](../RipollSanchez_2023.md), Fig 7E",
+    [],
+    EXC_INH_GJ_FUNC_CONT_SYN_CLASSES,
+)
+
+len_scale = 1.5
+
+pep_positions = {
+    "Periphery": [(0, 0), "Gainsboro"],
+    "Motor core": [(-1 * len_scale, len_scale), "DarkSeaGreen"],
+    "Sensory core": [(1 * len_scale, len_scale), "plum"],
+    "Hubs": [(0, 2 * len_scale), "burlywood"],
+}
+
+
+clusters = load_hub_info()
+
+for cluster, info in pep_positions.items():
+    pos = info[0]
+    color = info[1]
+
+    shape = "circle"
+
+    ns = NodeSet(
+        cluster,
+        clusters[cluster],
+        color=color,
+        shape=shape,
+        position=pos,
+        size=len_scale * 80,
+    )
+
+    PEP_HUBS_VIEW.node_sets.append(ns)
+
+
 SOCIAL_VIEW = View(
     "Social",
     "Social Network",
@@ -437,6 +476,7 @@ def get_index_of_cell(self, cell):
     SOCIAL_VIEW,
     ESCAPE_VIEW,
     COOK_FIG3_VIEW,
+    PEP_HUBS_VIEW,
 ]
 
 

diff --git a/cect/RipollSanchezDataReader.py b/cect/RipollSanchezDataReader.py
@@ -21,6 +21,12 @@
 import csv
 
 
+def standardise_cell_name(cell):
+    for pf in ["DA", "DB", "AS", "VA", "VB", "VC", "VD", "DD"]:
+        cell = cell.replace("%s0" % pf, pf)
+    return cell
+
+
 class RipollSanchezDataReader(ConnectomeDataset):
     verbose = False
 
@@ -47,15 +53,10 @@ def read_data(self):
 
         print_("Opened the CSV file: " + self.filename)
 
-        def fix_cell(cell):
-            for pf in ["DA", "DB", "AS", "VA", "VB", "VC", "VD", "DD"]:
-                cell = cell.replace("%s0" % pf, pf)
-            return cell
-
         for i in range(1, len(data)):
-            pre_cell = fix_cell(data[0][i])
+            pre_cell = standardise_cell_name(data[0][i])
             for j in range(1, len(data)):
-                post_cell = fix_cell(data[0][j])
+                post_cell = standardise_cell_name(data[0][j])
                 num = int(data[i][j])
                 if num > 0:
                     if self.verbose:
@@ -91,6 +92,36 @@ def get_instance():
 read_muscle_data = my_instance.read_muscle_data
 
 
+def load_hub_info():
+    from openpyxl import load_workbook
+
+    neuron_info_file = (
+        os.path.dirname(os.path.abspath(__file__))
+        + "/data/1-s2.0-S0896627323007560-mmc7.xlsx"
+    )
+
+    wb = load_workbook(neuron_info_file)
+    sheet = wb.worksheets[0]
+    print_("Opened the Excel file: " + neuron_info_file)
+
+    clusters = {}
+
+    for row in sheet.iter_rows(
+        min_row=3, values_only=True
+    ):  # Assuming data starts from the second row
+        cell = standardise_cell_name(str(row[0]))
+        # type_ = str(row[2])
+        cluster = str(row[12])
+
+        # print(f"Cell {cell} is of type {type_} and in cluster {cluster}")
+
+        if cluster not in clusters:
+            clusters[cluster] = []
+        clusters[cluster].append(cell)
+
+    return clusters
+
+
 def main():
     tdr_instance = RipollSanchezDataReader("short_range_model")
 
@@ -131,4 +162,5 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    # main()
+    load_hub_info()
diff --git a/cect/data/1-s2.0-S0896627323007560-mmc7.xlsx b/cect/data/1-s2.0-S0896627323007560-mmc7.xlsx
diff --git a/docs/AddDataset.md b/docs/AddDataset.md
@@ -0,0 +1,22 @@
+# Add a dataset
+
+We are very keen to incorporate other published datasets on worm neuronal connectivity into this package and add them to this website. You can either A) add it yourself to the code, or B) tell us about it and we can add it for you. 
+
+## A) Add it yourself
+
+1) [Fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo) the repository https://github.com/openworm/ConnectomeToolbox.
+
+2) Add the source file of your dataset (usually the adjacency matrices of connection weights) into the [data folder](https://github.com/openworm/ConnectomeToolbox/tree/main/cect/data).
+
+3) Create a Reader for the dataset, which converts it to our internal format. See examples for loading structured datasets in [CSV format](https://github.com/pgleeson/ConnectomeToolbox/blob/main/cect/Cook2020DataReader.py), [XLSX format](https://github.com/openworm/ConnectomeToolbox/blob/main/cect/Cook2019DataReader.py) or [XLS format](https://github.com/openworm/ConnectomeToolbox/blob/6847151db6a5dc9bc3fea1c5a40d01d1a6b024fa/cect/SpreadsheetDataReader.py). 
+
+4) Add the new data reader to the list of readers in [Comparison.py](https://github.com/openworm/ConnectomeToolbox/blob/6847151db6a5dc9bc3fea1c5a40d01d1a6b024fa/cect/Comparison.py) (at 2 locations - see how Cook2020 is handled there).
+
+5) Run [./test.sh](https://github.com/openworm/ConnectomeToolbox/blob/main/test.sh) in your local directory (or `./test.sh -q` for a quicker test) to install the latest version of the code and attempt to regenerate the website locally with your changes. 
+
+6) Commit the code and [open a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests) to https://github.com/openworm/ConnectomeToolbox with your changes. Note, enabling [GitHub Actions](https://github.com/features/actions) on the repository containing your fork will cause a number of [tests](https://github.com/openworm/ConnectomeToolbox/tree/main/.github/workflows) to be run automatically when you commit, which should pass before you open the pull request. 
+
+## B) Tell us about the dataset
+
+Please contact *p.gleeson AT ucl.ac.uk* with details of the missing resource (or just a link to the relevant publication) and we can discuss adding this to our codebase.
+