post request for indexed data. logging level in app server

enjalot · Feb 15, 2024 · da67515 · da67515
1 parent 523d4fe
commit da67515
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 58 deletions.
diff --git a/latentscope/server/app.py b/latentscope/server/app.py
@@ -1,6 +1,8 @@
 import re
 import os
+import sys
 import json
+import logging
 import argparse
 import pandas as pd
 import pkg_resources
@@ -12,6 +14,9 @@
 
 app = Flask(__name__)
 
+app.logger.addHandler(logging.StreamHandler(sys.stderr))
+app.logger.setLevel(logging.INFO)
+
 CORS(app)
 
 # DATA_DIR = update_data_dir(args.data_dir)
@@ -81,10 +86,11 @@ def send_file(datasetPath):
 """
 Given a list of indices (passed as a json array), return the rows from the dataset
 """
-@app.route('/api/indexed', methods=['GET'])
+@app.route('/api/indexed', methods=['POST'])
 def indexed():
-    dataset = request.args.get('dataset')
-    indices = json.loads(request.args.get('indices'))
+    data = request.get_json()
+    dataset = data['dataset']
+    indices = data['indices']
     if dataset not in DATAFRAMES:
         df = pd.read_parquet(os.path.join(DATA_DIR, dataset, "input.parquet"))
         DATAFRAMES[dataset] = df
@@ -95,8 +101,6 @@ def indexed():
     rows = df.iloc[indices]
     # send back the rows as json
     return rows.to_json(orient="records")
-
-
 @app.route('/', defaults={'path': ''})
 @app.route('/<path:path>')
 def catch_all(path):

diff --git a/web/src/App.jsx b/web/src/App.jsx
@@ -22,7 +22,8 @@ function App() {
       <Nav />
       <div className="page">
       <Routes>
-        <Route path="/" element={isMobileDevice() ? <Mobile/> : <Home />} />
+        {/* <Route path="/" element={isMobileDevice() ? <Mobile/> : <Home />} /> */}
+        <Route path="/" element={ <Home />} />
         <Route path="/datasets/:dataset/explore/:scope" element={isMobileDevice() ? <Mobile/> : <Explore />} />
         {readonly ? null : <Route path="/datasets/:dataset/setup" element={<Setup/>} />}
         {readonly ? null : <Route path="/datasets/:dataset/setup/:scope" element={<Setup/>} />}

diff --git a/web/src/components/Home.jsx b/web/src/components/Home.jsx
@@ -126,7 +126,9 @@ function Home() {
               {scopes[dataset.id] && scopes[dataset.id].map && scopes[dataset.id]?.map((scope,i) => (
                 <div className="scope-link" key={i} >
                   <Link to={`/datasets/${dataset.id}/explore/${scope.id}`}>{scope.label || scope.id}<br/>
-                  <img src={`${apiUrl}/files/${dataset.id}/clusters/${scope.cluster_id}.png`} />
+                  { scope.ignore_hulls ? 
+                    <img src={`${apiUrl}/files/${dataset.id}/umaps/${scope.umap_id}.png`} />
+                  : <img src={`${apiUrl}/files/${dataset.id}/clusters/${scope.cluster_id}.png`} /> }
                   </Link>
                   <br/>
                   <span className="scope-description">{scope.description}</span>

diff --git a/web/src/components/IndexDataTable.jsx b/web/src/components/IndexDataTable.jsx
@@ -21,26 +21,32 @@ function IndexDataTable({dataset, indices, distances = [], clusterIndices = [],
   const [rows, setRows] = useState([]);
   const hydrateIndices = useCallback((indices) => {
     if(dataset)
-      fetch(`${apiUrl}/indexed?dataset=${dataset.id}&indices=${JSON.stringify(indices)}`)
-        .then(response => response.json())
-        .then(data => {
-          let rows = data.map((row, index) => {
-            let idx = indices[index]
-            let ret = {
-              index: idx,
-              ...row
-            }
-            if(distances && distances.length)
-              ret['distance'] = distances[index]
-            if(clusterIndices && clusterIndices.length && clusterLabels && clusterLabels.length)
-              ret['cluster'] = clusterLabels[clusterIndices[idx].cluster].label // note the idx is the data points index into the original data
-            return ret
-          })
-          // TODO dataset.sort_column
-          // rows.sort((a, b) => b.score - a.score)
-          setRows(rows)
-          // console.log("rows", rows)
+      fetch(`${apiUrl}/indexed`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ dataset: dataset.id, indices: indices }),
+      })
+      .then(response => response.json())
+      .then(data => {
+        let rows = data.map((row, index) => {
+          let idx = indices[index]
+          let ret = {
+            index: idx,
+            ...row
+          }
+          if(distances && distances.length)
+            ret['distance'] = distances[index]
+          if(clusterIndices && clusterIndices.length && clusterLabels && clusterLabels.length)
+            ret['cluster'] = clusterLabels[clusterIndices[idx].cluster].label // note the idx is the data points index into the original data
+          return ret
         })
+        // TODO dataset.sort_column
+        // rows.sort((a, b) => b.score - a.score)
+        setRows(rows)
+        // console.log("rows", rows)
+      })
   }, [dataset, distances, clusterIndices, clusterLabels])
 
   useEffect(() => {

diff --git a/web/src/components/Setup/Cluster.jsx b/web/src/components/Setup/Cluster.jsx
@@ -79,7 +79,7 @@ function Cluster({ dataset, cluster, umap, onNew, onChange}) {
       <form onSubmit={(e) => handleNewCluster(e, umap)}>
         <label>
           Samples:
-          <input type="number" name="samples" defaultValue="5" disabled={!!clusterJob || !umap}/>
+          <input type="number" name="samples" defaultValue={dataset.length < 1000 ? 5 : 25} disabled={!!clusterJob || !umap}/>
         </label>
         <label>
           Min Samples:

diff --git a/web/src/components/Setup/ClusterLabels.jsx b/web/src/components/Setup/ClusterLabels.jsx
@@ -26,6 +26,7 @@ ClusterLabels.propTypes = {
 function ClusterLabels({ dataset, cluster, selectedLabelId, onChange, onLabels, onLabelIds, onHoverLabel, onClickLabel}) {
   const [clusterLabelsJob, setClusterLabelsJob] = useState(null);
   const { startJob: startClusterLabelsJob } = useStartJobPolling(dataset, setClusterLabelsJob, `${apiUrl}/jobs/cluster_label`);
+  const { startJob: rerunClusterLabelsJob } = useStartJobPolling(dataset, setClusterLabelsJob, `${apiUrl}/jobs/rerun`);
 
   const [chatModels, setChatModels] = useState([]);
   useEffect(() => {
@@ -109,6 +110,10 @@ function ClusterLabels({ dataset, cluster, selectedLabelId, onChange, onLabels,
     startClusterLabelsJob({chat_id: model, cluster_id: cluster_id, text_column, context})
   }, [dataset, cluster, startClusterLabelsJob])
 
+  function handleRerun(job) {
+    rerunJob({job_id: job?.id});
+  }
+
   return (
     <div className="dataset--setup-cluster-labels-content">
       <div className="dataset--slides-new">
@@ -127,7 +132,7 @@ function ClusterLabels({ dataset, cluster, selectedLabelId, onChange, onLabels,
           <button type="submit" disabled={!!clusterLabelsJob || !cluster}>Auto Label</button>
         </form>
 
-        <JobProgress job={clusterLabelsJob} clearJob={()=>setClusterLabelsJob(null)} killJob={setClusterLabelsJob} />
+        <JobProgress job={clusterLabelsJob} clearJob={()=>setClusterLabelsJob(null)} killJob={setClusterLabelsJob} rerunJob={handleRerun} />
 
       </div>
       {cluster ? <div className="dataset--setup-cluster-labels-list">

diff --git a/web/src/pages/Explore.jsx b/web/src/pages/Explore.jsx
@@ -140,18 +140,24 @@ function Explore() {
 
 
   const hydrateIndices = useCallback((indices, setter, distances = []) => {
-    fetch(`${apiUrl}/indexed?dataset=${datasetId}&indices=${JSON.stringify(indices)}`)
-      .then(response => response.json())
-      .then(data => {
-        if(!dataset) return;
-        let rows = data.map((row, index) => {
-          return {
-            index: indices[index],
-            ...row
-          }
-        })
-        setter(rows)
+    fetch(`${apiUrl}/indexed`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ dataset: datasetId, indices: indices }),
+    })
+    .then(response => response.json())
+    .then(data => {
+      if(!dataset) return;
+      let rows = data.map((row, index) => {
+        return {
+          index: indices[index],
+          ...row
+        }
       })
+      setter(rows)
+    })
   }, [dataset, datasetId])
 
 

diff --git a/web/src/pages/Job.jsx b/web/src/pages/Job.jsx
@@ -54,13 +54,6 @@ function Job() {
   }
 
   function handleRerun(job) {
-    // fetch(`${apiUrl}/jobs/rerun?dataset=${datasetId}&job_id=${job.id}`)
-    //   .then(response => response.json())
-    //   .then(data => {
-    //     console.log("rerun job", data);
-    //     setJob(data)
-    //   })
-    //   .catch(console.error);
     rerunJob({job_id: job?.id});
   }
 
@@ -70,7 +63,7 @@ function Job() {
       <h3>{dataset?.id} job {job?.id}</h3>
       {job ? <div>
           <span className="job-status" style={{fontWeight:"bold", padding: "5px"}}>{job.status}</span>
-          { job.status == "running" ? <button onClick={() => {handleKill(job)}}>💀 Kill</button> : null}
+          {/* { job.status == "running" ? <button onClick={() => {handleKill(job)}}>💀 Kill</button> : null} */}
           {/* { job.status == "error" || job.status == "dead" ? <button onClick={() => {handleRerun(job)}}>🔁 Rerun</button> : null} */}
           <JobProgress job={job} rerunJob={handleRerun} killJob={handleKill} />
       </div> : null }

diff --git a/web/src/pages/Setup.jsx b/web/src/pages/Setup.jsx
@@ -173,18 +173,24 @@ function Setup() {
 
 
   const hydrateIndices = useCallback((indices, setter, distances = []) => {
-    fetch(`${apiUrl}/indexed?dataset=${datasetId}&indices=${JSON.stringify(indices)}`)
-      .then(response => response.json())
-      .then(data => {
-        if(!dataset) return;
-        let rows = data.map((row, index) => {
-          return {
-            index: indices[index],
-            ...row
-          }
-        })
-        setter(rows)
+    fetch(`${apiUrl}/indexed`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ dataset: dataset.id, indices: indices }),
+    })
+    .then(response => response.json())
+    .then(data => {
+      if(!dataset) return;
+      let rows = data.map((row, index) => {
+        return {
+          index: indices[index],
+          ...row
+        }
       })
+      setter(rows)
+    })
   }, [dataset, datasetId])
 
   // Hover via scatterplot or tables