diff --git a/.gitmodules b/.gitmodules index bc0c2d7d..206163e6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,6 +7,3 @@ [submodule "pylib"] path = pylib url = git@github.com:Ganymede-Bio/pylib.git -[submodule "api-server"] - path = api-server - url = https://github.com/Ganymede-Bio/api-server.git diff --git a/.wordlist.txt b/.wordlist.txt index 558bbedc..f1bdec82 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -284,4 +284,8 @@ VPN EOL PowerShell netcat -MockGanymedeContext \ No newline at end of file +Webhook +webhook +AIA +CDF +MockGanymedeContext diff --git a/api-server b/api-server deleted file mode 160000 index e6a5f3ce..00000000 --- a/api-server +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e6a5f3ce3ee0cd12681dca50be093d797a15900a diff --git a/core-dev-operators b/core-dev-operators index e87fbb07..71472fe8 160000 --- a/core-dev-operators +++ b/core-dev-operators @@ -1 +1 @@ -Subproject commit e87fbb07a251f5cc3b3f8cf320c50a708dd74b04 +Subproject commit 71472fe8576ef0b1af5d9af202eb84fc195a46a8 diff --git a/core-operators b/core-operators index e87fbb07..71472fe8 160000 --- a/core-operators +++ b/core-operators @@ -1 +1 @@ -Subproject commit e87fbb07a251f5cc3b3f8cf320c50a708dd74b04 +Subproject commit 71472fe8576ef0b1af5d9af202eb84fc195a46a8 diff --git a/docs/nodes/Analysis/Branch_Python.md b/docs/nodes/Analysis/Branch_Python.md index 45242ba2..d92278b0 100644 --- a/docs/nodes/Analysis/Branch_Python.md +++ b/docs/nodes/Analysis/Branch_Python.md @@ -53,4 +53,4 @@ is written to the output bucket. ### Notes Files can also be retrieved and processed using the list_files and retrieve_files functions. -Documentation on these functions can be found at https://docs.ganymede.bio/sdk/GanymedeClass \ No newline at end of file +Documentation on these functions can be found at https://docs.ganymede.bio/sdk/GanymedeClass diff --git a/docs/nodes/Analysis/Python.md b/docs/nodes/Analysis/Python.md index af0cfeee..f20e904e 100644 --- a/docs/nodes/Analysis/Python.md +++ b/docs/nodes/Analysis/Python.md @@ -47,4 +47,4 @@ is written to the output bucket. ### Notes Files can also be retrieved and processed using the list_files and retrieve_files functions. -Documentation on these functions can be found at https://docs.ganymede.bio/sdk/GanymedeClass \ No newline at end of file +Documentation on these functions can be found at https://docs.ganymede.bio/sdk/GanymedeClass diff --git a/docs/nodes/Analysis/Transform_SQL.md b/docs/nodes/Analysis/Transform_SQL.md index 5db89be6..ed02bdef 100644 --- a/docs/nodes/Analysis/Transform_SQL.md +++ b/docs/nodes/Analysis/Transform_SQL.md @@ -17,4 +17,4 @@ Runs SQL query on data lake ### Notes -In the execute function, returning NodeReturn(tables_to_upload=\{'sql_result': df\}) would render the DataFrame df in the Flow Editor if Table Head visualization is enabled. \ No newline at end of file +In the execute function, returning NodeReturn(tables_to_upload=\{'sql_result': df\}) would render the DataFrame df in the Flow Editor if Table Head visualization is enabled. diff --git a/docs/nodes/Analysis/Trigger_Python.md b/docs/nodes/Analysis/Trigger_Python.md index f98ad0fc..9c452ebd 100644 --- a/docs/nodes/Analysis/Trigger_Python.md +++ b/docs/nodes/Analysis/Trigger_Python.md @@ -33,4 +33,4 @@ Processes data, then triggers downstream flow ### Notes The ID for the triggered flow can be found by navigating to the flow in the Ganymede UI -and clicking on Manage. The ID is an immutable string found on the Configuration panel. \ No newline at end of file +and clicking on Manage. The ID is an immutable string found on the Configuration panel. diff --git a/docs/nodes/App/APINode.md b/docs/nodes/App/APINode.md index fe63dbc4..988d7716 100644 --- a/docs/nodes/App/APINode.md +++ b/docs/nodes/App/APINode.md @@ -17,14 +17,15 @@ Demonstrates access of generic API ### Parameters - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` Object containing data to store in data lake and/or file storage. NodeReturn object takes 2 parameters: - - **tables_to_upload**: `dict[str, pd.DataFrame]` - keys are table names, values are pandas DataFrames to upload - - **files_to_upload**: `dict[str, bytes]` - keys are file names, values are file data to upload + +- **tables_to_upload**: `dict[str, pd.DataFrame]` + keys are table names, values are pandas DataFrames to upload +- **files_to_upload**: `dict[str, bytes]` + keys are file names, values are file data to upload diff --git a/docs/nodes/App/AirtableExport.md b/docs/nodes/App/AirtableExport.md index a8727b07..1050fe1b 100644 --- a/docs/nodes/App/AirtableExport.md +++ b/docs/nodes/App/AirtableExport.md @@ -22,6 +22,7 @@ Reads table(s) from Ganymede data lake and uploads to Airtable. ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **airtable_api_key**: API key from Airtable Secrets can be configured by clicking on your username in the upper-right hand of the Ganymede @@ -35,9 +36,9 @@ Process tabular data from user-defined SQL query, writing results to Airtable ### Parameters - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame]` - - Table(s) or list of tables retrieved from user-defined SQL query + - Table(s) or list of tables retrieved from user-defined SQL query - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns diff --git a/docs/nodes/App/AirtableImport.md b/docs/nodes/App/AirtableImport.md index 2462f8a9..4ab6a69c 100644 --- a/docs/nodes/App/AirtableImport.md +++ b/docs/nodes/App/AirtableImport.md @@ -22,6 +22,7 @@ Retrieves Airtable table and uploads data to Ganymede data lake. ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **airtable_api_key**: API key from Airtable Secrets can be configured by clicking on your username in the upper-right hand of the Ganymede diff --git a/docs/nodes/App/Airtable_Export.md b/docs/nodes/App/Airtable_Export.md index 3eff2d08..f70d23d5 100644 --- a/docs/nodes/App/Airtable_Export.md +++ b/docs/nodes/App/Airtable_Export.md @@ -22,6 +22,7 @@ Reads table(s) from Ganymede data lake and uploads to Airtable. ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **airtable_api_key**: API key from Airtable Secrets can be configured by clicking on your username in the upper-right hand of the Ganymede @@ -35,11 +36,11 @@ Process tabular data from user-defined SQL query, writing results to Airtable ### Parameters - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame]` - - Table(s) or list of tables retrieved from user-defined SQL query + - Table(s) or list of tables retrieved from user-defined SQL query - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/App/Airtable_Import.md b/docs/nodes/App/Airtable_Import.md index 403a08cc..c01b50b6 100644 --- a/docs/nodes/App/Airtable_Import.md +++ b/docs/nodes/App/Airtable_Import.md @@ -22,6 +22,7 @@ Retrieves Airtable table and uploads data to Ganymede data lake. ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **airtable_api_key**: API key from Airtable Secrets can be configured by clicking on your username in the upper-right hand of the Ganymede diff --git a/docs/nodes/App/Azure_Query.md b/docs/nodes/App/Azure_Query.md index eb08ed54..a6d5ca48 100644 --- a/docs/nodes/App/Azure_Query.md +++ b/docs/nodes/App/Azure_Query.md @@ -22,6 +22,7 @@ Query Azure SQL database, process data in Python, and upload results to Ganymede ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **azure_client_id**: App client ID - **azure_object_id**: App object ID - **azure_tenant_id**: App tenant ID @@ -61,4 +62,4 @@ Process tabular data from user-defined SQL query, writing results back to data l ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/App/Azure_Read.md b/docs/nodes/App/Azure_Read.md index ea846eec..a76881f4 100644 --- a/docs/nodes/App/Azure_Read.md +++ b/docs/nodes/App/Azure_Read.md @@ -24,6 +24,7 @@ Read files from Azure Blob Storage into data lake. ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **azure_client_id**: App client ID - **azure_tenant_id**: App tenant ID - **azure_subscription_id**: Azure subscription ID @@ -39,5 +40,5 @@ An example configuration is shown below: - **azure_storage_uri**: https://abc123.blob.core.windows.net - **src_azure_container_name**: abc123 -- **src_azure_blob_name**: "run_results/filename.parquet" -- **dest_blob_name**: filename.parquet \ No newline at end of file +- **src_azure_blob_name**: `"run_results/filename.parquet"` +- **dest_blob_name**: filename.parquet diff --git a/docs/nodes/App/Azure_Read_Multi.md b/docs/nodes/App/Azure_Read_Multi.md index 0973dba4..529c1164 100644 --- a/docs/nodes/App/Azure_Read_Multi.md +++ b/docs/nodes/App/Azure_Read_Multi.md @@ -20,6 +20,7 @@ Read all files from a Azure Blob Storage container into Ganymede. ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **azure_client_id**: App client ID - **azure_tenant_id**: App tenant ID - **azure_subscription_id**: Azure subscription ID @@ -34,4 +35,4 @@ assistance, please don't hesitate to reach out to Ganymede. An example configuration is shown below: - **azure_storage_uri**: https://abc123.blob.core.windows.net -- **src_azure_container_name**: abc123 \ No newline at end of file +- **src_azure_container_name**: abc123 diff --git a/docs/nodes/App/Azure_Write.md b/docs/nodes/App/Azure_Write.md index 0ddd9586..c99103e8 100644 --- a/docs/nodes/App/Azure_Write.md +++ b/docs/nodes/App/Azure_Write.md @@ -24,6 +24,7 @@ Write files from data lake to Azure Blob Storage. ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **azure_client_id**: App client ID - **azure_tenant_id**: App tenant ID - **azure_subscription_id**: Azure subscription ID @@ -40,4 +41,4 @@ An example configuration is shown below: - **azure_storage_uri**: https://abc123.blob.core.windows.net - **src_object_name**: filename.parquet - **dest_azure_container_Name**: abc123 -- **dest_azure_blob_name**: "\{\{dag.dag_id\}\}/filename.parquet" \ No newline at end of file +- **dest_azure_blob_name**: \{dag.dag_id\}/filename.parquet diff --git a/docs/nodes/App/Benchling_Api.md b/docs/nodes/App/Benchling_Api.md index 0efc8a29..e58ddf7c 100644 --- a/docs/nodes/App/Benchling_Api.md +++ b/docs/nodes/App/Benchling_Api.md @@ -25,6 +25,7 @@ in the relevant Benchling tenant. Prior to usage, the secrets below must be configured in your Ganymede environment. More information on acquiring these credentials can be found [here](https://docs.benchling.com/docs/authentication). + - **benchling_url**: URL for Benchling tenant; has form https://\.benchling.com - **benchling_app_client_id**: Client_ID; has form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - **benchling_client_secret**: Client_Secret; has form cs_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx @@ -43,9 +44,9 @@ conjunction with the Input_Param node. ### Parameters - **benchling_context** : `BenchlingContext` - - Benchling context variable, which stores Benchling connection information + - Benchling context variable, which stores Benchling connection information - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -54,4 +55,4 @@ conjunction with the Input_Param node. ### Notes -Documentation for the Benchling API can be found here: https://benchling.com/api/reference \ No newline at end of file +Documentation for the Benchling API can be found here: https://benchling.com/api/reference diff --git a/docs/nodes/App/Benchling_Event.md b/docs/nodes/App/Benchling_Event.md index fd69091d..478ae9d5 100644 --- a/docs/nodes/App/Benchling_Event.md +++ b/docs/nodes/App/Benchling_Event.md @@ -33,6 +33,7 @@ in the relevant Benchling tenant. Prior to usage, the secrets below must be configured in your Ganymede environment. More information on acquiring these credentials can be found [here](https://docs.benchling.com/docs/authentication). + - **benchling_url**: URL for Benchling tenant; has form https://\.benchling.com - **benchling_app_client_id**: Client_ID; has form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - **benchling_client_secret**: Client_Secret; has form cs_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx @@ -55,13 +56,13 @@ Calls Benchling to get data for flow to trigger ### Parameters - **object_id** : `str` - - ID used in Benchling API (e.g. - entity schema ID or notebook entry ID) + - ID used in Benchling API (e.g. - entity schema ID or notebook entry ID) - **event_type** : `str` - - Type of Benchling event (e.g. - v2.entity.registered) + - Type of Benchling event (e.g. - v2.entity.registered) - **benchling_context** : `BenchlingContext` - - Benchling context variable, which stores Benchling connection information + - Benchling context variable, which stores Benchling connection information - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -75,6 +76,7 @@ Documentation for the Benchling API can be found here: https://benchling.com/api Reference for Benchling events can be found here: https://docs.benchling.com/docs/events-reference benchling_context is an object of class BenchlingContext, consisting of + - **conn**: Benchling connection - **run_tag**: Benchling ID associated with run tag -- **display_tag**: Displayed value of Benchling run tag \ No newline at end of file +- **display_tag**: Displayed value of Benchling run tag diff --git a/docs/nodes/App/Benchling_Read.md b/docs/nodes/App/Benchling_Read.md index 8ad75d5f..16981266 100644 --- a/docs/nodes/App/Benchling_Read.md +++ b/docs/nodes/App/Benchling_Read.md @@ -32,6 +32,7 @@ in the relevant Benchling tenant. Prior to usage, the secrets below must be configured in your Ganymede environment. More information on acquiring these credentials can be found [here](https://docs.benchling.com/docs/authentication). + - **benchling_url**: URL for Benchling tenant; has form https://\.benchling.com - **benchling_app_client_id**: Client_ID; has form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - **benchling_client_secret**: Client_Secret; has form cs_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx @@ -50,9 +51,9 @@ conjunction with the Benchling_Tag node. ### Parameters - **benchling_context** : `BenchlingContext` - - Benchling context variable, which stores Benchling connection information + - Benchling context variable, which stores Benchling connection information - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -65,6 +66,7 @@ This template shows how a target plate could be pulled via the Benchling object Documentation for the Benchling API can be found here: https://benchling.com/api/reference benchling_context is an object of class BenchlingContext, consisting of + - **conn**: Benchling connection - **run_tag**: Benchling ID associated with run tag -- **display_tag**: Displayed value of Benchling run tag \ No newline at end of file +- **display_tag**: Displayed value of Benchling run tag diff --git a/docs/nodes/App/Benchling_Read_Object.md b/docs/nodes/App/Benchling_Read_Object.md index 66faa920..7ac18ac7 100644 --- a/docs/nodes/App/Benchling_Read_Object.md +++ b/docs/nodes/App/Benchling_Read_Object.md @@ -31,6 +31,7 @@ in the relevant Benchling tenant. Prior to usage, the secrets below must be configured in your Ganymede environment. More information on acquiring these credentials can be found [here](https://docs.benchling.com/docs/authentication). + - **benchling_url**: URL for Benchling tenant; has form https://\.benchling.com - **benchling_app_client_id**: Client_ID; has form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - **benchling_client_secret**: Client_Secret; has form cs_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx @@ -49,9 +50,9 @@ conjunction with the Input_Param node. ### Parameters - **benchling_context** : `BenchlingContext` - - Benchling context variable, which stores Benchling connection information + - Benchling context variable, which stores Benchling connection information - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -64,6 +65,7 @@ This template shows how a Benchling entry could be accessed via the Benchling ob Documentation for the Benchling API can be found here: https://benchling.com/api/reference benchling_context is an object of class BenchlingContext, consisting of + - **conn**: Benchling connection - **run_tag**: Benchling ID associated with run tag -- **display_tag**: Displayed value of Benchling run tag \ No newline at end of file +- **display_tag**: Displayed value of Benchling run tag diff --git a/docs/nodes/App/Benchling_Warehouse_Query.md b/docs/nodes/App/Benchling_Warehouse_Query.md index e64791c5..98645e0a 100644 --- a/docs/nodes/App/Benchling_Warehouse_Query.md +++ b/docs/nodes/App/Benchling_Warehouse_Query.md @@ -13,6 +13,7 @@ Query Benchling Warehouse Postgres database, process data in Python, and upload ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **benchling_postgres_host**: Host for Benchling warehouse database. Of the form: postgres-warehouse.\.benchling.com - **benchling_postgres_username**: Username for Benchling warehouse database. Note that this is not the same as your Benchling username. - **benchling_postgres_password**: Password for Benchling warehouse database. Note that this is not the same as your Benchling password. @@ -33,11 +34,11 @@ Process tabular data from user-defined SQL query, writing results back to data l ### Parameters - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame]` - - Table(s) or list of tables retrieved from user-defined SQL query +- Table(s) or list of tables retrieved from user-defined SQL query - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata +- Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/App/Benchling_Warehouse_Sync.md b/docs/nodes/App/Benchling_Warehouse_Sync.md index 1cb1da9f..317f2838 100644 --- a/docs/nodes/App/Benchling_Warehouse_Sync.md +++ b/docs/nodes/App/Benchling_Warehouse_Sync.md @@ -36,7 +36,7 @@ assistance, please don't hesitate to reach out to Ganymede. ## User-Defined Python -This node helps to sync tables from Benchling warehouse to Ganymede data lake. A dataframe is +This node helps to sync tables from Benchling warehouse to Ganymede datalake. A dataframe is constructed with the following columns - schema: str @@ -52,9 +52,9 @@ Ganymede. ### Parameters - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `pd.DataFrame` - DataFrame containing schema, from_benchling_table, and to_ganymede_table \ No newline at end of file + DataFrame containing schema, from_benchling_table, and to_ganymede_table diff --git a/docs/nodes/App/Benchling_Write.md b/docs/nodes/App/Benchling_Write.md index 50eae5c7..d39c158d 100644 --- a/docs/nodes/App/Benchling_Write.md +++ b/docs/nodes/App/Benchling_Write.md @@ -23,6 +23,7 @@ in the relevant Benchling tenant. Prior to usage, the secrets below must be configured in your Ganymede environment. More information on acquiring these credentials can be found [here](https://docs.benchling.com/docs/authentication). + - **benchling_url**: URL for Benchling tenant; has form https://\.benchling.com - **benchling_app_client_id**: Client_ID; has form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - **benchling_client_secret**: Client_Secret; has form cs_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx @@ -38,13 +39,13 @@ Processes input DataFrame for upload to Benchling ### Parameters - **df** : `pd.DataFrame` - - Tabular result of user-defined SQL query + - Tabular result of user-defined SQL query - **benchling_context** : `BenchlingContext` - - Benchling context variable, which stores Benchling connection information + - Benchling context variable, which stores Benchling connection information - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or blob storage \ No newline at end of file + Object containing data to store in data lake and/or blob storage diff --git a/docs/nodes/App/Benchling_Write_Object.md b/docs/nodes/App/Benchling_Write_Object.md index 813de9c2..ea7779e4 100644 --- a/docs/nodes/App/Benchling_Write_Object.md +++ b/docs/nodes/App/Benchling_Write_Object.md @@ -29,6 +29,7 @@ in the relevant Benchling tenant. Prior to usage, the secrets below must be configured in your Ganymede environment. More information on acquiring these credentials can be found [here](https://docs.benchling.com/docs/authentication). + - **benchling_url**: URL for Benchling tenant; has form https://\.benchling.com - **benchling_app_client_id**: Client_ID; has form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - **benchling_client_secret**: Client_Secret; has form cs_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx @@ -46,15 +47,15 @@ If new custom entities are created, returns DataFrame associated with custom ent ### Parameters - **data** : `dict[str, bytes]` - - Data to retrieve from Ganymede cloud storage + - Data to retrieve from Ganymede cloud storage - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame]` - - Tabular results of user-defined SQL query + - Tabular results of user-defined SQL query - **benchling_context** : `BenchlingContext` - - Benchling context variable, which stores Benchling connection information + - Benchling context variable, which stores Benchling connection information - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/App/Coda_Write.md b/docs/nodes/App/Coda_Write.md index 0310ee4f..d831801b 100644 --- a/docs/nodes/App/Coda_Write.md +++ b/docs/nodes/App/Coda_Write.md @@ -17,6 +17,7 @@ the specified Coda.io page. ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **coda_api_key**: Coda API Key Secrets can be configured by clicking on your username in the upper-right hand of the Ganymede @@ -31,9 +32,9 @@ are populated, with the doc_id from the generated document returned. ### Parameters - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame]` - - Table(s) to retrieve from data lake + - Table(s) to retrieve from data lake - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -42,4 +43,4 @@ are populated, with the doc_id from the generated document returned. ### Notes -See [Coda API](https://coda.io/developers/apis/) for more details on API usage. \ No newline at end of file +See [Coda API](https://coda.io/developers/apis/) for more details on API usage. diff --git a/docs/nodes/App/ELabNext_Write.md b/docs/nodes/App/ELabNext_Write.md index cda2a1ee..2a08f429 100644 --- a/docs/nodes/App/ELabNext_Write.md +++ b/docs/nodes/App/ELabNext_Write.md @@ -27,6 +27,7 @@ the specified eLabNext page. ### Notes Prior to usage, the following secrets must be configured in your Ganymede environment: + - **elabnext_base_url**: eLabNext Base URL - **elabnext_user**: eLabNext Username - **elabnext_pass**: eLabNext Password @@ -42,15 +43,15 @@ Example demonstrating submission of user-defined SQL query into eLabNext ### Parameters - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame]` - - Table(s) to retrieve from data lake + - Table(s) to retrieve from data lake - **data_input** : `dict[str, bytes]` - - Objects to retrieve from cloud storage + - Objects to retrieve from cloud storage - **base_url** : `str` - - Base URL for eLabNext to retrieve data from / post data to + - Base URL for eLabNext to retrieve data from / post data to - **token** : `str` - - token used for API access + - token used for API access - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -59,4 +60,4 @@ Example demonstrating submission of user-defined SQL query into eLabNext ### Notes -See [eLabNext API](https://www.elabjournal.com/docs/api/index) for more details on API usage. \ No newline at end of file +See [eLabNext API](https://www.elabjournal.com/docs/api/index) for more details on API usage. diff --git a/docs/nodes/App/Load_Parquet_to_Table.md b/docs/nodes/App/Load_Parquet_to_Table.md index 97930795..fa5b2562 100644 --- a/docs/nodes/App/Load_Parquet_to_Table.md +++ b/docs/nodes/App/Load_Parquet_to_Table.md @@ -8,12 +8,12 @@ displayed_sidebar: webUiSidebar ### Node Description -Loads parquet files saved in Ganymede storage into data lake tables. Data will be appended to the -data lake table if it already exists, otherwise create a new one. +Loads parquet files saved in Ganymede storage into datalake tables. Data will be appended to the +datalake table if it already exists, otherwise create a new one. ### Node Attributes - **table_name: str** - - The new or existing table name in the data lake + - The new or existing table name in the datalake - **storage_regex: str** - - Regex to identify parquet files \ No newline at end of file + - Regex to identify parquet files diff --git a/docs/nodes/App/S3_Event.md b/docs/nodes/App/S3_Event.md index 1b2d43d9..5c560a63 100644 --- a/docs/nodes/App/S3_Event.md +++ b/docs/nodes/App/S3_Event.md @@ -20,15 +20,20 @@ Node for triggering Flow upon receiving an S3 Event. ### Notes Prior to usage, the following secrets must be configured in the environment: + - **aws_s3_role_arn**: AWS Service Account Name -- **s3_api_key**: API key configured for EventBridge -This ARN will be used to enable the Airflow connection to the S3 bucket. Configuring this ARN will require the service account ID associated with the workflow orchestrator. Please reach out to the Ganymede team for instructions on configuring access within AWS to enable Ganymede connectivity. -Once configured, add the AWS role ARN secret to your environment by clicking on your username in the upper-right hand of the Ganymede application, then selecting Environment Settings and navigating to the Secrets tab. +This ARN will be used to enable the Airflow connection to the S3 bucket. Configuring this ARN +will require the service account ID associated with the workflow orchestrator. Please reach +out to the Ganymede team for instructions on configuring access within AWS to enable Ganymede +connectivity. -In addition to configuring role ARN, EventBridge must be configured to send events to Ganymede for the buckets that you would like to reference. The API destination for EventBridge should be configured to https://\/event/aws. +Once configured, add the AWS role ARN secret to your environment by clicking on your username in +the upper-right hand of the Ganymede application, then selecting Environment Settings and +navigating to the Secrets tab. -For more details on configuring EventBridge, visit https://docs.ganymede.bio/app/configuration/S3EventConfig. +In addition to configuring role ARN, EventBridge must be configured to send events to Ganymede. +If you need assistance on configuring EventBridge, please reach out to the Ganymede team. ### Example @@ -44,13 +49,13 @@ Calls AWS to get data for flow to trigger ### Parameters - **object_name** : `str` - - File name in S3 bucket + - File name in S3 bucket - **bucket_name** : `str` - - Name of S3 bucket -- **s3_hook**: S3Hook - - AWS S3 hook + - Name of S3 bucket +- **aws_session**: `AWS Session` + - AWS Session ### Returns `FlowInputs` - Object containing data for kicking off subsequent Flow \ No newline at end of file + Object containing data for kicking off subsequent Flow diff --git a/docs/nodes/App/S3_Read.md b/docs/nodes/App/S3_Read.md index 8e44ea80..73cef486 100644 --- a/docs/nodes/App/S3_Read.md +++ b/docs/nodes/App/S3_Read.md @@ -71,7 +71,7 @@ Specify list of S3 keys to upload to GCS Retrieve body of S3 object of first record in df_s3_objects (assuming boto3 client is passed in as client) -```python -obj = df_s3_objects.iloc[0].to_dict() -file_contents = client.get_object(Key=obj["Key"], Bucket=obj["Bucket"])['Body'].read() -``` +```bash +>>> obj = df_s3_objects.iloc[0].to_dict() +>>> file_contents = client.get_object(Key=obj["Key"], Bucket=obj["Bucket"])['Body'].read() +``` \ No newline at end of file diff --git a/docs/nodes/App/S3_Write.md b/docs/nodes/App/S3_Write.md index 8dab5eb6..2ca2ad92 100644 --- a/docs/nodes/App/S3_Write.md +++ b/docs/nodes/App/S3_Write.md @@ -24,8 +24,6 @@ Sync GCS bucket -\> S3 bucket - "replace" or "append" - whether to replace existing files or only add new files to the destination S3 bucket when a file in the S3 bucket has the same name as the source file being synced - **dest_s3_key** : `str` - Destination S3 key to write to. If this string ends with a '/', it will be treated as a prefix (i.e. - S3 folder), and the file will be written with the same name as the source file -- **glob_pattern** : `str` - - Glob pattern to match files that need to be synced. Specify "*" to avoid filtering by glob pattern. ### Notes @@ -51,7 +49,6 @@ An example configuration for an S3 Write Node in the Flow Editor (src_object_pre - **src_all_or_current_run**: current - **src_input_or_output_bucket**: output - **src_object_prefix**: -- **glob_pattern**: \* ### Parameters @@ -68,6 +65,3 @@ An example configuration for an S3 Write Node in the Flow Editor (src_object_pre - "replace" or "append" - whether to replace existing files or only add new files to the destination S3 bucket when a file in the S3 bucket has the same name as the source file being synced - **dest_s3_key** : `str` - Destination S3 key to write to. If this string ends with a '/', it will be treated as a prefix (i.e. - S3 folder), and the file will be written with the same name as the source file -- **glob_pattern**: `str` - - Glob pattern to match files that need to be synced. Specify "*" to avoid filtering by glob pattern. - \ No newline at end of file diff --git a/docs/nodes/App/SciNote_API.md b/docs/nodes/App/SciNote_API.md index 5988ac26..2a697965 100644 --- a/docs/nodes/App/SciNote_API.md +++ b/docs/nodes/App/SciNote_API.md @@ -69,4 +69,4 @@ Example demonstrating access to SciNote API ### Notes -See [SciNote API](https://scinote-eln.github.io/scinote-api-v1-docs/) for more details on API usage. \ No newline at end of file +See [SciNote API](https://scinote-eln.github.io/scinote-api-v1-docs/) for more details on API usage. diff --git a/docs/nodes/App/Smartsheet_Read.md b/docs/nodes/App/Smartsheet_Read.md index 001b207d..5005bcfb 100644 --- a/docs/nodes/App/Smartsheet_Read.md +++ b/docs/nodes/App/Smartsheet_Read.md @@ -39,4 +39,4 @@ Process Smartsheet table ### Notes -See [Smartsheet API documentation](https://smartsheet.redoc.ly/) for more details on API usage. \ No newline at end of file +See [Smartsheet API documentation](https://smartsheet.redoc.ly/) for more details on API usage. diff --git a/docs/nodes/App/Snowflake_Write.md b/docs/nodes/App/Snowflake_Write.md index 9e71fb3d..3025ec93 100644 --- a/docs/nodes/App/Snowflake_Write.md +++ b/docs/nodes/App/Snowflake_Write.md @@ -32,4 +32,4 @@ Prior to usage, the following secrets must be configured in your Ganymede enviro Secrets can be configured by clicking on your username in the upper-right hand of the Ganymede application, then selecting Environment Settings and navigating to the Secrets tab. If you need -assistance, please don't hesitate to reach out to Ganymede. \ No newline at end of file +assistance, please don't hesitate to reach out to Ganymede. diff --git a/docs/nodes/App/Webhook_Event.md b/docs/nodes/App/Webhook_Event.md new file mode 100644 index 00000000..1b4c0956 --- /dev/null +++ b/docs/nodes/App/Webhook_Event.md @@ -0,0 +1,33 @@ +--- +sidebar_label: Webhook_Event +title: Webhook_Event +displayed_sidebar: webUiSidebar +--- + +## Node + +### Node Description + +### Node Attributes + +- **input_filter_path** + - Input [JSONPath](https://jsonpath.com) used for applying filtering logic + +### Notes + +Webhook secrets will need to be configured prior to usage; please reach out to [Ganymede support](../../app/Support) to configure secrets. + +## User-Defined Python + +Get raw event from webhook and store in table or file for further processing + +### Parameters + +- **event_data_json** : `dict` + - raw event payload from webhook + +### Returns + +`NodeReturn` + Object containing data to store in data lake and/or file storage. + diff --git a/docs/nodes/App/sidebar.json b/docs/nodes/App/sidebar.json index b1a81a75..959fff8b 100644 --- a/docs/nodes/App/sidebar.json +++ b/docs/nodes/App/sidebar.json @@ -1 +1 @@ -{"type": "category", "label": "App", "items": ["nodes/App/APINode", "nodes/App/Airtable_Export", "nodes/App/Airtable_Import", "nodes/App/Azure_Query", "nodes/App/Azure_Read", "nodes/App/Azure_Read_Multi", "nodes/App/Azure_Write", "nodes/App/Benchling_Api", "nodes/App/Benchling_Event", "nodes/App/Benchling_Read", "nodes/App/Benchling_Read_Object", "nodes/App/Benchling_Warehouse_Query", "nodes/App/Benchling_Warehouse_Sync", "nodes/App/Benchling_Write", "nodes/App/Benchling_Write_Object", "nodes/App/Coda_Write", "nodes/App/ELabNext_Write", "nodes/App/Load_Parquet_to_Table", "nodes/App/S3_Event", "nodes/App/S3_Read", "nodes/App/S3_Write", "nodes/App/SciNote_API", "nodes/App/Smartsheet_Read", "nodes/App/Snowflake_Write"]} \ No newline at end of file +{"type": "category", "label": "App", "items": ["nodes/App/APINode", "nodes/App/Airtable_Export", "nodes/App/Airtable_Import", "nodes/App/Azure_Query", "nodes/App/Azure_Read", "nodes/App/Azure_Read_Multi", "nodes/App/Azure_Write", "nodes/App/Benchling_Api", "nodes/App/Benchling_Event", "nodes/App/Benchling_Read", "nodes/App/Benchling_Read_Object", "nodes/App/Benchling_Warehouse_Query", "nodes/App/Benchling_Warehouse_Sync", "nodes/App/Benchling_Write", "nodes/App/Benchling_Write_Object", "nodes/App/Coda_Write", "nodes/App/ELabNext_Write", "nodes/App/Load_Parquet_to_Table", "nodes/App/S3_Event", "nodes/App/S3_Read", "nodes/App/S3_Write", "nodes/App/SciNote_API", "nodes/App/Smartsheet_Read", "nodes/App/Snowflake_Write", "nodes/App/Webhook_Event"]} \ No newline at end of file diff --git a/docs/nodes/File/AVI_Read.md b/docs/nodes/File/AVI_Read.md index c92d7327..e2086532 100644 --- a/docs/nodes/File/AVI_Read.md +++ b/docs/nodes/File/AVI_Read.md @@ -10,7 +10,7 @@ displayed_sidebar: webUiSidebar Reads contents of an avi file into data lake -The contents of avi file are passed to the user-defined function, which +The contents of the avi file are passed to the user-defined function, which return one or more tables for storage in Ganymede data lake. ### Node Attributes @@ -34,11 +34,11 @@ data tables stored in the data lake ### Parameters - **avi_file** : `dict[str, BytesIO]` - - avi files, indexed by file name + - avi files, indexed by file name - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame]` - - Table(s) or list of tables retrieved from user-defined SQL query + - Table(s) or list of tables retrieved from user-defined SQL query - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -51,4 +51,4 @@ If a DataFrame is returned, the table name corresponds to the **results** parame If a dict of dataframes is returned, the keys of the dict are used as table names for the corresponding DataFrames to store. The table with **results** as its key -is displayed on the Flow Editor. \ No newline at end of file +is displayed on the Flow Editor. diff --git a/docs/nodes/File/AVI_Read_Multi.md b/docs/nodes/File/AVI_Read_Multi.md index beb4a897..6b9c8138 100644 --- a/docs/nodes/File/AVI_Read_Multi.md +++ b/docs/nodes/File/AVI_Read_Multi.md @@ -34,11 +34,11 @@ data tables stored in the data lake ### Parameters - **avi_file** : `dict[str, BytesIO]` - - avi files, indexed by file name + - avi files, indexed by file name - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame]` - - Table(s) or list of tables retrieved from user-defined SQL query + - Table(s) or list of tables retrieved from user-defined SQL query - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -51,4 +51,4 @@ If a DataFrame is returned, the table name corresponds to the **results** parame If a dict of dataframes is returned, the keys of the dict are used as table names for the corresponding DataFrames to store. The table with **results** as its key -is displayed on the Flow Editor. \ No newline at end of file +is displayed on the Flow Editor. diff --git a/docs/nodes/File/CSV_Read.md b/docs/nodes/File/CSV_Read.md index 2fda26d1..37023838 100644 --- a/docs/nodes/File/CSV_Read.md +++ b/docs/nodes/File/CSV_Read.md @@ -32,11 +32,11 @@ stored in data lake ### Parameters - **csv_file** : `dict[str, BytesIO]` - - CSV files, indexed by file name + - CSV files, indexed by file name - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/File/CSV_Read_Multi.md b/docs/nodes/File/CSV_Read_Multi.md index c7409d91..6b5a2a2c 100644 --- a/docs/nodes/File/CSV_Read_Multi.md +++ b/docs/nodes/File/CSV_Read_Multi.md @@ -8,7 +8,7 @@ displayed_sidebar: webUiSidebar ### Node Description -Reads contents of multiple CSV files into data lake +Reads contents of a CSV file into data lake The contents of CSV file are passed to the user-defined function, which return one or more tables for storage in Ganymede data lake. @@ -32,11 +32,11 @@ stored in data lake ### Parameters - **csv_file** : `dict[str, BytesIO]` - - CSV files, indexed by file name + - CSV files, indexed by file name - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/File/CSV_Write.md b/docs/nodes/File/CSV_Write.md index 3e2cb74d..3485594d 100644 --- a/docs/nodes/File/CSV_Write.md +++ b/docs/nodes/File/CSV_Write.md @@ -17,7 +17,7 @@ data lake. ### Node Attributes - **output_csv_filename** - - Name of the output CSV file + - name of output CSV file ## User-Defined Python @@ -26,11 +26,11 @@ Process table(s) resulting from user-defined SQL query into CSV file ### Parameters - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame]` - - Table(s) or list of tables retrieved from user-defined SQL query + - Table(s) or list of tables retrieved from user-defined SQL query - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/File/Excel_Read.md b/docs/nodes/File/Excel_Read.md index 997b7109..f4909578 100644 --- a/docs/nodes/File/Excel_Read.md +++ b/docs/nodes/File/Excel_Read.md @@ -39,9 +39,9 @@ Reads Excel file and stores processed table(s) in data lake. ### Parameters - **excel_file** : `bytes | dict[str, bytes]` - - Excel file as a bytes object or as dict indexed by filename + - Excel file as a bytes object or as dict indexed by filename - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -50,5 +50,5 @@ Reads Excel file and stores processed table(s) in data lake. ### Notes -excel_file is represented in bytes so user can handle cases where Excel spreadsheet is -a binary object within this function \ No newline at end of file +Excel_file is represented in bytes so user can handle cases where Excel spreadsheet is +a binary object within this function diff --git a/docs/nodes/File/Excel_Read_Multi.md b/docs/nodes/File/Excel_Read_Multi.md index d665b10c..ab91a94b 100644 --- a/docs/nodes/File/Excel_Read_Multi.md +++ b/docs/nodes/File/Excel_Read_Multi.md @@ -8,9 +8,9 @@ displayed_sidebar: webUiSidebar ### Node Description -Reads multiple Excel files and uploads processed result to data lake +Reads Excel file and uploads processed result to data lake -Ingests Excel files, which are passed to the user-defined function as a bytes object +Ingests one Excel file, which is passed to the user-defined function as a bytes object for processing. The user-defined function returns one or more tables, which are uploaded to the Ganymede data lake. @@ -39,9 +39,9 @@ Reads Excel file and stores processed table(s) in data lake. ### Parameters - **excel_file** : `bytes | dict[str, bytes]` - - Excel file as a bytes object or as dict indexed by filename + - Excel file as a bytes object or as dict indexed by filename - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -50,5 +50,5 @@ Reads Excel file and stores processed table(s) in data lake. ### Notes -excel_file is represented in bytes so user can handle cases where Excel spreadsheet is -a binary object within this function \ No newline at end of file +Excel_file is represented in bytes so user can handle cases where Excel spreadsheet is +a binary object within this function diff --git a/docs/nodes/File/Excel_Write.md b/docs/nodes/File/Excel_Write.md index 71017e33..93a1ae85 100644 --- a/docs/nodes/File/Excel_Write.md +++ b/docs/nodes/File/Excel_Write.md @@ -29,13 +29,13 @@ Write table(s) from SQL query and stores Excel sheet in cloud storage. ### Parameters - **df_sql_result** : `pd.DataFrame | list[pd.DataFrame], optional` - - Table(s) or list of tables retrieved from user-defined SQL query + - Table(s) or list of tables retrieved from user-defined SQL query - **output_spreadsheet_name** : `str` - - Name of the output spreadsheet + - Name of the output spreadsheet - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/File/FCS_Extract_Load.md b/docs/nodes/File/FCS_Extract_Load.md index 13ef90d0..211ae556 100644 --- a/docs/nodes/File/FCS_Extract_Load.md +++ b/docs/nodes/File/FCS_Extract_Load.md @@ -29,11 +29,11 @@ The fcsparser python package is used to parse FCS files into metadata and data ### Notes -- **data** contains a record for each flow cytometer detection event +*data* contains a record for each flow cytometer detection event (generally passage of single cell past detector); for each event, a value is recorded for each channel -- **metadata** comprises of the following components: +*metadata* comprises of the following components: - **header**: contains FCS version number and bytes corresponding to where binary positions of text, data, and analysis in FCS file. FCS format: FCS version number @@ -101,4 +101,4 @@ Process FCS data/metadata file ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/File/HDF5_Read.md b/docs/nodes/File/HDF5_Read.md index cc507ee9..f87c4e17 100644 --- a/docs/nodes/File/HDF5_Read.md +++ b/docs/nodes/File/HDF5_Read.md @@ -31,11 +31,11 @@ Processes HDF5 data ### Parameters - **h5data** : `dict[str, object]` - - HDF5 data presented as a nested dict + - HDF5 data presented as a nested dict - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/File/Image_Read.md b/docs/nodes/File/Image_Read.md index a1a4f1e5..e4c01cec 100644 --- a/docs/nodes/File/Image_Read.md +++ b/docs/nodes/File/Image_Read.md @@ -31,11 +31,11 @@ Process image data into a dictionary of images indexed by filename to save in cl ### Parameters - **img** : `bytes` - - Image file, represented as bytes + - Image file, represented as bytes - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/File/Image_Read_Multi.md b/docs/nodes/File/Image_Read_Multi.md index df5ebb76..95ea0314 100644 --- a/docs/nodes/File/Image_Read_Multi.md +++ b/docs/nodes/File/Image_Read_Multi.md @@ -38,11 +38,11 @@ Process image data into a dictionary of images indexed by filename to save in cl ### Parameters - **img** : `bytes` - - Image file, represented as bytes + - Image file, represented as bytes - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/File/Image_Write.md b/docs/nodes/File/Image_Write.md index 761050a3..b27b3718 100644 --- a/docs/nodes/File/Image_Write.md +++ b/docs/nodes/File/Image_Write.md @@ -22,11 +22,11 @@ Process image data into a dictionary of images indexed by filename to save in cl ### Parameters - **img** : `bytes` - - Image file, represented as bytes + - Image file, represented as bytes - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/File/Input_File.md b/docs/nodes/File/Input_File.md index 8da93bb5..068b9ba2 100644 --- a/docs/nodes/File/Input_File.md +++ b/docs/nodes/File/Input_File.md @@ -22,7 +22,7 @@ result to Ganymede cloud storage. The Node configuration below would capture files ending in .xml: -- **file_pattern**: *.xml +file_pattern: *.xml ## User-Defined Python @@ -31,9 +31,9 @@ Processes file data for saving in cloud storage ### Parameters - **file_data** : `dict[str, bytes]` - - Bytes object to process, indexed by filename + - Bytes object to process, indexed by filename - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -43,4 +43,4 @@ Processes file data for saving in cloud storage ### Notes Files can also be retrieved and processed using the list_files and retrieve_files functions. -Documentation on these functions can be found at https://docs.ganymede.bio/sdk/GanymedeClass \ No newline at end of file +Documentation on these functions can be found at https://docs.ganymede.bio/sdk/GanymedeClass diff --git a/docs/nodes/File/Input_File_Multi.md b/docs/nodes/File/Input_File_Multi.md index e4fdf03b..6b67fa81 100644 --- a/docs/nodes/File/Input_File_Multi.md +++ b/docs/nodes/File/Input_File_Multi.md @@ -8,7 +8,7 @@ displayed_sidebar: webUiSidebar ### Node Description -Retrieves files for processing by user-defined function and uploads +Retrieves file for processing by user-defined function and uploads result to Ganymede cloud storage. ### Node Attributes @@ -22,7 +22,7 @@ result to Ganymede cloud storage. The Node configuration below would capture files ending in .xml: -- **file_pattern**: *.xml +file_pattern: *.xml ## User-Defined Python @@ -31,9 +31,9 @@ Processes file data for saving in cloud storage ### Parameters - **file_data** : `dict[str, bytes]` - - Bytes object to process, indexed by filename + - Bytes object to process, indexed by filename - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns @@ -43,4 +43,4 @@ Processes file data for saving in cloud storage ### Notes Files can also be retrieved and processed using the list_files and retrieve_files functions. -Documentation on these functions can be found at https://docs.ganymede.bio/sdk/GanymedeClass \ No newline at end of file +Documentation on these functions can be found at https://docs.ganymede.bio/sdk/GanymedeClass diff --git a/docs/nodes/File/PDF_Read.md b/docs/nodes/File/PDF_Read.md index 6e26c263..acfc3684 100644 --- a/docs/nodes/File/PDF_Read.md +++ b/docs/nodes/File/PDF_Read.md @@ -32,11 +32,11 @@ data tables stored in the data lake ### Parameters - **pdf_file** : `dict[str, BytesIO]` - - pdf files, indexed by file name + - pdf files, indexed by file name - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/File/PDF_Read_Multi.md b/docs/nodes/File/PDF_Read_Multi.md index 77f7e64c..73110129 100644 --- a/docs/nodes/File/PDF_Read_Multi.md +++ b/docs/nodes/File/PDF_Read_Multi.md @@ -32,11 +32,11 @@ data tables stored in the data lake ### Parameters - **pdf_file** : `dict[str, BytesIO]` - - pdf files, indexed by file name + - pdf files, indexed by file name - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/File/Powerpoint_Write.md b/docs/nodes/File/Powerpoint_Write.md index 55dc9a6c..a5cdc9b3 100644 --- a/docs/nodes/File/Powerpoint_Write.md +++ b/docs/nodes/File/Powerpoint_Write.md @@ -20,11 +20,11 @@ Processes table(s) from user-defined SQL query into Powerpoint output ### Parameters - **df_sql_result** : `Optional[pd.DataFrame | list[pd.DataFrame]]` - - table or list of tables from SQL query to process + - table or list of tables from SQL query to process - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/File/XML_Read.md b/docs/nodes/File/XML_Read.md index 7ad427b8..99dd49b0 100644 --- a/docs/nodes/File/XML_Read.md +++ b/docs/nodes/File/XML_Read.md @@ -29,11 +29,11 @@ Parse XML file into table for storage in data lake ### Parameters - **xml_file** : `BytesIO` - - Contents of input XML file + - Contents of input XML file - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/File/Zip_Read.md b/docs/nodes/File/Zip_Read.md index 36d990ba..7f04a512 100644 --- a/docs/nodes/File/Zip_Read.md +++ b/docs/nodes/File/Zip_Read.md @@ -30,11 +30,11 @@ Reads Zip file and stores extracted files in the data lake. ### Parameters - **input_zip_file** : `bytes` - - Zip file as a bytes object + - Zip file as a bytes object - **ganymede_context** : `Optional[GanymedeContext]` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/Instrument/Agilent_HPLC_Read.md b/docs/nodes/Instrument/Agilent_HPLC_Read.md index 3eafa1b1..0e0450a3 100644 --- a/docs/nodes/Instrument/Agilent_HPLC_Read.md +++ b/docs/nodes/Instrument/Agilent_HPLC_Read.md @@ -28,14 +28,14 @@ and another for the metadata. ### Parameters - **datafile** : `object` - - An object that contains the data, metadata, detector, xlabels and ylabels attributes. - - - datafile.data: A list of lists, where each sublist represents a row of data values. - - - datafile.metadata: A dictionary of metadata. - - - datafile.detector: A string representing the detector used. - - - datafile.xlabels: A list of labels for the x-axis. - - - datafile.ylabels: A list of labels for the y-axis. + - An object that contains the data, metadata, detector, xlabels and ylabels attributes. + - datafile.data: A list of lists, where each sublist represents a row of data values. + - datafile.metadata: A dictionary of metadata. + - datafile.detector: A string representing the detector used. + - datafile.xlabels: A list of labels for the x-axis. + - datafile.ylabels: A list of labels for the y-axis. - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns diff --git a/docs/nodes/Instrument/Instron_Tensile_Read.md b/docs/nodes/Instrument/Instron_Tensile_Read.md index 5d5406b9..290e5887 100644 --- a/docs/nodes/Instrument/Instron_Tensile_Read.md +++ b/docs/nodes/Instrument/Instron_Tensile_Read.md @@ -32,11 +32,11 @@ stored in data lake ### Parameters - **is_tens_file** : `dict[str, BytesIO]` - - is_tens file, indexed by file name + - is_tens file, indexed by file name - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/Instrument/LCMS_Read.md b/docs/nodes/Instrument/LCMS_Read.md index 4fad3c2a..4a965bed 100644 --- a/docs/nodes/Instrument/LCMS_Read.md +++ b/docs/nodes/Instrument/LCMS_Read.md @@ -19,7 +19,7 @@ Parse and ingest data from an LCMS file into Ganymede data lake. The Node configuration below would capture mzML files: -- **lcms**: *.mzML +lcms: *.mzML ## User-Defined Python @@ -28,11 +28,11 @@ Function to process LCMS file data ### Parameters - **lcms_file_paths** : `str` - - Path to LCMS file to be read + - Path to LCMS file to be read - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/Instrument/LCMS_Read_Multi.md b/docs/nodes/Instrument/LCMS_Read_Multi.md index 0cd6191c..dde5e75e 100644 --- a/docs/nodes/Instrument/LCMS_Read_Multi.md +++ b/docs/nodes/Instrument/LCMS_Read_Multi.md @@ -19,7 +19,7 @@ Parse and ingest data from multiple LCMS files into Ganymede data lake. The Node configuration below would capture mzML files: -- **lcms**: *.mzML +lcms: *.mzML ## User-Defined Python @@ -28,11 +28,11 @@ Function to process LCMS file data ### Parameters - **lcms_file_paths** : `list[str]` - - Paths to LCMS files to be read + - Paths to LCMS files to be read - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/Instrument/LC_Read.md b/docs/nodes/Instrument/LC_Read.md index baee1982..9ac74bf6 100644 --- a/docs/nodes/Instrument/LC_Read.md +++ b/docs/nodes/Instrument/LC_Read.md @@ -19,7 +19,7 @@ Parse data from an LC file and ingest into Ganymede data lake. The Node configuration below would capture CDF files: -- **lc**: *.cdf +lc: *.cdf ## User-Defined Python @@ -28,11 +28,11 @@ Function to process LC file data ### Parameters - **lc_file_path** : `list[str]` - - Path to LC file to be read + - Path to LC file to be read - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/Instrument/LC_Read_Multi.md b/docs/nodes/Instrument/LC_Read_Multi.md index a6cfe9c9..289979af 100644 --- a/docs/nodes/Instrument/LC_Read_Multi.md +++ b/docs/nodes/Instrument/LC_Read_Multi.md @@ -19,7 +19,7 @@ Parse data from multiple LC files and ingest into Ganymede data lake. The Node configuration below would capture CDF files: -- **lc**: *.cdf +lc: *.cdf ## User-Defined Python @@ -28,11 +28,11 @@ Function to process LC file data ### Parameters - **lc_file_paths** : `list[str]` - - Paths to LC files to be read + - Paths to LC files to be read - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/Instrument/Profilometer_Read.md b/docs/nodes/Instrument/Profilometer_Read.md index 6d94d531..bedf4817 100644 --- a/docs/nodes/Instrument/Profilometer_Read.md +++ b/docs/nodes/Instrument/Profilometer_Read.md @@ -28,11 +28,11 @@ Processes h5data into bytes object for saving in cloud storage ### Parameters - **h5data** : `dict` - - HDF5 file contents structured as a Python dict + - HDF5 file contents structured as a Python dict - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage \ No newline at end of file + Object containing data to store in data lake and/or file storage diff --git a/docs/nodes/Instrument/Synergy_Read.md b/docs/nodes/Instrument/Synergy_Read.md index 009e4f60..07a89ab5 100644 --- a/docs/nodes/Instrument/Synergy_Read.md +++ b/docs/nodes/Instrument/Synergy_Read.md @@ -31,11 +31,11 @@ Processes Synergy text file(s) into data tables stored in data lake ### Parameters - **synergy_file** : `dict[str, bytes]` - - Synergy text files, indexed by file name + - Synergy text files, indexed by file name - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/Instrument/Synergy_Read_Multi.md b/docs/nodes/Instrument/Synergy_Read_Multi.md index 0a274647..86b80aa5 100644 --- a/docs/nodes/Instrument/Synergy_Read_Multi.md +++ b/docs/nodes/Instrument/Synergy_Read_Multi.md @@ -31,11 +31,11 @@ Processes Synergy text file(s) into data tables stored in data lake ### Parameters - **synergy_file** : `dict[str, bytes]` - - Synergy text files, indexed by file name + - Synergy text files, indexed by file name - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/Instrument/WSP_Read.md b/docs/nodes/Instrument/WSP_Read.md index c51c30c0..a0ee2c06 100644 --- a/docs/nodes/Instrument/WSP_Read.md +++ b/docs/nodes/Instrument/WSP_Read.md @@ -29,11 +29,11 @@ Parse WSP file ### Parameters - **wsp_file** : `BytesIO` - - Contents of WSP file + - Contents of WSP file - **ganymede_context** : `GanymedeContext` - - Ganymede context variable, which stores flow run metadata + - Ganymede context variable, which stores flow run metadata ### Returns `NodeReturn` - Object containing data to store in data lake and/or file storage. \ No newline at end of file + Object containing data to store in data lake and/or file storage. diff --git a/docs/nodes/Overview.mdx b/docs/nodes/Overview.mdx index c0d2a6f5..60c1c49a 100644 --- a/docs/nodes/Overview.mdx +++ b/docs/nodes/Overview.mdx @@ -36,8 +36,8 @@ The table below provides a list of available Nodes, along with details on whethe | Analysis | Transform_SQL | Table(s) | Table | True | False | | Analysis | Trigger_Python | Table(s) and/or File(s) | FlowInputs | True | False | | App | APINode | API | NodeReturn | True | False | -| App | AirtableExport | Table | API | True | False | -| App | AirtableImport | API | NodeReturn | False | False | +| App | Airtable_Export | Table | API | True | False | +| App | Airtable_Import | API | NodeReturn | False | False | | App | Azure_Query | Table | NodeReturn | True | False | | App | Azure_Read | API | FileAny | False | False | | App | Azure_Read_Multi | API | FileAny | False | True | @@ -59,6 +59,7 @@ The table below provides a list of available Nodes, along with details on whethe | App | SciNote_API | Table | NodeReturn | True | False | | App | Smartsheet_Read | API | NodeReturn | True | False | | App | Snowflake_Write | Table | NodeReturn | False | False | +| App | Webhook_Event | Event | NodeReturn | True | False | | File | AVI_Read | FileAVI | NodeReturn | True | False | | File | AVI_Read_Multi | Set[FileAVI] | NodeReturn | True | True | | File | CSV_Read | FileCSV | NodeReturn | True | False | @@ -233,13 +234,13 @@ The _execute_ function may call classes and functions found within the User-Defi | [Category](#node-categories) | Name | Brief Description | |------------|---------------------------|------------------------------------------------------------------------------| -| Analysis | Branch_Python | Process data with Python and conditionally execute downstream Nodes | +| Analysis | Branch_Python | Process data with Python and conditionally execute downstream nodes | | Analysis | Python | Process data with python | | Analysis | Transform_SQL | SQL analysis Function | | Analysis | Trigger_Python | Process data with Python and trigger subsequent flow | | App | APINode | Generic API Access Node | -| App | AirtableExport | Export data from Ganymede data lake to Airtable | -| App | AirtableImport | Import data from Airtable into Ganymede data lake | +| App | Airtable_Export | Export data from Ganymede data lake to Airtable | +| App | Airtable_Import | Import data from Airtable into Ganymede data lake | | App | Azure_Query | Query data from Azure SQL Server | | App | Azure_Read | Read data from Azure Blob Storage | | App | Azure_Read_Multi | Read all data from Azure Blob Storage | @@ -261,6 +262,7 @@ The _execute_ function may call classes and functions found within the User-Defi | App | SciNote_API | Create and write SciNote entry | | App | Smartsheet_Read | Read sheet from Smartsheet | | App | Snowflake_Write | Sync tables in Ganymede data lake to Snowflake | +| App | Webhook_Event | Capture events from a webhook for triggering flows | | File | AVI_Read | Read in contents of an AVI file to a table | | File | AVI_Read_Multi | Read in contents of multiple avi files to a table | | File | CSV_Read | Read in contents of a CSV file | diff --git a/docs/nodes/Tag/Benchling_Tag.md b/docs/nodes/Tag/Benchling_Tag.md index d8dc5127..77a34a46 100644 --- a/docs/nodes/Tag/Benchling_Tag.md +++ b/docs/nodes/Tag/Benchling_Tag.md @@ -23,10 +23,11 @@ in the relevant Benchling tenant. Prior to usage, the secrets below must be configured in your Ganymede environment. More information on acquiring these credentials can be found [here](https://docs.benchling.com/docs/authentication). + - **benchling_url**: URL for Benchling tenant; has form https://\.benchling.com - **benchling_app_client_id**: Client_ID; has form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - **benchling_client_secret**: Client_Secret; has form cs_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Secrets can be configured by clicking on your username in the upper-right hand of the Ganymede application, then selecting Environment Settings and navigating to the Secrets tab. If you need -assistance, please don't hesitate to reach out to Ganymede. \ No newline at end of file +assistance, please don't hesitate to reach out to Ganymede. diff --git a/docs/nodes/Tag/Input_Param.md b/docs/nodes/Tag/Input_Param.md index 3c25cf51..a76cc5d3 100644 --- a/docs/nodes/Tag/Input_Param.md +++ b/docs/nodes/Tag/Input_Param.md @@ -30,4 +30,4 @@ from ganymede_sdk import Ganymede g = Ganymede() param_value = g['Input_Param.param'] -``` \ No newline at end of file +``` diff --git a/docs/releases/202402.mdx b/docs/releases/202402.mdx index d88bc1a6..9a447ec9 100644 --- a/docs/releases/202402.mdx +++ b/docs/releases/202402.mdx @@ -8,7 +8,7 @@ displayed_sidebar: releaseSideBar **Improvements** - Agent executables are now names after the agent and agent version. - The Benchling class in Ganymede SDK now supports [notebook entry creation](../sdk/markdowns/benchling.benchling#function-benchlingcreate_entry). -- The LC class in Ganymede SDK now supports validating AIA file structure (CDF files enabling transport of data between different chromatography data systems). Associated methods [are part of the LC class](../sdk/markdowns/lc.md#function-lccompare_aia_files). +- The LC class in Ganymede SDK now supports validating AIA file structure (CDF files enabling transport of data between different chromatography data systems). Associated methods [are part of the LC class](../sdk/markdowns/lc.cdf). **Bug Fixes** - Editor notebook saves are now more performant. diff --git a/docs/releases/202409.mdx b/docs/releases/202409.mdx index d8263464..1cf150e2 100644 --- a/docs/releases/202409.mdx +++ b/docs/releases/202409.mdx @@ -41,4 +41,4 @@ displayed_sidebar: releaseSideBar **Bug Fixes** - **UI**: The list of flows in an environment will now be fetched when logging in from a direct link - **Notebook**: In certain cases, single input notebook templates were retrieved for multi nodes and vice versa. Faulty template retrieval was addressed in this release. -- **SDK**: Fixed function call for applying compensation within [WSP.calculate_population_counts](../sdk/markdowns/fcs#function-wspcalculate-population-counts) +- **SDK**: Fixed function call for applying compensation within [WSP.calculate_population_counts](../sdk/markdowns/fcs#function-wspcalculate_population_counts) diff --git a/docs/sdk/markdowns/AgentModelsAndMethods.mdx b/docs/sdk/markdowns/AgentModelsAndMethods.mdx index 379d7459..8f0811a2 100644 --- a/docs/sdk/markdowns/AgentModelsAndMethods.mdx +++ b/docs/sdk/markdowns/AgentModelsAndMethods.mdx @@ -264,9 +264,9 @@ def execute(new_file: FileParam, **kwargs) -> None: ### Cron Agent -#### Upload file if modified between _recency_min_ and _recency_max_ seconds ago +#### Upload file if modified in the last day -This Agent would require _parent_dir_ and _file_to_watch_ to be configured. Recency of last modified date can be configured if desired; if it isn't configured, then the watched file would be uploaded if it was modified in the last hour. +This Agent would require _watch_dir_ to be configured. Recency of last modified date can be configured if desired; if it isn't configured, then the watched file would be uploaded if it was modified in the last hour. ```python from ganymede_sdk.agent.models import FileParam, UploadFileParams @@ -291,32 +291,30 @@ def execute(**kwargs) -> UploadFileParams | None: info('Connection requires parent_dir and file_to_watch to be specified.') return None - # grab watch_dir, filename, recency_min, and recency_max from variables configurable in UI + # grab watch_dir, recency_min, and recency_max from variables configurable in UI watch_dir = agent_variables.get('parent_dir', None) - filename = agent_variables.get('file_to_watch', None) recency_min = float(agent_variables.get('recency_min', 0)) - recency_max = float(agent_variables.get('recency_max', 3600)) - - if watch_dir and filename: - full_file_path = Path(watch_dir) / Path(filename) - - if not full_file_path.is_file(): - error(f'File {full_file_path} does not exist') - return None - else: - info('Connection requires parent_dir and file_to_watch to be specified.') + recency_max = float(agent_variables.get('recency_max', 86400)) # last day + + if not watch_dir: + error('Connection requires watch_dir to be specified.') return None - elapsed_time = time.time() - os.stat(file_path).st_mtime - if elapsed_time >= recency_min and elapsed_time < recency_max: - if not os.access(full_file_path, os.R_OK): - error(f'Do not have read permissions for file {full_file_path}') - return None - - new_file_param = FileParam(filename=str(full_file_path), body=body) - return UploadFileParams(files=[new_file_param]) + list_of_files = glob.glob(f'{watch_dir}/*.csv') + + file_params = [] + for file_path in list_of_files: + file_elapsed_time = time.time() - os.stat(file_path).st_mtime + if file_elapsed_time >= recency_min and file_elapsed_time < recency_max: + if not os.access(file_path, os.R_OK): + error(f'Do not have read permissions for file {file_path}') + return None + with open(file_path, 'rb') as f: + body = f.read() + file_params.append(FileParam(filename=file_path, body=body)) + if (len(file_params) > 0): + return TriggerFlowParams(single_file_params=file_params) - info(f'File exists, but was last modified {elapsed_time} seconds ago. Only files modified between {recency_min} and {recency_max} seconds are uploaded.') return None ``` diff --git a/docs/sdk/markdowns/benchling.benchling.md b/docs/sdk/markdowns/benchling.benchling.md index 95ba5ef3..d4224add 100644 --- a/docs/sdk/markdowns/benchling.benchling.md +++ b/docs/sdk/markdowns/benchling.benchling.md @@ -36,7 +36,7 @@ Set up the Benchling object **ganymede_context** : `GanymedeContext`         Ganymede context to get run attributes -**benchling_context** : `Optional[BenchlingContext]` +**benchling_context** : `BenchlingContext | None`         Benchling context variable, which stores Benchling connection information @@ -46,13 +46,13 @@ Lists schemas, filtered by type if provided ### Parameters -**type** : `Optional[str]` +**type** : `str | None`         Schema type - potential values are "custom_entity", "assay_result", "molecule", "plate",         "aa_sequence", "box", "container", "entry", "location", "mixture", "workflow_task", etc. ### Returns -`List[Dict]` +`list[dict]`         list of schemas @@ -64,7 +64,7 @@ Get schema by name **schema_name** : `str`         Schema name -**type** : `Optional[str]` +**type** : `str | None`         Subset search by schema type - potential values are "custom_entity", "assay_result", etc. ### Returns @@ -81,30 +81,30 @@ Creates custom entity in Benchling.    If the entity does not exist, f **entity_name** : `str`         Name of new entity to be created -**folder_id** : `str` +**folder_id** : `str | Unset`         Folder ID containing Benchling entity to be created. This should be a string starting         with "lib_".    If updating a custom entity, specifying the folder ID is optional (if specified,         moves the custom entity to the specified folder). -**schema_id** : `str` +**schema_id** : `str | Unset`         Input schema ID Tag. schema associated with Benchling entity to be created. This should         be a string starting with "ts_" -**registry_id** : `str` +**registry_id** : `str | Unset`         Project associated with custom entity.    This identifies the registry that your run         entity will be registered to.    This can be found by clicking on Avatar - Feature         Settings - Registry Settings, and you will be able to find it in the URL.    This should         be a string starting with "src_" **naming_strategy** : `NamingStrategy`         Naming strategy to use when creating new entities. See NamingStrategy for more details. -**custom_entity_fields** : `Optional[Dict]` +**custom_entity_fields** : `dict | Unset`         Dictionary of field names and values to associate with custom entity -**author_id** : `Optional[str]` +**author_id** : `str | Unset`         Author ID to associate with custom entity. Should be a string starting with "ent_" **if_exists** : `str`         Either "fail" or "update". If "fail", will raise an error if the entity already exists. ### Returns -`Dict[str, str]` +`dict[str, str]`         Dictionary with custom entity name as key and custom entity ID as value @@ -119,7 +119,7 @@ Bulk creates custom entities in Benchling.    If the entity does not e         Columns should include name_field and all fields in schema **name_field** : `str`         Name of column in custom_entities_dataframe containing entity names -**folder_id** : `Optional[str]` +**folder_id** : `str | None`         Folder ID containing Benchling entity to be created. This should be a string starting         with "lib_".    If updating a custom entity, specifying the folder ID is optional (if specified,         moves the custom entity to the specified folder). @@ -142,7 +142,7 @@ Bulk creates custom entities in Benchling.    If the entity does not e ### Returns -`Dict[str, List[str]]` +`dict[str, list[str]]`         Dictionary with keys "created" and "updated" and values of lists of custom entity IDs @@ -167,11 +167,11 @@ Creates molecule in Benchling.    If the entity does not exist, first         be a string starting with "src_" **naming_strategy** : `NamingStrategy`         Naming strategy to use when creating new entities. See NamingStrategy for more details. -**custom_entity_fields** : `Optional[Dict]` +**custom_entity_fields** : `dict | None`         Dictionary of field names and values to associate with custom entity -**author_id** : `Optional[str]` +**author_id** : `str | None`         Author ID to associate with custom entity. Should be a string starting with "ent_" -**molecule_fields** : `Optional[Dict]` +**molecule_fields** : `dict | None`         Dictionary of field names and values to associate with molecule **if_exists** : `str`         Either "fail" or "update". If "fail", will raise an error if the entity already exists. @@ -208,9 +208,9 @@ Bulk creates molecules in Benchling.    If the entity does not exist,         Whether to wait for the task to complete before returning **error_on_fail** : `bool`         Whether to raise an error if the task fails -**molecular_structure_field** : `Optional[str]` +**molecular_structure_field** : `str | None`         Name of column in molecules_dataframe containing molecular structure -**molecular_structure_format** : `Optional[str]` +**molecular_structure_format** : `str | None`         Format of molecular structure.    This should be one of ["smiles", "mol-v3000"] **check_for_existing** : `bool`         Whether or not to check for the existence of the entities in `name_field`. This is useful to turn off when you know you'll be creating new entities (for example with many naming strategies) and want to avoid many api calls for checking existence of molecules, @@ -219,7 +219,7 @@ Bulk creates molecules in Benchling.    If the entity does not exist, ### Returns -`Dict[str, List[str]]` +`dict[str, list[str]]`         Dictionary with keys "created" and "updated" and values of lists of molecule IDs @@ -230,7 +230,7 @@ Upload blob files to Benchling and return a dictionary of ### Parameters -**files** : `Dict[str, bytes]` +**files** : `dict[str, bytes]`         Files to upload to Benchling **process_file_names** : `bool`         Use to process file_name for benchling by converting string to lowercase and removing @@ -238,7 +238,7 @@ Upload blob files to Benchling and return a dictionary of ### Returns -`Dict[str, Union[Unset, str]]` +`dict[str, Unset | str]`         Returns a dictionary of IDs from benchling_sdk.models.Blob where the keys are the blob         names and the benchling IDs are the values @@ -292,14 +292,14 @@ Process input DataFrame into assay results for upload to Benchling. **error_on_empty_result** : `bool`         If True, raise an error if the DataFrame is empty. Default is True. **upload** : `bool` -        Whether to upload List[AssayResults] to Benchling +        Whether to upload list[AssayResults] to Benchling **\*\*kwargs**         Keyword args to pass to create_assay_result_from_dict -        drop_na (Optional[bool]) +        drop_na (bool | None) ### Returns -`List[AssayResultCreate]` +`list[AssayResultCreate]`         List of AssayResultCreate's to be uploaded to Benchling ### Example @@ -358,9 +358,9 @@ Transfer entities to a plate in Benchling.                 Volume of the entity         - volume_units: str                 Units of the volume. See benchling_sdk.models.ContainerQuantityUnits for options -        - concentration: Optional[float] +        - concentration: float | None                 Concentration of the entity -        - concentration_units: Optional[str] +        - concentration_units: str | None                 Units of the concentration                 Valid units are molar (M), mass (g/L), and count (cells/L) concentration units. Otherwise, use U/L units. See ContainerQuantityUnits for options to use in the form `f"\{mass or count unit\}/\{volume unit\}"` @@ -388,7 +388,7 @@ there are appropriate permissions to write to the notebook entry.         If you don't see "Copy API ID" as an option, click on your avatar, click Settings,         and scroll to the bottom and verify that         "Enable Copy API ID button" is checked. -**table_id** : `Optional[str]` +**table_id** : `str | None`         ID of the table to write to.    This should be a string starting with "strtbl_". You can         find this using b.get("entries", id="etr_") or b.get("entries", name="EXP12345678") in         the apiId field. The default is None. @@ -403,7 +403,7 @@ there are appropriate permissions to write to the notebook entry.         Whether to wait for the task to complete before returning **\*\*kwargs**         Keyword args to pass to create_assay_result_from_dict -        drop_na (Optional[bool]) +        drop_na (bool | None) ### Returns @@ -457,18 +457,18 @@ if passed the entry_template_id.         The ID of the folder where the entry will be created. **name** : `str`         The name of the new entry. -**author_ids** : `Union[str, List[str]], optional` +**author_ids** : `str | list[str] | Unset`         The IDs of the authors of the entry. -**custom_fields** : `models.CustomFields, optional` +**custom_fields** : `CustomFields | Unset`         Any custom fields to include in the entry. -**entry_template_id** : `str, optional` +**entry_template_id** : `str | Unset`         The ID of the template to use for the entry. The API ID can be copied from Template         Collections in Feature Settings. -**fields** : `models.Fields, optional` +**fields** : `Fields | Unset`         The fields to include in the entry. -**initial_tables** : `models.InitialTable, optional` +**initial_tables** : `InitialTable | list[InitialTable] | Unset`         The initial tables to include in the entry. -**schema_id** : `str, optional` +**schema_id** : `str | Unset`         The ID of the schema to use for the entry. ### Returns @@ -531,7 +531,7 @@ name. Can pass a dropdown id to look up the id of an option of a dropdown         Dropdown name to identify id for **\*args**         Optional positional arguments to pass to list method of benchling_sdk dropdown service -**dropdown_id** : `Optional[str]` +**dropdown_id** : `str | None`         If supplied, get the ID of an option in the dropdown specified by the dropdown_name. The         default is None. **\*\*kwargs** @@ -539,7 +539,7 @@ name. Can pass a dropdown id to look up the id of an option of a dropdown ### Returns -`Dict[str, str]` +`dict[str, str]`         Dictionary of dropdown name - dropdown id for a specified dropdown name. Raises an         error if dropdown id is not found @@ -565,7 +565,7 @@ List available Benchling services ### Returns -`List[str]` +`list[str]`         List of available Benchling services @@ -580,7 +580,7 @@ results data.         Benchling object service (e.g. - 'custom_entities', 'assay_results') **\*args**         Optional arguments to pass to list methods of the benchling service -**benchling_filter** : `Optional[Dict]` +**benchling_filter** : `dict | None`         Filter to apply to the list of Benchling objects **\*\*kwargs**         Optional keyword arguments to pass to list methods of the benchling service @@ -626,7 +626,7 @@ Get all Benchling objects of a specific type, optionally filtered by object attr         Optional arguments to pass to list methods of the benchling service **as_dict** : `bool`         Whether to return each Benchling object as a dictionary. Default is True. -**benchling_filter** : `Optional[Dict]` +**benchling_filter** : `dict | None`         Filter to apply to the list of Benchling objects **\*\*kwargs**         Optional keyword arguments to pass to list methods of the benchling service @@ -688,7 +688,7 @@ used in conjunction with Benchling.get() ### Parameters **benchling_service_results** : `str` -        Results of the form List[dict] returned from get() where "id" and "name" are common keys +        Results of the form list[dict] returned from get() where "id" and "name" are common keys         in the dict. **keys** : `str`         The key name of the inner dictionaries in the list used to set the key in the returned @@ -796,7 +796,7 @@ Archives assay results based on assay result schema and condition.         Assay result schema ID to archive; should start with 'assaysch_' **app_name** : `str, optional`         Creator of assay result to filter by, by default "Ganymede App" -**filter_fields** : `Dict[str, str], optional` +**filter_fields** : `dict[str, str], optional`         Fields on assay result to filter by, specified as a dictionary of field name to field display value **execute** : `bool`         Whether to execute the archive.    If False, will return assay results to archive. @@ -809,7 +809,7 @@ Archives assay results based on assay result schema and condition. ### Returns -`List[Dict]` +`list[dict]`         List of assay results to archive diff --git a/docs/sdk/markdowns/benchling.io.events.md b/docs/sdk/markdowns/benchling.io.events.md index 5e9e3d6f..9765ddd4 100644 --- a/docs/sdk/markdowns/benchling.io.events.md +++ b/docs/sdk/markdowns/benchling.io.events.md @@ -12,11 +12,11 @@ displayed_sidebar: SDKSidebar Representation of all inputs for a flow -files: Optional[List[FlowInputFile]] +files: list[FlowInputFile] | None         list of files for flow -params: Optional[List[FlowInputFile]] +params: list[FlowInputParam] | None         list of params for flow -tags_benchling: Optional[List[FlowInputFile]] +tags_benchling: list[BenchlingInputTag] | None         list of Benchling tags for flow @@ -58,7 +58,7 @@ Retrieve file based on Benchling field name         Node in triggered flow that will be receiving file input **triggered_flow_param_name** : `str`         Parameter for node in triggered flow -**file_extension** : `Optional[str]` +**file_extension** : `str | None`         File extension (if different from what is param name) ### Returns diff --git a/docs/sdk/markdowns/benchling.io.generic_read.md b/docs/sdk/markdowns/benchling.io.generic_read.md index 901852f4..e696f0bd 100644 --- a/docs/sdk/markdowns/benchling.io.generic_read.md +++ b/docs/sdk/markdowns/benchling.io.generic_read.md @@ -42,7 +42,7 @@ A page is of the form: benchling_sdk.helpers.pagination_helpers.PageIterator         Benchling context object **\*args**         Optional positional arguments to pass to list method of benchling_sdk dropdown service -**dropdown_id** : `Optional[str]` +**dropdown_id** : `str | None`         Optional benchling dropdown id. If none, list all dropdowns. If given, list all dropdowns of         that id. **\*\*kwargs** @@ -50,6 +50,6 @@ A page is of the form: benchling_sdk.helpers.pagination_helpers.PageIterator ### Returns -`List[dict]` +`list[dict]`         Flattened list of dropdown summaries converted to dicts from         benchling_sdk.models.DropdownSummary diff --git a/docs/sdk/markdowns/benchling.utils.mini_utils.md b/docs/sdk/markdowns/benchling.utils.mini_utils.md index 51d35650..0252b80f 100644 --- a/docs/sdk/markdowns/benchling.utils.mini_utils.md +++ b/docs/sdk/markdowns/benchling.utils.mini_utils.md @@ -14,11 +14,11 @@ This method will flatten nested lists while preserving the original order. ### Parameters -**lists** : `Iterator[Union[List, List[list]]]` +**lists** : `Iterator[list | list[list]]]`         Nested lists -**benchling_filter** : `Optional[Dict]` +**benchling_filter** : `dict | None`         Optional filter to apply to the flattened list -**as_dict** : `Optional[bool]` +**as_dict** : `bool`         Optional flag to return each item as a dictionary. Default is True ### Returns @@ -59,7 +59,7 @@ Fill null values to to pass Benchling validations         Input DataFrame **string_fill_na** : `str`         Value to replace NA values with, default is "" -**numeric_fill_na** : `Optional[int]` +**numeric_fill_na** : `int | None`         Optional value to replace numeric na with. Default is None. Will raise a ValueError if         numeric NA's are detected @@ -80,7 +80,7 @@ Serialize dictionary for transmission to Benchling ### Returns -`Union[benchling_sdk.helpers.serialization_helpers.D, Unset]` +`Fields`         Serialized dictionary @@ -99,7 +99,6 @@ Check if an entity exists in Benchling ### Returns -`Optional` `benchling_sdk.models`         The entity if it exists, otherwise None diff --git a/docs/sdk/markdowns/benchling.utils.write_entity.md b/docs/sdk/markdowns/benchling.utils.write_entity.md index ba8edc58..6ef52afc 100644 --- a/docs/sdk/markdowns/benchling.utils.write_entity.md +++ b/docs/sdk/markdowns/benchling.utils.write_entity.md @@ -43,27 +43,27 @@ Creates and returns the id of the custom entity.         Benchling context object to get connection **entity_name** : `str`         Name of new entity to be created -**folder_id** : `str` +**folder_id** : `str | Unset`         Folder ID containing Benchling entity to be created. This should be a string starting with         "lib_" -**input_schema_id** : `str` +**input_schema_id** : `str | Unset`         Input schema ID Tag. schema associated with Benchling entity to be created This should be         a string starting with "ts_" -**registry_id** : `str` +**registry_id** : `str | Unset`         Project associated with custom entity.    This identifies the registry that your run entity         will be registered to.    This can be found by clicking on Avatar - Feature Settings -         Registry Settings, and you will be able to find it in the URL.    This should be a string         starting with "src_" -**custom_entity_fields** : `Optional[Dict]` +**custom_entity_fields** : `dict | None`         Fields associated with Benchling entity -**author_id** : `Optional[str]` +**author_id** : `str | None`         Author ID to associate with custom entity. Should be a string starting with "ent_" **naming_strategy** : `NamingStrategy`         Strategy to use for naming the entity.    See NamingStrategy for more details. ### Returns -`Dict[str, str]` +`dict[str, str]`         Dictionary with custom entity name as key and custom entity ID as value ### Raises @@ -93,12 +93,12 @@ Updates custom entity         will be registered to.    This can be found by clicking on Avatar - Feature Settings -         Registry Settings, and you will be able to find it in the URL.    This should be a string         starting with "src_" -**custom_entity_fields** : `Optional[Dict]` +**custom_entity_fields** : `dict | None`         Fields associated with Benchling entity ### Returns -`Dict[str, str]` +`dict[str, str]`         Dictionary of entity name, custom entity id pair @@ -111,9 +111,9 @@ values are the benchling IDs **benchling_context** : `Benchling Context object`         Benchling context object to get connection -**file_name** : `List[str]` +**file_name** : `list[str]`         File names to associate to file content -**file_content** : `Union[List[bytes], List[pd.DataFrame]]` +**file_content** : `list[bytes] | list[pd.DataFrame]`         Blob or data content to upload **keys_as_benchling_name** : `bool`         Use to process file_name for benchling by converting string to lowercase and removing @@ -121,7 +121,7 @@ values are the benchling IDs ### Returns -`Dict[str, Union[Unset, str]]` +`dict[str, Unset | str]`         Returns a dictionary of IDs from benchling_sdk.models.Blob @@ -140,7 +140,7 @@ Get output file id for blob or data created from bytes. ### Returns -`Union[Unset, str]` +`str`         Returns the id of benchling_sdk.models.Blob @@ -159,7 +159,7 @@ Get output file id for blob created from bytes. ### Returns -`Union[Unset, str]` +`str`         Returns the id of benchling_sdk.models.Blob @@ -178,7 +178,7 @@ Get output file id for blob created from bytes. ### Returns -`Union[Unset, str]` +`str`         Returns the id of benchling_sdk.models.Blob ### Notes @@ -217,11 +217,11 @@ Processes input DataFrame for upload to Benchling.         If True, raise an error if the DataFrame is empty. Default is True. **\*\*kwargs**         Keyword args to pass to create_assay_result_from_dict -        drop_na (Optional[bool]) +        drop_na (bool | None) ### Returns -`List[AssayResultCreate]` +`list[AssayResultCreate]`         List of AssayResultCreate's to be uploaded to Benchling @@ -237,7 +237,7 @@ Set up AssayResultCreate from dictionary         Benchling schema id **project_id** : `str`         Benchling project id -**drop_na** : `Optional[bool]` +**drop_na** : `bool`         Determine if you should remove na from benchling results dict ### Returns @@ -260,9 +260,9 @@ Create a custom entity object to create in Benchling         ID of the entity schema to use **registry_id** : `str`         ID of the registry to use -**fields** : `Optional[Dict]` +**fields** : `dict | None`         Dictionary of fields to add to the entity -**author_id** : `Optional[str]` +**author_id** : `str | None`         ID of the author to assign to the entity **naming_strategy** : `NamingStrategy`         Naming strategy to use for the entity, by default NamingStrategy.NEW_IDS @@ -330,13 +330,13 @@ Update a custom entity object to update in Benchling **existing_entity** : `CustomEntity`         Existing custom entity to update -**folder_id** : `Optional[str]` +**folder_id** : `str | None`         ID of the folder to move the custom entity to (if specified) **input_schema_id** : `str`         ID of the entity schema to use -**fields** : `Optional[Dict]` +**fields** : `dict | None`         Dictionary of fields to add to the entity -**author_id** : `Optional[str]` +**author_id** : `str | None`         ID of the author to assign to the entity ### Returns diff --git a/docs/sdk/markdowns/benchling.utils.write_molecule.md b/docs/sdk/markdowns/benchling.utils.write_molecule.md index bf471966..794c176d 100644 --- a/docs/sdk/markdowns/benchling.utils.write_molecule.md +++ b/docs/sdk/markdowns/benchling.utils.write_molecule.md @@ -22,9 +22,9 @@ Create a molecule object to create in Benchling         ID of the molecule schema to use **registry_id** : `str`         ID of the registry to use -**fields** : `Optional[Dict]` +**fields** : `dict | None`         Dictionary of fields to add to the molecule -**author_id** : `Optional[str]` +**author_id** : `str | None`         ID of the author to assign to the molecule **naming_strategy** : `NamingStrategy`         Naming strategy to use for the molecule, by default NamingStrategy.NEW_IDS @@ -59,7 +59,7 @@ Args:         error_on_fail (bool, optional): Whether to raise an error if the task fails. Defaults to True. Returns: -        List[str]: A list of task IDs for each bulk create request. +        list[str]: A list of task IDs for each bulk create request. ## `function` update_molecules_bulk @@ -101,14 +101,14 @@ Create a molecule update object to update in Benchling         ID of the folder to create the entity in **input_schema_id** : `str`         ID of the schema to use for the entity -**fields** : `Optional[Dict]` +**fields** : `dict | None`         Dictionary of fields to add to the entity, by default None -**author_id** : `Optional[str]` +**author_id** : `str | None`         ID of the author to assign to the entity, by default None **bulk_update** : `bool`         Whether to return a MoleculeBulkUpdate object instead of a MoleculeUpdate object ### Returns -`Dict[str, str]` +`dict[str, str]`         Dictionary containing the name and ID of the created entity diff --git a/docs/sdk/markdowns/coda.coda.md b/docs/sdk/markdowns/coda.coda.md index baf02664..29fb42a5 100644 --- a/docs/sdk/markdowns/coda.coda.md +++ b/docs/sdk/markdowns/coda.coda.md @@ -20,7 +20,7 @@ A class to interact with the Coda API.         The maximum number of records that can be requested from the Coda API at once. **coda_api_address** : `str`         The base URL of the Coda API. -**authorization_header** : `str` +**authorization_header** : `dict[str, str]`         The authorization header used in requests to the Coda API. @@ -44,7 +44,7 @@ Retrieve data from Coda API **endpoint** : `str`         Endpoint to retrieve data from -**params** : `Dict, optional` +**params** : `dict, optional`         Key/Value parameters to pass to get method **limit** : `int, optional`         Maximum number of records to request, by default None @@ -53,8 +53,8 @@ Retrieve data from Coda API ### Returns -`Dict` -        Coda API response +`list[requests.Response]` +        List of Coda API responses ## `function` Coda.post @@ -65,14 +65,14 @@ Post data to Coda API **endpoint** : `str`         Endpoint to retrieve data from -**data** : `Dict` +**data** : `dict`         Data to post to endpoint **sleep_time** : `float, optional`         Time in seconds to wait after posting data before checking response ### Returns -`Dict` +`requests.Response`         Coda API response @@ -84,14 +84,14 @@ Puts data in Coda API **endpoint** : `str`         Endpoint to push data to -**data** : `Dict` +**data** : `dict`         Data to post to endpoint **sleep_time** : `float, optional`         Time in seconds to wait after putting data before checking response ### Returns -`Dict` +`requests.Response`         Coda API response @@ -110,7 +110,7 @@ Get IDs from Coda API ### Returns -`List` +`list`         List of IDs retrieved from Coda API @@ -120,7 +120,7 @@ Convenience method for listing documents ### Returns -`List` +`list`         Document objects @@ -135,7 +135,7 @@ Convenience method for listing table IDs associated with a document template ### Returns -`List` +`list`         Tables within document @@ -150,7 +150,7 @@ Convenience method for listing pages within document ### Returns -`Dict` +`dict`         Pages listing within document @@ -166,7 +166,7 @@ Creates a browser link to a Coda doc containing additional metadata ### Returns -`Dict` +`list[requests.Response]`         Metadata associated with browser link submitted @@ -183,7 +183,7 @@ Convenience method for listing columns within a table ### Returns -`Dict` +`list`         Columns listing within table @@ -197,12 +197,12 @@ Upserting rows into a table in Coda         Coda Document ID **table_id_or_name** : `str`         Coda table ID or name -**data** : `Dict` +**data** : `dict`         Data to post ### Returns -`Dict` +`dict`         Response from posting data @@ -230,8 +230,8 @@ Uploads 1 set of files as a row to a table in Coda         Coda Document ID **table_id_or_name** : `str`         Coda table ID or name -**coda_to_signed_url_map** : `Dict[str, str]` -        Dictionary containing Ganymede filepaths as keys and Coda column id as values +**coda_to_signed_url_map** : `dict[str, str]` +        dictionary containing Ganymede filepaths as keys and Coda column id as values ## `function` Coda.create_doc_from_template @@ -245,5 +245,5 @@ Convenience method for creating document from template ### Returns -`Dict` +`requests.Response`         Response from call to create document from template diff --git a/docs/sdk/markdowns/fcs.md b/docs/sdk/markdowns/fcs.md index e3a371e4..d1116036 100644 --- a/docs/sdk/markdowns/fcs.md +++ b/docs/sdk/markdowns/fcs.md @@ -4,48 +4,48 @@ title: fcs displayed_sidebar: SDKSidebar --- - +# fcs ## `class` WSP A class to parse a FlowJo WSP file following the Gating-ML 2.0 standard.    Contains methods for extracting gates, table output structure, population statistics, and compensation matrices from WSP file. - -```python + +```bash >>> wsp = WSP(wsp_bytes) ``` -### View gates +### View gates -```python +```bash >>> wsp.gates ``` ### View population statistics computed in FlowJo -```python +```bash >>> wsp.stats_flowjo ``` ### View table output structure, as configured in Table Editor in FlowJo -```python +```bash >>> wsp.table_output_structure ``` ### Generate output table based on table output structure and FlowJo population statistics -```python ->>> WSP.apply_table_output_structure( - wsp.table_output_structure["Table"][["gate", "value_type", "statistic", "rename"]], - wsp.stats_flowjo -) +```bash +>>> WSP.apply_table_output_structure( + wsp.table_output_structure["Table"][["gate", "value_type", "statistic", "rename"]], + wsp.stats_flowjo) ``` With corresponding FCS files, you can calculate population counts from FCS events data using the _apply_compensation_ and _calculate_population_counts_ methods. + ## `function` WSP.\_\_init\_\_ Setup parsed WSP object, which contains gates, population statistics, and @@ -56,6 +56,7 @@ table output structure as configured in Table Editor in FlowJo **wsp_bytes** : `bytes or file_like object`         FlowJo Workspace file in bytes + ## `function` WSP.get_population_stats Retrieve population statistics from WSP file @@ -65,15 +66,17 @@ Retrieve population statistics from WSP file `pd.DataFrame`         Population statistics as a DataFrame + ## `function` WSP.get_gates Retrieve gates from WSP file -### Returns +### Returns `pd.DataFrame`         Gates as a DataFrame + ## `function` WSP.get_single_well_stats Retrieve populations for a given well from a WSP file.    To use this method, loop through the @@ -91,9 +94,10 @@ sure to capture the unique identifier for each well in the loop. ### Returns -`Dict` +`dict`         Dictionary containing single well stats requested + ## `function` WSP.get_table_output_structure Retrieve table structure as specified in Table Editor of FlowJo @@ -108,6 +112,7 @@ Retrieve table structure as specified in Table Editor of FlowJo `pd.DataFrame`         DataFrame consisting of gate and tables + ## `function` WSP.apply_table_output_structure Apply table structure as specified in Table Editor of FlowJo to get population proportions @@ -125,8 +130,8 @@ Apply table structure as specified in Table Editor of FlowJo to get population p         population (series of gates applied to determine population)         statistic_name (name of statistic (e.g. count, MFI, etc.))         statistic_value (value of statistic) -**unique_id** : `Union[List[str], str]` -        Unique identifying column names(s) for single flow run.    Commonly, this would be well position, +**unique_id** : `list[str] | str` +        Unique identifying column names(s) for single flow run. Commonly, this would be well position,         well position + timestamp, or filename. ### Returns @@ -155,12 +160,12 @@ observed in file ### Parameters -**population_name** : `List[str]` +**population_name** : `list[str]`         List of populations, delimited by '/' ### Returns -`List[str]` +`list[str]`         List of populations in order ### Example @@ -182,15 +187,17 @@ Retrieve gate structure as a nested dict from a WSP file **node** : `etree.Element`         XML node to start traversal in, typically node corresponding to well + ## `function` WSP.get_compensation_matrices Retrieve compensation matrices from WSP file as a DataFrame ### Returns -`Dict[str, pd.DataFrame]` +`dict[str, pd.DataFrame]`         Dictionary of compensation matrices, keyed by matrix name + ## `function` WSP.apply_compensation Adjust flow cytometry data for compensation @@ -201,7 +208,7 @@ Adjust flow cytometry data for compensation         DataFrame containing FCS event data **compensation_matrix** : `pd.DataFrame`         DataFrame containing compensation matrix; rows are from, columns are to -**compensation_channels** : `List[str]` +**compensation_channels** : `list[str] | None`         Fluorescent channels to adjust for compensation ### Returns @@ -209,6 +216,7 @@ Adjust flow cytometry data for compensation `pd.DataFrame`         DataFrame containing compensated events + ## `function` WSP.calculate_population_counts Calculate population counts from FCS events data, using compensation matrix @@ -231,7 +239,7 @@ Calculate population counts from FCS events data, using compensation matrix **compensation_matrix** : `pd.DataFrame`         Compensation matrix as a DataFrame; rows are from labels, columns are to labels -**fcs_files_dict** : `Dict[str, bytes]` +**fcs_files_dict** : `dict[str, bytes]`         Dictionary of FCS files as bytes, keyed by filename ### Returns @@ -239,6 +247,7 @@ Calculate population counts from FCS events data, using compensation matrix `pd.DataFrame`         DataFrame containing filename, population name, and counts + ## `class` FCS A class to represent an FCS (Flow Cytometry Standard) file. @@ -261,6 +270,7 @@ A class to represent an FCS (Flow Cytometry Standard) file. **parse_metadata(metadata: dict)**         Parses FCS file metadata. + ## `function` FCS.parse_metadata Parses FCS file metadata @@ -288,6 +298,7 @@ channels: pd.DataFrame channel_names: Tuple         contains flow cytometer channel names + ## `function` FCS.gate_population_polygon Gate FCS file using a polygon @@ -308,6 +319,7 @@ Gate FCS file using a polygon `pd.DataFrame`         DataFrame containing gated population events + ## `function` FCS.gate_population_limit Gate FCS file using a limit @@ -328,6 +340,7 @@ Gate FCS file using a limit `pd.DataFrame`         DataFrame containing gated population events + ## `function` FCS.create_comp_matrix_xml Creates XML representation of compensation matrix @@ -351,9 +364,9 @@ Creates XML representation of compensation matrix ```python For a WSP object, insert a compensation matrix into the XML tree by executing the steps shown below: ->>> xml_comp_mat = WSP.create_comp_matrix_xml(df_comp_matrix) ->>> matrix = xml_comp_mat.xpath("./transforms:spilloverMatrix", namespaces=WSP.ns_transforms).pop() ->>> wsp_obj.wsp_root.xpath("./Matrices")[0].append(xml_comp_mat) +>>>> xml_comp_mat = WSP.create_comp_matrix_xml(df_comp_matrix) +>>>> matrix = xml_comp_mat.xpath("./transforms:spilloverMatrix", namespaces=WSP.ns_transforms).pop() +>>>> wsp_obj.wsp_root.xpath("./Matrices")[0].append(xml_comp_mat) In this example, df_comp_matrix is a Pandas DataFrame with a format mirroring an element from an item in the 'compensation_matrices' dictionary returned by the 'get_compensation_matrices' method. diff --git a/docs/sdk/markdowns/lc.cdf.md b/docs/sdk/markdowns/lc.cdf.md index 121a33c4..271cc423 100644 --- a/docs/sdk/markdowns/lc.cdf.md +++ b/docs/sdk/markdowns/lc.cdf.md @@ -132,7 +132,7 @@ Append peak information to an existing CDF file         If False, append data to the existing CDF file -## `function` CDFOutput.__init__ +## `function` CDFOutput.\_\_init\_\_ Initialize LCOutput object for outputting data from netCDF file in AIA format @@ -142,7 +142,7 @@ Initialize LCOutput object for outputting data from netCDF file in AIA format         Name of the netCDF file **cdf** : `nc.Dataset`         netCDF file containing LC run data in AIA format -**baseline** : `np.array, optional` +**baseline** : `np.ndarray, optional`         Array containing baseline data @@ -152,7 +152,7 @@ Outputs the data from the netCDF file to an Excel file ### Parameters -**filename** : `str` +**filename** : `str | None`         Name of the file to write to         If None, the method will return a tuple of DataFrames (peak_purity, peak_stats) **mode** : `str` diff --git a/docs/sdk/markdowns/lc.lc.md b/docs/sdk/markdowns/lc.lc.md index ef2f2f18..6726cabc 100644 --- a/docs/sdk/markdowns/lc.lc.md +++ b/docs/sdk/markdowns/lc.lc.md @@ -20,7 +20,7 @@ Reads liquid chromatography runs in CDF format         Get elution peaks from raw chromatogram data -## `function` LC.__init__ +## `function` LC.\_\_init\_\_ Initialize LC class diff --git a/docs/sdk/markdowns/lc.md b/docs/sdk/markdowns/lc.md deleted file mode 100644 index 512f4adc..00000000 --- a/docs/sdk/markdowns/lc.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -sidebar_label: lc -title: lc -displayed_sidebar: SDKSidebar ---- - - -# lc - - -## `class` LC - -Reads liquid chromatography runs in CDF format - -### Methods - -**from_cdf(cdf_data: bytes)** -        Reads in CDF data from Agilent LC and stores it in the class -**get_elution_peaks(chromatogram: pd.Series, peak_prominence_threshold=10, num_peaks_to_find=None)** -        Get elution peaks from raw chromatogram data - - -## `function` LC.__init__ - -Initialize LC class - - -## `function` LC.from_cdf - -Reads in CDF data from Agilent LC and stores it in the class - -### Parameters - -**cdf_data** : `bytes` -        CDF data in bytes - - -## `function` LC.get_elution_peaks - -Get elution peaks from raw chromatogram data - -### Parameters - -**chromatogram** : `pd.DataFrame` -        Pandas Series with time as index -**peak_prominence_threshold** : `int, optional` -        Minimum prominence of peaks to be considered, by default 10 -**num_peaks_to_find** : `int, optional` -        Number of peaks to find - - -## `function` LC.decode_peaks_name - -Decodes the peak names from bytes to strings; move peak name to front of DataFrame - -### Parameters - -**df_peaks** : `pd.DataFrame` -        Dataframe containing peak names - -### Returns - -`pd.DataFrame` -        Dataframe with peak names decoded - - -## `function` LC.get_cdf_dataframes - -Loads in the cdf data and proceeses it into dataframes, extracting the chromotography data, -peak info table, and metadata. - -### Parameters - -**cdf_data** : `bytes` -        CDF data in bytes containing HPLC run data - -### Returns - -`Dict[str, pd.DataFrame]` -        Dictionary containing dataframes for metadata, peak info, and chromatography data - - -## `function` LC.compare_aia_files - -Compare 2 AIA files, printing out differences - -### Parameters - -**ds_base** : `nc.Dataset` -        Base AIA file -**ds_cmp** : `nc.Dataset` -        Comparison AIA file - - -## `function` LC.verify_aia_file - -Verifies AIA fields against spec.    E1948-98 is the original spec. - -### Parameters - -**ds** : `nc.Dataset` -        AIA file -**spec** : `str` -        Spec to validate against diff --git a/docs/sdk/markdowns/signals.signals.md b/docs/sdk/markdowns/signals.signals.md index 9187539e..fe60eecc 100644 --- a/docs/sdk/markdowns/signals.signals.md +++ b/docs/sdk/markdowns/signals.signals.md @@ -49,7 +49,7 @@ Convert string container position to grid coordinates ### Returns -`list` +`list[int, int]`         Container position in grid coordinates (e.g. - [0, 0]) @@ -57,9 +57,6 @@ Convert string container position to grid coordinates ## `function` Signals.create_container Create a container in Signals -### Notes - -Container contents cannot be modified once created, except for adjusting amount. ### Parameters @@ -76,9 +73,9 @@ Container contents cannot be modified once created, except for adjusting amount.         ID of the plate location to create the container in **batch_id** : `str`         ID of the batch to add to the container -**x_coord** : `int` +**x_coord** : `int | None`         X coordinate of the container inside a matrix plate -**y_coord** : `int` +**y_coord** : `str | None`         Y coordinate of the container inside a matrix plate **barcode** : `str`         Barcode of the container @@ -89,11 +86,14 @@ Container contents cannot be modified once created, except for adjusting amount. **container_fields** : `list, optional`         List of dictionaries containing fields to add to the container, by default [] - ### Returns `str`         ID of the created container + +### Notes + +Container contents cannot be modified once created, except for adjusting amount. ## `function` Signals.create_location diff --git a/pydoc/generateOperatorDocs.py b/pydoc/generateOperatorDocs.py index b20c7f2c..45aa0fd6 100644 --- a/pydoc/generateOperatorDocs.py +++ b/pydoc/generateOperatorDocs.py @@ -17,13 +17,21 @@ "Examples", ] +params_to_highlight = [ + "tables_to_upload", + "files_to_upload", + "downstream_nodes_to_execute", + "src_azure_blob_name", +] +params_to_highlight_str = "(" + "|".join(params_to_highlight) + "):" + -def extract_docstring(filename: str, search_str: str = "class", python_spaces: str = 4) -> List: +def extract_docstring(filename: str, search_str: str = "class", python_spaces: int = 4) -> List: """ Extracts docstring from Ganymede operator class """ - docstrings = [] + docstrings: list = [] is_comment = False search_str_found = False lines = open(filename, "r").readlines() @@ -41,6 +49,9 @@ def extract_docstring(filename: str, search_str: str = "class", python_spaces: s if "branch" in filename.lower(): print(line) + if re.search(params_to_highlight_str, line): + line = re.sub(r"([^\s:]+): (.+)", r"**\1**: `\2`", line) + if prev_line.strip().endswith(":") and line_stripped.startswith('"""'): is_comment = True elif line_stripped.endswith('"""') and is_comment: @@ -109,7 +120,7 @@ def extract_docstring(filename: str, search_str: str = "class", python_spaces: s shutil.rmtree(os.path.join(markdown_dir, path)) # copy markdown files to subdirectory by operator type - missing_files = [] + missing_files: list = [] for name, desc in operators.items(): operator_filename = ( os.path.join(operators_dir, "/".join(desc["path"].split(".")[1:-1])) + ".py" diff --git a/pylib b/pylib index 81da6974..857ec76f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 81da697472af10562b05b90c0d68c4e78b3b7d62 +Subproject commit 857ec76fe0435df30844890e68a41640372649f6