From 7730d424ae13e32ab928277364561e76c9af5f80 Mon Sep 17 00:00:00 2001 From: Pan Date: Mon, 25 Nov 2024 09:26:37 -0500 Subject: [PATCH 1/3] update gh actions --- .github/workflows/check_links.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 .github/workflows/check_links.yml diff --git a/.github/workflows/check_links.yml b/.github/workflows/check_links.yml deleted file mode 100644 index 5b3d487..0000000 --- a/.github/workflows/check_links.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Check Links - -on: - push: - branches: - - '*' - pull_request: - branches: - - '*' -jobs: - check-links: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: 16 - - - name: Install dependencies - run: | - npm install -g markdown-link-check - - - name: Check links in Markdown files - run: find . -name '*.md' -print0 | xargs -0 -n1 markdown-link-check -q -c .markdown-link-check.json From 42aa387d061a6c8a1f67874a8495d099bea2b192 Mon Sep 17 00:00:00 2001 From: Pan Date: Mon, 25 Nov 2024 09:26:59 -0500 Subject: [PATCH 2/3] update gh-action --- .github/workflows/check-links-self.yaml | 12 ++++++++++++ .github/workflows/notebook-lint-self.yaml | 14 ++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 .github/workflows/check-links-self.yaml create mode 100644 .github/workflows/notebook-lint-self.yaml diff --git a/.github/workflows/check-links-self.yaml b/.github/workflows/check-links-self.yaml new file mode 100644 index 0000000..2c4b225 --- /dev/null +++ b/.github/workflows/check-links-self.yaml @@ -0,0 +1,12 @@ +name: 'Check Links' +on: + workflow_dispatch: + push: + pull_request: + +jobs: + link_check: + name: 'Link Check' + uses: STRIDES/NIHCloudLab/.github/workflows/check-links.yaml@main + with: + repo_link_ignore_list: "" diff --git a/.github/workflows/notebook-lint-self.yaml b/.github/workflows/notebook-lint-self.yaml new file mode 100644 index 0000000..9688b9b --- /dev/null +++ b/.github/workflows/notebook-lint-self.yaml @@ -0,0 +1,14 @@ +name: 'Lint Notebook' +on: + push: + workflow_dispatch: +permissions: + contents: write + id-token: write + +jobs: + lint: + name: 'Linting' + uses: STRIDES/NIHCloudLab/.github/workflows/notebook-lint.yaml@main + with: + directory: . From a343fe853f66bfd05a24d2a66bb309f9bc821e87 Mon Sep 17 00:00:00 2001 From: github-action Date: Mon, 25 Nov 2024 14:27:24 +0000 Subject: [PATCH 3/3] Github Action: Lint Notebooks --- .../DL-gwas-gcp-example/1-d10-run-first.ipynb | 483 +----------------- .../2-mse-run-second-in-jupyter.ipynb | 466 +---------------- notebooks/GWASCoatColor/GWAS_coat_color.ipynb | 32 +- notebooks/GenAI/GCP_Agent_Builder.ipynb | 68 +-- .../GenAI/GCP_Code_Chatbot_wGrounding.ipynb | 30 +- notebooks/GenAI/GCP_GenAI_Huggingface.ipynb | 138 ++--- notebooks/GenAI/GCP_Grounding.ipynb | 54 +- notebooks/GenAI/GCP_MedLM_Intro.ipynb | 67 +-- notebooks/GenAI/Gemini_Intro.ipynb | 104 +--- notebooks/GenAI/Google_Drive_chatbot.ipynb | 60 +-- notebooks/GenAI/Pubmed_RAG_chatbot.ipynb | 119 +---- notebooks/GenAI/VertexAIStudioGCP.ipynb | 54 +- notebooks/GenAI/langchain_on_vertex.ipynb | 69 +-- .../nextflow/Part1_GBatch_Nextflow.ipynb | 137 +---- .../nextflow/Part2_GBatch_Nextflow.ipynb | 32 +- .../nextflow/Part1_LS_API_Nextflow.ipynb | 142 +---- .../nextflow/Part2_LS_API_Nextflow.ipynb | 30 +- .../snakemake/LS_API_Snakemake.ipynb | 44 +- notebooks/SRADownload/SRA-Download.ipynb | 35 +- .../SpleenSeg_Pretrained-4_27.ipynb | 32 +- .../elasticBLAST/run_elastic_blast.ipynb | 31 +- .../ncbi-stat-tutorial/STAT-tutorial.ipynb | 31 +- notebooks/pangolin/pangolin_pipeline.ipynb | 81 +-- 23 files changed, 310 insertions(+), 2029 deletions(-) diff --git a/notebooks/DL-gwas-gcp-example/1-d10-run-first.ipynb b/notebooks/DL-gwas-gcp-example/1-d10-run-first.ipynb index 26a4057..012e835 100644 --- a/notebooks/DL-gwas-gcp-example/1-d10-run-first.ipynb +++ b/notebooks/DL-gwas-gcp-example/1-d10-run-first.ipynb @@ -2,6 +2,7 @@ "cells": [ { "cell_type": "markdown", + "id": "51e9938a", "metadata": {}, "source": [ "# Deep Learning GWAS with Kubefl" @@ -9,6 +10,7 @@ }, { "cell_type": "markdown", + "id": "3962d7d9", "metadata": {}, "source": [ "## Overview\n", @@ -17,6 +19,7 @@ }, { "cell_type": "markdown", + "id": "733e4582", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -27,6 +30,7 @@ }, { "cell_type": "markdown", + "id": "6c9fe14d", "metadata": {}, "source": [ "## Prerequisites\n", @@ -35,6 +39,7 @@ }, { "cell_type": "markdown", + "id": "ecae558c", "metadata": {}, "source": [ "## Get Started" @@ -42,6 +47,7 @@ }, { "cell_type": "markdown", + "id": "19f7ab6c", "metadata": {}, "source": [ "### Install packages" @@ -51,9 +57,7 @@ "cell_type": "code", "execution_count": null, "id": "ddee0923-844f-42c8-9273-7e32d7178628", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -69,11 +73,7 @@ "cell_type": "code", "execution_count": null, "id": "7352040d-d4ca-41af-81ca-8df30e7de6cb", - "metadata": { - "tags": [ - "skip" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# mark in Kale as skip\n", @@ -91,11 +91,7 @@ "cell_type": "code", "execution_count": null, "id": "619de0db-3878-4254-93a4-4404553d0ecf", - "metadata": { - "tags": [ - "imports" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -116,11 +112,7 @@ "cell_type": "code", "execution_count": null, "id": "42103b5f-1c53-4047-99ba-690e5bb7d398", - "metadata": { - "tags": [ - "skip" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# Set time in the pipeline parameters and in Katib (as string and the only value) to the output of this\n", @@ -129,6 +121,7 @@ }, { "cell_type": "markdown", + "id": "cd1c927b", "metadata": {}, "source": [ "### Begin Kayle Analysis" @@ -138,11 +131,7 @@ "cell_type": "code", "execution_count": null, "id": "fd604bf6-0d00-4178-9deb-28465c915c29", - "metadata": { - "tags": [ - "pipeline-parameters" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# mark in Kayle as pipeline parameters\n", @@ -195,11 +184,7 @@ "cell_type": "code", "execution_count": null, "id": "e06747ae-77b6-4524-b803-fc0d87512fbb", - "metadata": { - "tags": [ - "block:preprocessing" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# mark in Kayle as pipeline step: \"preprocessing\": Depends on none\n", @@ -259,12 +244,7 @@ "cell_type": "code", "execution_count": null, "id": "dd01e8b2-78ba-4315-9e12-9153b1c57ce5", - "metadata": { - "tags": [ - "block:train", - "prev:preprocessing" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# mark in Kayle as pipeline step \"train\": depends on \"data-preprocessing\"\n", @@ -429,12 +409,7 @@ "cell_type": "code", "execution_count": null, "id": "74954dbc-9133-40d5-9f23-400a39807724", - "metadata": { - "tags": [ - "block:saliency_observed", - "prev:preprocessing" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# mark in Kayle as pipeline step: \"saliency - known\": depends on \"data-preprocessing\"\n", @@ -464,12 +439,7 @@ "cell_type": "code", "execution_count": null, "id": "464ffbb4-eb9e-438e-ac1e-292a2e45a99f", - "metadata": { - "tags": [ - "block:manhattan_observed", - "prev:saliency_observed" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -494,12 +464,7 @@ "cell_type": "code", "execution_count": null, "id": "f18fae63-9e34-4d5b-b5d7-76e08e94142c", - "metadata": { - "tags": [ - "block:saliency_predicted", - "prev:train" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -535,12 +500,7 @@ "cell_type": "code", "execution_count": null, "id": "043678ab-c0ae-45d7-8fdc-259d3138adba", - "metadata": { - "tags": [ - "block:manttan_predcted", - "prev:saliency_predicted" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -565,13 +525,7 @@ "cell_type": "code", "execution_count": null, "id": "bae392ab-a377-49a1-afaf-b5330892d89a", - "metadata": { - "tags": [ - "block:qq_plot", - "prev:saliency_observed", - "prev:saliency_predicted" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -603,9 +557,7 @@ { "cell_type": "markdown", "id": "be03ced0-99f2-43fd-b29e-407d0f85212c", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "\n", "## Conclusion\n", @@ -616,11 +568,7 @@ "cell_type": "code", "execution_count": null, "id": "4b240cfb-b264-49dc-ae2c-73b10ef8a69d", - "metadata": { - "tags": [ - "pipeline-metrics" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -628,394 +576,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "tf2-gpu.2-10.m100", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-10:m100" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "kubeflow_notebook": { - "autosnapshot": true, - "deploy_config": {}, - "docker_image": "", - "experiment": { - "id": "new", - "name": "gwas-7-d" - }, - "experiment_name": "gwas-10-d", - "katib_metadata": { - "algorithm": { - "algorithmName": "bayesianoptimization", - "algorithmSettings": [ - { - "name": "random_state", - "value": "10" - }, - { - "name": "acq_optimizer", - "value": "auto" - }, - { - "name": "acq_func", - "value": "gp_hedge" - }, - { - "name": "base_estimator", - "value": "GP" - } - ] - }, - "maxFailedTrialCount": 10, - "maxTrialCount": 40, - "objective": { - "additionalMetricNames": [], - "goal": 0.05, - "objectiveMetricName": "val-mean-absolute-error", - "type": "minimize" - }, - "parallelTrialCount": 2, - "parameters": [ - { - "feasibleSpace": { - "list": [ - "IMP_height.txt" - ] - }, - "name": "data_file_to_run", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "list": [ - "GWAS on soy height" - ] - }, - "name": "experiment_description", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "max": "0.3", - "min": "0.00001", - "step": "0.00001" - }, - "name": "learning_rate", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": ".98", - "min": "0.0001", - "step": "0.0001" - }, - "name": "conv_1_dropout_rate", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_1_kernel_l1", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_1_kernel_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_1_bias_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_1_activity_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_x_kernel_l1", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_x_kernel_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_x_bias_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_x_activity_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_x_kernel_l1", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_x_kernel_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_x_bias_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_x_activity_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_out_kernel_l1", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_out_kernel_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_out_bias_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": ".0000001", - "step": "0.0000001" - }, - "name": "dense_out_activity_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "list": [ - "TruncatedNormal", - "glorot_uniform", - "GlorotNormal", - "HeNormal", - "random_normal" - ] - }, - "name": "conv_initializer", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "list": [ - "TruncatedNormal", - "glorot_uniform", - "GlorotNormal", - "HeNormal", - "random_normal" - ] - }, - "name": "dese_initializer", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.01", - "step": "0.1" - }, - "name": "dropout_rate", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "5", - "min": "1", - "step": "1" - }, - "name": "num_dense_layers", - "parameterType": "int" - }, - { - "feasibleSpace": { - "max": "20", - "min": "1", - "step": "1" - }, - "name": "num_dense_units", - "parameterType": "int" - }, - { - "feasibleSpace": { - "list": [ - "elu", - "relu", - "gelu", - "linear" - ] - }, - "name": "conv_activation", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "list": [ - "elu", - "relu", - "gelu", - "linear" - ] - }, - "name": "activation", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "list": [ - "huber_loss", - "mean_absolute_error", - "mse" - ] - }, - "name": "loss", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "max": "5", - "min": "1.2", - "step": "0.01" - }, - "name": "final_activation_scale_factor", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "40", - "min": "1", - "step": "1" - }, - "name": "batch_size", - "parameterType": "int" - }, - { - "feasibleSpace": { - "max": "10", - "min": "1", - "step": "1" - }, - "name": "epochs", - "parameterType": "int" - }, - { - "feasibleSpace": { - "list": [ - "2023-01-051310" - ] - }, - "name": "time", - "parameterType": "categorical" - } - ] - }, - "katib_run": true, - "pipeline_description": "gwas-10-d", - "pipeline_name": "gwas-10-d", - "snapshot_volumes": true, - "volumes": [ - { - "annotations": [], - "mount_point": "/home/jovyan", - "name": "gwas-11-a-workspace-gxcvc", - "size": 5, - "size_type": "Gi", - "snapshot": false, - "type": "clone" - } - ] - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.0" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/DL-gwas-gcp-example/2-mse-run-second-in-jupyter.ipynb b/notebooks/DL-gwas-gcp-example/2-mse-run-second-in-jupyter.ipynb index cc3f958..22f4cd6 100644 --- a/notebooks/DL-gwas-gcp-example/2-mse-run-second-in-jupyter.ipynb +++ b/notebooks/DL-gwas-gcp-example/2-mse-run-second-in-jupyter.ipynb @@ -2,6 +2,7 @@ "cells": [ { "cell_type": "markdown", + "id": "6ed665c1", "metadata": {}, "source": [ "# Deep Learning GWAS Notebook 2" @@ -9,6 +10,7 @@ }, { "cell_type": "markdown", + "id": "ff985493", "metadata": {}, "source": [ "## Overview\n", @@ -19,11 +21,7 @@ "cell_type": "code", "execution_count": null, "id": "619de0db-3878-4254-93a4-4404553d0ecf", - "metadata": { - "tags": [ - "imports" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -43,9 +41,7 @@ { "cell_type": "markdown", "id": "365d8767-2dd3-4317-b556-2273f1510e19", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "## Get the best params from Katib from running the first notebook with Kale \n", "![assets/x002-final-results-page.png](assets/x002-final-results-page.png)" @@ -55,11 +51,7 @@ "cell_type": "code", "execution_count": null, "id": "fd604bf6-0d00-4178-9deb-28465c915c29", - "metadata": { - "tags": [ - "pipeline-parameters" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# Get the best params from Katib from running the first notebook with Kale \n", @@ -112,11 +104,7 @@ "cell_type": "code", "execution_count": null, "id": "e06747ae-77b6-4524-b803-fc0d87512fbb", - "metadata": { - "tags": [ - "block:preprocessing" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# Don't run in kale this time\n", @@ -176,12 +164,7 @@ "cell_type": "code", "execution_count": null, "id": "dd01e8b2-78ba-4315-9e12-9153b1c57ce5", - "metadata": { - "tags": [ - "block:train", - "prev:preprocessing" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# Don't run in Kale this time\n", @@ -350,12 +333,7 @@ "cell_type": "code", "execution_count": null, "id": "74954dbc-9133-40d5-9f23-400a39807724", - "metadata": { - "tags": [ - "block:saliency_observed", - "prev:preprocessing" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -385,12 +363,7 @@ "cell_type": "code", "execution_count": null, "id": "464ffbb4-eb9e-438e-ac1e-292a2e45a99f", - "metadata": { - "tags": [ - "block:manhattan_observed", - "prev:saliency_observed" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -415,12 +388,7 @@ "cell_type": "code", "execution_count": null, "id": "f18fae63-9e34-4d5b-b5d7-76e08e94142c", - "metadata": { - "tags": [ - "block:saliency_predicted", - "prev:train" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -456,12 +424,7 @@ "cell_type": "code", "execution_count": null, "id": "043678ab-c0ae-45d7-8fdc-259d3138adba", - "metadata": { - "tags": [ - "block:manttan_predcted", - "prev:saliency_predicted" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -486,13 +449,7 @@ "cell_type": "code", "execution_count": null, "id": "bae392ab-a377-49a1-afaf-b5330892d89a", - "metadata": { - "tags": [ - "block:qq_plot", - "prev:saliency_observed", - "prev:saliency_predicted" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -524,9 +481,7 @@ { "cell_type": "markdown", "id": "be03ced0-99f2-43fd-b29e-407d0f85212c", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "\n", "## There is a second notebook to run after you run this pipeline. Please find the model folder for the best model that Katib found.\n" @@ -536,11 +491,7 @@ "cell_type": "code", "execution_count": null, "id": "4b240cfb-b264-49dc-ae2c-73b10ef8a69d", - "metadata": { - "tags": [ - "pipeline-metrics" - ] - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -556,394 +507,7 @@ "source": [] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "tf2-gpu.2-10.m100", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-10:m100" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "kubeflow_notebook": { - "autosnapshot": true, - "deploy_config": {}, - "docker_image": "gcr.io/arrikto/jupyter-kale-py38@sha256:2e1ce3427b780c0c78e7cfec527ee10c391092fdc4a8344cd76f8b83c61c5234", - "experiment": { - "id": "new", - "name": "gwas-7-d" - }, - "experiment_name": "gwas-10-d", - "katib_metadata": { - "algorithm": { - "algorithmName": "bayesianoptimization", - "algorithmSettings": [ - { - "name": "random_state", - "value": "10" - }, - { - "name": "acq_optimizer", - "value": "auto" - }, - { - "name": "acq_func", - "value": "gp_hedge" - }, - { - "name": "base_estimator", - "value": "GP" - } - ] - }, - "maxFailedTrialCount": 10, - "maxTrialCount": 40, - "objective": { - "additionalMetricNames": [], - "goal": 0.05, - "objectiveMetricName": "val-mean-absolute-error", - "type": "minimize" - }, - "parallelTrialCount": 2, - "parameters": [ - { - "feasibleSpace": { - "list": [ - "IMP_height.txt" - ] - }, - "name": "data_file_to_run", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "list": [ - "GWAS on soy height" - ] - }, - "name": "experiment_description", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "max": "0.3", - "min": "0.00001", - "step": "0.00001" - }, - "name": "learning_rate", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": ".98", - "min": "0.0001", - "step": "0.0001" - }, - "name": "conv_1_dropout_rate", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_1_kernel_l1", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_1_kernel_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_1_bias_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_1_activity_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_x_kernel_l1", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_x_kernel_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_x_bias_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "conv_x_activity_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_x_kernel_l1", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_x_kernel_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_x_bias_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_x_activity_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_out_kernel_l1", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_out_kernel_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.0000001", - "step": "0.0000001" - }, - "name": "dense_out_bias_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": ".0000001", - "step": "0.0000001" - }, - "name": "dense_out_activity_l2", - "parameterType": "double" - }, - { - "feasibleSpace": { - "list": [ - "TruncatedNormal", - "glorot_uniform", - "GlorotNormal", - "HeNormal", - "random_normal" - ] - }, - "name": "conv_initializer", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "list": [ - "TruncatedNormal", - "glorot_uniform", - "GlorotNormal", - "HeNormal", - "random_normal" - ] - }, - "name": "dese_initializer", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "max": "0.95", - "min": "0.01", - "step": "0.1" - }, - "name": "dropout_rate", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "5", - "min": "1", - "step": "1" - }, - "name": "num_dense_layers", - "parameterType": "int" - }, - { - "feasibleSpace": { - "max": "20", - "min": "1", - "step": "1" - }, - "name": "num_dense_units", - "parameterType": "int" - }, - { - "feasibleSpace": { - "list": [ - "elu", - "relu", - "gelu", - "linear" - ] - }, - "name": "conv_activation", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "list": [ - "elu", - "relu", - "gelu", - "linear" - ] - }, - "name": "activation", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "list": [ - "huber_loss", - "mean_absolute_error", - "mse" - ] - }, - "name": "loss", - "parameterType": "categorical" - }, - { - "feasibleSpace": { - "max": "5", - "min": "1.2", - "step": "0.01" - }, - "name": "final_activation_scale_factor", - "parameterType": "double" - }, - { - "feasibleSpace": { - "max": "40", - "min": "1", - "step": "1" - }, - "name": "batch_size", - "parameterType": "int" - }, - { - "feasibleSpace": { - "max": "10", - "min": "1", - "step": "1" - }, - "name": "epochs", - "parameterType": "int" - }, - { - "feasibleSpace": { - "list": [ - "2023-01-051310" - ] - }, - "name": "time", - "parameterType": "categorical" - } - ] - }, - "katib_run": true, - "pipeline_description": "gwas-10-d", - "pipeline_name": "gwas-10-d", - "snapshot_volumes": true, - "volumes": [ - { - "annotations": [], - "mount_point": "/home/jovyan", - "name": "gwas-11-a-workspace-gxcvc", - "size": 5, - "size_type": "Gi", - "snapshot": false, - "type": "clone" - } - ] - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.0" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GWASCoatColor/GWAS_coat_color.ipynb b/notebooks/GWASCoatColor/GWAS_coat_color.ipynb index 4b25b49..c14ba4c 100644 --- a/notebooks/GWASCoatColor/GWAS_coat_color.ipynb +++ b/notebooks/GWASCoatColor/GWAS_coat_color.ipynb @@ -14,6 +14,7 @@ }, { "cell_type": "markdown", + "id": "7518a4df", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -22,6 +23,7 @@ }, { "cell_type": "markdown", + "id": "af2188bc", "metadata": {}, "source": [ "## Prerequisites\n", @@ -30,6 +32,7 @@ }, { "cell_type": "markdown", + "id": "84f3e44b", "metadata": {}, "source": [ "## Get Started" @@ -381,6 +384,7 @@ }, { "cell_type": "markdown", + "id": "fcba5439", "metadata": {}, "source": [ "## Conclusions" @@ -398,6 +402,7 @@ }, { "cell_type": "markdown", + "id": "c79c7892", "metadata": {}, "source": [ "## Clean Up\n", @@ -406,35 +411,12 @@ }, { "cell_type": "markdown", + "id": "6010276d", "metadata": {}, "source": [] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "tf2-gpu.2-11.m110", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-11:m110" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GenAI/GCP_Agent_Builder.ipynb b/notebooks/GenAI/GCP_Agent_Builder.ipynb index 0a551aa..f81bd35 100644 --- a/notebooks/GenAI/GCP_Agent_Builder.ipynb +++ b/notebooks/GenAI/GCP_Agent_Builder.ipynb @@ -3,9 +3,7 @@ { "cell_type": "markdown", "id": "d9cf6009-c86b-4ba8-b83c-0f41d75ba82f", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "**Skill Level:** Intermediate" ] @@ -149,11 +147,9 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "85209a6a-9c7a-4294-b6b7-110ffb23bae4", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#urls list of articles\n", @@ -232,9 +228,7 @@ { "cell_type": "markdown", "id": "9407c375-7b45-40b6-8504-3d6eced555da", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![grounding1](../../images/grounding_1.png)" ] @@ -266,9 +260,7 @@ { "cell_type": "markdown", "id": "0b59fa7c-b082-4372-9d9d-c299565f67dd", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![agent_builder2](../../images/agent_builder2.png)" ] @@ -288,9 +280,7 @@ { "cell_type": "markdown", "id": "94d92cb0-9195-4a39-a9fb-3297a0788e2f", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![agent_builder3](../../images/agent_builder3.png)" ] @@ -306,9 +296,7 @@ { "cell_type": "markdown", "id": "526ccb25-b210-4a80-aa50-89879c884f11", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![agent_builder4](../../images/agent_builder4.png)" ] @@ -356,9 +344,7 @@ { "cell_type": "markdown", "id": "8d1fd198-00ad-433c-84de-3744acad8f4a", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "**Option A** If its your first time creating a data store tool you will probably see an option like the following. If you have this view, go ahead and click **'Create a data store'**." ] @@ -382,9 +368,7 @@ { "cell_type": "markdown", "id": "531ff71a-4833-4452-bc6a-cc82d8899960", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![agent_builder7](../../images/agent_builder7.png)" ] @@ -408,9 +392,7 @@ { "cell_type": "markdown", "id": "bc647145-2da1-4870-8abc-0512714c24c2", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "**Note:** You may be prompted to configure your agent app if so enter the company name for your agent app. This can be anything you like. Set the location to **'US'**. Click **'Continue'**." ] @@ -434,9 +416,7 @@ { "cell_type": "markdown", "id": "7895ddda-5775-43b6-a723-7088c9b69ecb", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![grounding4](../../images/grounding_4.png)" ] @@ -704,31 +684,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m114", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m114" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GenAI/GCP_Code_Chatbot_wGrounding.ipynb b/notebooks/GenAI/GCP_Code_Chatbot_wGrounding.ipynb index adab673..397e563 100644 --- a/notebooks/GenAI/GCP_Code_Chatbot_wGrounding.ipynb +++ b/notebooks/GenAI/GCP_Code_Chatbot_wGrounding.ipynb @@ -314,9 +314,7 @@ { "cell_type": "markdown", "id": "69c4bee3-f700-4cd0-9630-08a444c2ea07", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "## Clean Up\n", "\n", @@ -332,31 +330,7 @@ "source": [] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m124", - "type": "gcloud", - "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/base-cpu:m124" - }, - "kernelspec": { - "display_name": "Python 3 (Local)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GenAI/GCP_GenAI_Huggingface.ipynb b/notebooks/GenAI/GCP_GenAI_Huggingface.ipynb index 7222b63..be91099 100644 --- a/notebooks/GenAI/GCP_GenAI_Huggingface.ipynb +++ b/notebooks/GenAI/GCP_GenAI_Huggingface.ipynb @@ -10,6 +10,7 @@ }, { "cell_type": "markdown", + "id": "40d0c590", "metadata": {}, "source": [ "## Overview" @@ -27,6 +28,7 @@ }, { "cell_type": "markdown", + "id": "1705116a", "metadata": {}, "source": [ "## Prerequisites \n", @@ -35,6 +37,7 @@ }, { "cell_type": "markdown", + "id": "5870770c", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -44,6 +47,7 @@ }, { "cell_type": "markdown", + "id": "565a354d", "metadata": {}, "source": [ "## Get Started" @@ -69,10 +73,7 @@ "cell_type": "code", "execution_count": null, "id": "a6e5884b-ac90-42d4-aafd-d34d5495d24d", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! pip install \"transformers\" \"datasets\" \"rouge_score\" \"evaluate\" \"keras_nlp\" \"tf_keras\"" @@ -80,6 +81,7 @@ }, { "cell_type": "markdown", + "id": "560867d9", "metadata": {}, "source": [ "## Get Started" @@ -105,10 +107,7 @@ "cell_type": "code", "execution_count": null, "id": "8f59e17e-c006-45ee-be0b-766774f9d420", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from datasets import load_dataset\n", @@ -129,9 +128,7 @@ "cell_type": "code", "execution_count": null, "id": "760c9128-793a-4bed-a127-b92ef496e33b", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "print(train)" @@ -168,9 +165,7 @@ "cell_type": "code", "execution_count": null, "id": "bfd433b3-9790-4a10-ac08-6c90c194d8b0", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#model name\n", @@ -181,10 +176,7 @@ "cell_type": "code", "execution_count": null, "id": "1988cbcb-4bec-4aa2-a356-a211584ceacb", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from transformers import TFAutoModelForSeq2SeqLM, AutoTokenizer\n", @@ -212,9 +204,7 @@ "cell_type": "code", "execution_count": null, "id": "f101c309-f214-4b3f-b77b-d55491e48a59", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "prefix = \"summarize: \"\n", @@ -244,9 +234,7 @@ "cell_type": "code", "execution_count": null, "id": "5e58eb58-a655-4e2b-8665-b4b770bc87a7", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "tokenized_train = train.map(preprocess_function, batched=True)\n", @@ -272,9 +260,7 @@ "cell_type": "code", "execution_count": null, "id": "80a25bc8-00db-4b8d-9b68-d52c5d6ca7fe", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "print(tokenized_train)" @@ -292,9 +278,7 @@ "cell_type": "code", "execution_count": null, "id": "875ef33d-5ef3-4b07-b1de-6d471743a8ad", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from transformers import DataCollatorForSeq2Seq\n", @@ -313,9 +297,7 @@ "cell_type": "code", "execution_count": null, "id": "6fcac5a8-912f-461f-bfab-990e472c01ca", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "tf_train_set = model.prepare_tf_dataset(\n", @@ -347,9 +329,7 @@ "cell_type": "code", "execution_count": null, "id": "11aaf028-c713-4064-84cc-f699df3151ec", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "print (tf_train_set)" @@ -367,9 +347,7 @@ "cell_type": "code", "execution_count": null, "id": "50ed6068-763a-46a6-8aed-4862f84413a9", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from transformers import AdamWeightDecay\n", @@ -397,9 +375,7 @@ "cell_type": "code", "execution_count": null, "id": "f9c6b45b-7349-4965-938b-3a334ced3882", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "import keras_nlp\n", @@ -432,9 +408,7 @@ "cell_type": "code", "execution_count": null, "id": "06e014b1-e9d2-4d9f-a149-c6c0381f7407", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from transformers.keras_callbacks import KerasMetricCallback\n", @@ -454,9 +428,7 @@ "cell_type": "code", "execution_count": null, "id": "b8fd0c64-4d85-4b5e-86fe-538c7dc65da7", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3, callbacks=metric_callback)\n", @@ -567,10 +539,7 @@ "cell_type": "code", "execution_count": null, "id": "d69aa008-80c0-4a19-aa0f-8f5798673c47", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! wget --user-agent=\"Chrome\" https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7784226/pdf/12248_2020_Article_532.pdf" @@ -588,10 +557,7 @@ "cell_type": "code", "execution_count": null, "id": "1347b3cd-5ce0-44c9-864d-a688bcacb1d0", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "!pip install \"fitz\" \"PyMuPDF\"" @@ -635,10 +601,7 @@ "cell_type": "code", "execution_count": null, "id": "9a1f5dbd-6a9f-4533-a12e-8a6c4073df74", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from transformers import AutoTokenizer\n", @@ -666,9 +629,7 @@ { "cell_type": "markdown", "id": "6ac841f6-c65e-4ebf-8c42-3030e2f92cb0", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "### Setting up our Datasets for Training " ] @@ -711,10 +672,7 @@ "cell_type": "code", "execution_count": null, "id": "0066ad72-e451-41c0-b30a-c3a7dfa5f17c", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#Create bucket\n", @@ -789,9 +747,7 @@ "cell_type": "code", "execution_count": null, "id": "ebc1bc39-a554-473b-949a-d9588f6e7fb8", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "# save train_dataset to s3\n", @@ -1080,10 +1036,7 @@ "cell_type": "code", "execution_count": null, "id": "764980e6-9bc1-4715-b540-9e254b12f1f3", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#to view options and defaults you can run the command below\n", @@ -1150,10 +1103,7 @@ "cell_type": "code", "execution_count": null, "id": "252d8e16-5b3d-409b-bc86-9da0ce996f72", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "!gcloud ai custom-jobs create \\\n", @@ -1291,6 +1241,7 @@ }, { "cell_type": "markdown", + "id": "bf29655a", "metadata": {}, "source": [ "## Conclusion\n", @@ -1325,10 +1276,7 @@ "cell_type": "code", "execution_count": null, "id": "9721f52d-040f-4dc7-808e-8d1ffb5efb4a", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "!gcloud ai custom-jobs list --project=$project --region=$location" @@ -1446,31 +1394,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "conda-root-py", - "name": "workbench-notebooks.m119", - "type": "gcloud", - "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m119" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel) (Local)", - "language": "python", - "name": "conda-root-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GenAI/GCP_Grounding.ipynb b/notebooks/GenAI/GCP_Grounding.ipynb index 1bd18bc..163c483 100644 --- a/notebooks/GenAI/GCP_Grounding.ipynb +++ b/notebooks/GenAI/GCP_Grounding.ipynb @@ -85,9 +85,7 @@ { "cell_type": "markdown", "id": "9407c375-7b45-40b6-8504-3d6eced555da", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![grounding1](../../images/grounding_1.png)" ] @@ -121,9 +119,7 @@ { "cell_type": "markdown", "id": "d2d4afa0-bdf7-4afc-8b74-d3e791bbaebd", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![grounding3](../../images/grounding_3.png)" ] @@ -147,9 +143,7 @@ { "cell_type": "markdown", "id": "7895ddda-5775-43b6-a723-7088c9b69ecb", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![grounding4](../../images/grounding_4.png)" ] @@ -294,9 +288,7 @@ "cell_type": "code", "execution_count": null, "id": "f7107b36-061c-4073-8e47-3a399273577a", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "PROJECT_ID = ''\n", @@ -315,9 +307,7 @@ "cell_type": "code", "execution_count": null, "id": "04dac266-90ec-49e3-aa7c-9fd63d25a89b", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "print(f'projects/{PROJECT_ID}/locations/global/collections/default_collection/dataStores/{DATA_STORE_ID}')" @@ -366,9 +356,7 @@ { "cell_type": "markdown", "id": "77f0da56-5ff2-4223-9ebb-15fca421461d", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![grounding15](../../images/grounding_15.png)" ] @@ -376,9 +364,7 @@ { "cell_type": "markdown", "id": "d9dc3253-26dc-4770-bc90-cb387f365dfe", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "Then ask the model a question like you did before. The image below shows the grounding response using your own data at the top and Google searches response at the bottom." ] @@ -424,31 +410,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m114", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m114" - }, - "kernelspec": { - "display_name": "Python (Local)", - "language": "python", - "name": "base" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GenAI/GCP_MedLM_Intro.ipynb b/notebooks/GenAI/GCP_MedLM_Intro.ipynb index dbf7e27..b5e7477 100644 --- a/notebooks/GenAI/GCP_MedLM_Intro.ipynb +++ b/notebooks/GenAI/GCP_MedLM_Intro.ipynb @@ -113,10 +113,7 @@ "cell_type": "code", "execution_count": null, "id": "3a8cc9c8-95a0-4fb5-a84b-b9cbc2d4180f", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "vertexai.init(project=PROJECT_ID, location)\n", @@ -150,9 +147,7 @@ "cell_type": "code", "execution_count": null, "id": "62d453be-71c8-4ae5-9d29-42d241c0f359", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "query1 = 'What causes you to get ringworm?'" @@ -162,9 +157,7 @@ "cell_type": "code", "execution_count": null, "id": "b2c74c3f-57c1-43af-89a2-c2b0c055e9b6", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "query2 = 'What is the monthly average of covid-19 patients in Virginia?'" @@ -182,9 +175,7 @@ "cell_type": "code", "execution_count": null, "id": "67c94766-3f3a-4a96-854e-4d2e8a823032", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "medlm_invoke(query1)" @@ -194,9 +185,7 @@ "cell_type": "code", "execution_count": null, "id": "a6b798d2-4286-4a92-a75c-d1d21560dcb2", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "medlm_invoke(query2)" @@ -240,9 +229,7 @@ "cell_type": "code", "execution_count": null, "id": "ff52f53a-5435-4542-a82c-0e592d005070", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "request = { \"instances\": \n", @@ -272,9 +259,7 @@ "cell_type": "code", "execution_count": null, "id": "6864fb73-7b45-445f-bc94-67f21d1d2ab9", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "print(request)" @@ -292,9 +277,7 @@ "cell_type": "code", "execution_count": null, "id": "585f7dc0-d1d4-495c-8f51-11c7f0687098", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "project_id = \"\"\n", @@ -304,9 +287,7 @@ { "cell_type": "markdown", "id": "597e7203-4250-45fb-8bf0-4a0910f4c52d", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "Now we can pass our request to our curl command which will invoke the model and send back a response." ] @@ -315,9 +296,7 @@ "cell_type": "code", "execution_count": null, "id": "fe62d6fa-360b-4df7-8fe6-3e572acaf2e2", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "!curl -X POST \\\n", @@ -360,31 +339,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m114", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m114" - }, - "kernelspec": { - "display_name": "Python (Local)", - "language": "python", - "name": "base" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GenAI/Gemini_Intro.ipynb b/notebooks/GenAI/Gemini_Intro.ipynb index 712bca3..b6dda99 100644 --- a/notebooks/GenAI/Gemini_Intro.ipynb +++ b/notebooks/GenAI/Gemini_Intro.ipynb @@ -10,6 +10,7 @@ }, { "cell_type": "markdown", + "id": "29ca828b", "metadata": {}, "source": [ "## Overview" @@ -25,6 +26,7 @@ }, { "cell_type": "markdown", + "id": "755ceab1", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -33,6 +35,7 @@ }, { "cell_type": "markdown", + "id": "453c727b", "metadata": {}, "source": [ "## Prerequisites\n", @@ -41,6 +44,7 @@ }, { "cell_type": "markdown", + "id": "1b7227b7", "metadata": {}, "source": [ "## Install Packages" @@ -49,9 +53,7 @@ { "cell_type": "markdown", "id": "b8ec6d40-b5b3-434f-adc4-2838b7f49d1d", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "Update the google-cloud-aiplatform package" ] @@ -60,10 +62,7 @@ "cell_type": "code", "execution_count": null, "id": "ed9781dd-9764-4e9c-88ba-fcd7bb95842a", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! pip install --upgrade google-cloud-aiplatform langchain langchain-community" @@ -86,9 +85,7 @@ "cell_type": "code", "execution_count": null, "id": "47dc9232-383f-405b-b1a8-fab64a80492d", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from google.cloud import aiplatform\n", @@ -103,6 +100,7 @@ }, { "cell_type": "markdown", + "id": "91245100", "metadata": {}, "source": [ "## Get Started" @@ -128,9 +126,7 @@ "cell_type": "code", "execution_count": null, "id": "70bc5b25-c796-4015-82dc-6bc861bb525f", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "model = GenerativeModel(\"gemini-pro\")\n", @@ -164,9 +160,7 @@ "cell_type": "code", "execution_count": null, "id": "342a0e3d-fbcb-4562-bb5f-b439a92e80e2", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "prompt = \"List gen ai use cases that are Life Science or Health Care related. \"\n", @@ -185,9 +179,7 @@ "cell_type": "code", "execution_count": null, "id": "f0b917b2-22b5-4011-a9c4-d8a667cf6b1d", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "prompt = \"create a python code that will replace all null values to zero within a csv file\"\n", @@ -222,9 +214,7 @@ "cell_type": "code", "execution_count": null, "id": "ee4cf184-c815-425c-9742-7625123e02bf", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#download the article\n", @@ -235,9 +225,7 @@ "cell_type": "code", "execution_count": null, "id": "3becd6c2-daf0-4287-80e5-06cf419287bd", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from langchain_community.document_loaders import PyPDFLoader\n", @@ -338,9 +326,7 @@ "cell_type": "code", "execution_count": null, "id": "501efefe-d52f-43b3-b4eb-3d3fe81f4a3e", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "def img2text(image_path: str, img_prompt: str) -> str:\n", @@ -376,9 +362,7 @@ "cell_type": "code", "execution_count": null, "id": "b939f105-89c2-4c38-80f8-2cddf8dcb0ca", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! wget -O example_image_covid.jpg \"https://phil.cdc.gov//PHIL_Images/23312/23312_lores.jpg\" " @@ -396,9 +380,7 @@ "cell_type": "code", "execution_count": null, "id": "34e81656-4943-439d-9fbe-df439e0e30df", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "print(img2text(\"example_image_covid.jpg\", \"describe this image.\"))" @@ -416,9 +398,7 @@ "cell_type": "code", "execution_count": null, "id": "81197d53-dd3d-4358-9835-ef513ec11d33", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "print(img2text(\"gs://generativeai-downloads/images/scones.jpg\", \"describe this image.\"))" @@ -436,9 +416,7 @@ "cell_type": "code", "execution_count": null, "id": "d8395784-ea68-4a95-a0bb-b3d618f68054", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "img_prompt=\"How do you make whats in this image?\"\n", @@ -482,9 +460,7 @@ "cell_type": "code", "execution_count": null, "id": "c18561c6-8f8f-46e4-b3ee-0d5fc96f2d31", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "def video2text(video_path: str, video_prompt: str) -> str:\n", @@ -517,9 +493,7 @@ "cell_type": "code", "execution_count": null, "id": "55990074-0365-45f5-9fa6-bedbe93c9932", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "video_prompt = \"What is this video about in detail?\"\n", @@ -562,9 +536,7 @@ { "cell_type": "markdown", "id": "ec72f9cd-0d06-47e4-b67c-45039372d967", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![Gemini3](../../images/Gemini_3.png)" ] @@ -580,9 +552,7 @@ { "cell_type": "markdown", "id": "1952cd5c-c93d-428f-8036-ac658eebfba4", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![Gemini2](../../images/Gemini_2.png)" ] @@ -598,9 +568,7 @@ { "cell_type": "markdown", "id": "36b4635d-d8f3-42df-959d-dff92259813c", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "![Gemini4](../../images/Gemini_4.png)" ] @@ -614,31 +582,7 @@ "source": [] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m114", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m114" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GenAI/Google_Drive_chatbot.ipynb b/notebooks/GenAI/Google_Drive_chatbot.ipynb index 1f24d7d..6ae70ec 100644 --- a/notebooks/GenAI/Google_Drive_chatbot.ipynb +++ b/notebooks/GenAI/Google_Drive_chatbot.ipynb @@ -99,10 +99,7 @@ "cell_type": "code", "execution_count": null, "id": "ed412af0-1e5f-4250-acc5-df46632802b6", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! pip install --upgrade google-api-python-client vertexai unstructured" @@ -112,9 +109,7 @@ "cell_type": "code", "execution_count": null, "id": "5aeaa4f6-5cd5-4cc7-a710-97a6122fc9e3", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from vertexai.preview import rag\n", @@ -192,10 +187,7 @@ "cell_type": "code", "execution_count": null, "id": "99d49432-cf03-4f19-aa82-ef7f8bad5bde", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "import subprocess\n", @@ -245,9 +237,7 @@ "cell_type": "code", "execution_count": null, "id": "639352c6-a62a-4ed8-a04e-cd5fd35549d9", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "vertexai.init(project=project_id, location=location)" @@ -284,9 +274,7 @@ "cell_type": "code", "execution_count": null, "id": "8ceab1af-caa5-41a6-b6c8-ea58578a8b02", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "print(rag_corpus.name)" @@ -342,9 +330,7 @@ "cell_type": "code", "execution_count": null, "id": "4a32b138-7ce8-49a7-80eb-fca6a0d6b4f7", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#list files in corpus\n", @@ -365,9 +351,7 @@ "cell_type": "code", "execution_count": null, "id": "df56d730-87b0-4296-b2d2-1b2ed90a2d63", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "query = \"What is Long Covid?\"\n", @@ -432,9 +416,7 @@ "cell_type": "code", "execution_count": null, "id": "0e89c4ce-437a-4183-8e4c-688be2a5faee", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "question = \"What is Long Covid?\"\n", @@ -562,31 +544,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m114", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m114" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GenAI/Pubmed_RAG_chatbot.ipynb b/notebooks/GenAI/Pubmed_RAG_chatbot.ipynb index a81bdd8..a12ba66 100644 --- a/notebooks/GenAI/Pubmed_RAG_chatbot.ipynb +++ b/notebooks/GenAI/Pubmed_RAG_chatbot.ipynb @@ -102,9 +102,7 @@ "cell_type": "code", "execution_count": null, "id": "c39e2160-660a-40cd-886d-e4179fbe6c13", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! pip install langchain langchain-google-vertexai langchain-community unstructured" @@ -162,9 +160,7 @@ "cell_type": "code", "execution_count": null, "id": "99d49432-cf03-4f19-aa82-ef7f8bad5bde", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#make bucket\n", @@ -183,9 +179,7 @@ "cell_type": "code", "execution_count": null, "id": "7b395e34-062d-4f77-afee-3601d471954a", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#download the metadata file\n", @@ -204,9 +198,7 @@ "cell_type": "code", "execution_count": null, "id": "c26b0f29-2b07-43a6-800d-4aa5e957fe52", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#import the file as a dataframe\n", @@ -229,9 +221,7 @@ "cell_type": "code", "execution_count": null, "id": "ff77b2aa-ed1b-4d27-8163-fdaa7a304582", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "first_50.head()" @@ -249,10 +239,7 @@ "cell_type": "code", "execution_count": null, "id": "b3cca24a-59d0-4dc7-b887-c8cc8547774f", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from google.cloud import storage\n", @@ -298,9 +285,7 @@ "cell_type": "code", "execution_count": null, "id": "6cf5092c-23f3-4f28-9308-f34b8d90c62b", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "import uuid\n", @@ -317,9 +302,7 @@ "cell_type": "code", "execution_count": null, "id": "8e8a4c42-dc17-48a3-a0bb-0cbea527ee7f", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#move inital embeddings file to bucket\n", @@ -340,9 +323,7 @@ "cell_type": "code", "execution_count": null, "id": "39aa7bba-3d15-4a3f-86c2-59d2c92a95ef", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "from google.cloud import aiplatform\n", @@ -381,9 +362,7 @@ "cell_type": "code", "execution_count": null, "id": "55596202-13b9-4e35-8099-0602a2b13e72", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#Create the endpoint\n", @@ -398,9 +377,7 @@ "cell_type": "code", "execution_count": null, "id": "3f771328-31c6-4da2-9d7d-8a548abd12a1", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#save endpoint id\n", @@ -419,9 +396,7 @@ "cell_type": "code", "execution_count": null, "id": "51412f2f-f32b-44a9-93bc-3e2f6185cada", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#deploy our index to our endpoint\n", @@ -452,9 +427,7 @@ "cell_type": "code", "execution_count": null, "id": "b9016f15-db02-4073-b4c7-288d919bbb55", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", @@ -476,9 +449,7 @@ "cell_type": "code", "execution_count": null, "id": "69ce004e-ab8d-4b9c-91d8-9320e1679fcd", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "first_50_dict[0]" @@ -496,9 +467,7 @@ "cell_type": "code", "execution_count": null, "id": "47170e83-3e9e-48e6-ab0f-cabdd39507e1", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#add metadata\n", @@ -711,9 +680,7 @@ { "cell_type": "markdown", "id": "07b3bc6b-8c43-476f-a662-abda830dc2da", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "### Creating an Interactive Inference Script " ] @@ -749,9 +716,7 @@ { "cell_type": "markdown", "id": "6f0ad48d-c6c8-421a-a48b-88e979d15b57", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "```python\n", "from langchain_community.retrievers import PubMedRetriever\n", @@ -778,9 +743,7 @@ { "cell_type": "markdown", "id": "decbb901-f811-4b8e-a956-4c8c7f914ae2", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "```python\n", "class bcolors:\n", @@ -841,9 +804,7 @@ { "cell_type": "markdown", "id": "8cadb1af-2c46-4ab1-92f9-6e0861f83324", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "```python\n", "llm = VertexAI(\n", @@ -873,9 +834,7 @@ { "cell_type": "markdown", "id": "21c61724-23d3-4b49-8c72-cbd208bdb5df", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "```python\n", "retriever= PubMedRetriever()\n", @@ -923,9 +882,7 @@ { "cell_type": "markdown", "id": "c0316dc5-6274-4a5e-92e4-3d266ed6a4df", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "```python\n", "prompt_template = \"\"\"\n", @@ -1092,9 +1049,7 @@ "cell_type": "code", "execution_count": null, "id": "ba97df23-6893-438d-8a67-cb7dbf83e407", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#retreive our index and endpoint id\n", @@ -1121,9 +1076,7 @@ { "cell_type": "markdown", "id": "bbe127e6-c0b1-4e07-ad56-38c30a9bf858", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "You should see similar results on the terminal. In this example we ask the chatbot to explain brain cancer!" ] @@ -1205,31 +1158,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m114", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m114" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GenAI/VertexAIStudioGCP.ipynb b/notebooks/GenAI/VertexAIStudioGCP.ipynb index 8f436d6..219f1dc 100644 --- a/notebooks/GenAI/VertexAIStudioGCP.ipynb +++ b/notebooks/GenAI/VertexAIStudioGCP.ipynb @@ -2,18 +2,14 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "id": "Hny4I-ODTIS6" - }, + "metadata": {}, "source": [ "# Vertex AI Studio on GCP - Article Summary\n" ] }, { "cell_type": "markdown", - "metadata": { - "id": "-nLS57E2TO5y" - }, + "metadata": {}, "source": [ "## Overview\n", "\n", @@ -39,9 +35,7 @@ }, { "cell_type": "markdown", - "metadata": { - "id": "skXAu__iqks_" - }, + "metadata": {}, "source": [ "### A note on costs\n", "\n", @@ -60,9 +54,7 @@ }, { "cell_type": "markdown", - "metadata": { - "id": "x_xMwRLuyDrj" - }, + "metadata": {}, "source": [ "Here you will use LLM via the API to summarize the extracted texts. Please note that LLMs currently have input text limit and stuffing a large input text might not be accepted. You can read more about quotas and limits [here](https://cloud.google.com/vertex-ai/docs/quotas)." ] @@ -109,9 +101,7 @@ }, { "cell_type": "markdown", - "metadata": { - "id": "N5aVrDWkJs3Y" - }, + "metadata": {}, "source": [ "### Troubleshooting\n", "\n", @@ -120,9 +110,7 @@ }, { "cell_type": "markdown", - "metadata": { - "id": "Vtp21WX3T7d_" - }, + "metadata": {}, "source": [ "## Conclusion\n", "\n", @@ -138,35 +126,7 @@ ] } ], - "metadata": { - "colab": { - "name": "summarization_large_documents.ipynb", - "toc_visible": true - }, - "environment": { - "kernel": "conda-root-py", - "name": "workbench-notebooks.m119", - "type": "gcloud", - "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m119" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel) (Local)", - "language": "python", - "name": "conda-root-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 4 } diff --git a/notebooks/GenAI/langchain_on_vertex.ipynb b/notebooks/GenAI/langchain_on_vertex.ipynb index 8ef0511..c8267ec 100644 --- a/notebooks/GenAI/langchain_on_vertex.ipynb +++ b/notebooks/GenAI/langchain_on_vertex.ipynb @@ -2,6 +2,7 @@ "cells": [ { "cell_type": "markdown", + "id": "3f00e116", "metadata": {}, "source": [ "# Using LangChain on Google Cloud" @@ -18,6 +19,7 @@ }, { "cell_type": "markdown", + "id": "38aff184", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -29,6 +31,7 @@ }, { "cell_type": "markdown", + "id": "700d170e", "metadata": {}, "source": [ "## Prerequisites\n", @@ -37,6 +40,7 @@ }, { "cell_type": "markdown", + "id": "cf708984", "metadata": {}, "source": [ "## Get Started" @@ -54,10 +58,7 @@ "cell_type": "code", "execution_count": null, "id": "8662e8f8-66ce-4ca6-a121-d087c499390f", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "!pip install -U google-cloud-aiplatform langchain langchain-community langchain-google-vertexai pypdf faiss-cpu --user" @@ -75,9 +76,7 @@ "cell_type": "code", "execution_count": null, "id": "27e6851a-f15d-4881-8173-9b788a009201", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "import bs4\n", @@ -105,9 +104,7 @@ "cell_type": "code", "execution_count": null, "id": "46d1b6cc-862e-4a67-a755-fbc4f7595c6f", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "loader = WebBaseLoader(\"https://pubmed.ncbi.nlm.nih.gov/37883540/\")\n", @@ -118,9 +115,7 @@ "cell_type": "code", "execution_count": null, "id": "e34bd138-d852-40ba-87bd-ee559483aa20", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "llm = ChatVertexAI()\n", @@ -135,9 +130,7 @@ "cell_type": "code", "execution_count": null, "id": "dee2c20d-7678-4f6d-81c7-0b2a2b62d055", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "print('the summary of the document in a single paragraph is: ')\n", @@ -165,9 +158,7 @@ "cell_type": "code", "execution_count": null, "id": "0ad234c3-47c4-4aaf-a5b1-a3323555a8a5", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "template = \"\"\"Question: {question}\n", @@ -181,9 +172,7 @@ "cell_type": "code", "execution_count": null, "id": "126cdbda-6446-4bbb-8018-f24fce5a7216", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "chain = prompt | llm" @@ -193,9 +182,7 @@ "cell_type": "code", "execution_count": null, "id": "7323a512-5826-4498-baa6-65dca1dc6a6f", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "question = \"What evidence do we have for chimpanzees going through menopause?\"\n", @@ -241,9 +228,7 @@ "cell_type": "code", "execution_count": null, "id": "2c5bcbbb-8e24-424d-931d-c9b6c09fb888", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "loader = PyPDFLoader(\"41586_2024_Article_7159.pdf\")\n", @@ -573,6 +558,7 @@ }, { "cell_type": "markdown", + "id": "b836b24d", "metadata": {}, "source": [ "## Clean Up" @@ -580,37 +566,14 @@ }, { "cell_type": "markdown", + "id": "c66e487b", "metadata": {}, "source": [ "Make sure to stop your VM" ] } ], - "metadata": { - "environment": { - "kernel": "conda-root-py", - "name": "workbench-notebooks.m119", - "type": "gcloud", - "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m119" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel) (Local)", - "language": "python", - "name": "conda-root-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GoogleBatch/nextflow/Part1_GBatch_Nextflow.ipynb b/notebooks/GoogleBatch/nextflow/Part1_GBatch_Nextflow.ipynb index 7121976..0fe2fce 100644 --- a/notebooks/GoogleBatch/nextflow/Part1_GBatch_Nextflow.ipynb +++ b/notebooks/GoogleBatch/nextflow/Part1_GBatch_Nextflow.ipynb @@ -10,6 +10,7 @@ }, { "cell_type": "markdown", + "id": "d65493ca", "metadata": {}, "source": [ "## Overview\n", @@ -26,6 +27,7 @@ }, { "cell_type": "markdown", + "id": "3aca01d1", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -35,6 +37,7 @@ }, { "cell_type": "markdown", + "id": "126243b9", "metadata": {}, "source": [ "## Prerequisites\n", @@ -53,6 +56,7 @@ }, { "cell_type": "markdown", + "id": "8a9b9ca6", "metadata": {}, "source": [ "## Get Started" @@ -60,6 +64,7 @@ }, { "cell_type": "markdown", + "id": "0fdb7b18", "metadata": {}, "source": [ "### Install packages and set up environment" @@ -68,9 +73,7 @@ { "cell_type": "markdown", "id": "f2e4a5ca-8a2b-4156-b83e-c89f0c1ffc9c", - "metadata": { - "id": "f2e4a5ca-8a2b-4156-b83e-c89f0c1ffc9c" - }, + "metadata": {}, "source": [ "### Create a bucket" ] @@ -156,9 +159,7 @@ { "cell_type": "markdown", "id": "33f6045f-3336-46ae-917c-6528b4c0c0db", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "#### Submitting a job through the command line" ] @@ -262,11 +263,7 @@ { "cell_type": "markdown", "id": "1b8f7dfd-5a6a-4d04-8325-88e4210cb2c3", - "metadata": { - "id": "3cb5bd4b-032a-47f0-bee4-299a547c3b48", - "outputId": "b0e740fa-dabc-4d45-c95b-15f72d32bffa", - "tags": [] - }, + "metadata": {}, "source": [ "#### Submitting a job through the console" ] @@ -324,10 +321,7 @@ { "cell_type": "markdown", "id": "f4892a16-f4d9-4db9-a171-6e9245df2a72", - "metadata": { - "id": "f4892a16-f4d9-4db9-a171-6e9245df2a72", - "tags": [] - }, + "metadata": {}, "source": [ "### Check job status\n", "\n", @@ -352,9 +346,7 @@ { "cell_type": "markdown", "id": "9f056585-6c10-41b6-b7b6-0c75bebed811", - "metadata": { - "id": "9f056585-6c10-41b6-b7b6-0c75bebed811" - }, + "metadata": {}, "source": [ "### View your output" ] @@ -373,10 +365,7 @@ "cell_type": "code", "execution_count": null, "id": "02faf944-0143-49c7-bf4c-6b8e377fcd81", - "metadata": { - "id": "02faf944-0143-49c7-bf4c-6b8e377fcd81", - "outputId": "251ad4db-dcea-4d72-ff9a-01b3080acc8e" - }, + "metadata": {}, "outputs": [], "source": [ "! gsutil cp gs://$BUCKET/hello_world.txt .\n", @@ -386,9 +375,7 @@ { "cell_type": "markdown", "id": "33a142e0-bd9a-405d-91f9-827503ff5fb1", - "metadata": { - "id": "33a142e0-bd9a-405d-91f9-827503ff5fb1" - }, + "metadata": {}, "source": [ "## Run Nextflow Locally" ] @@ -404,9 +391,7 @@ { "cell_type": "markdown", "id": "b709c718-96d0-4925-99dd-525a7e7b6c76", - "metadata": { - "id": "b709c718-96d0-4925-99dd-525a7e7b6c76" - }, + "metadata": {}, "source": [ "Nextflow interacts with many different files to have a proper working workflow:\n", "\n", @@ -422,9 +407,7 @@ { "cell_type": "markdown", "id": "9bea3004-ff40-4918-ac16-83aad9427ad7", - "metadata": { - "id": "9bea3004-ff40-4918-ac16-83aad9427ad7" - }, + "metadata": {}, "source": [ "### Run a nextflow 'Hello World' process locally" ] @@ -432,9 +415,7 @@ { "cell_type": "markdown", "id": "4715ef92-e3a6-44cf-9b1e-50f247dd0daf", - "metadata": { - "id": "4715ef92-e3a6-44cf-9b1e-50f247dd0daf" - }, + "metadata": {}, "source": [ "We are going to first run Hello World locally using the config file called hello.nf. \n", "\n", @@ -468,10 +449,7 @@ "cell_type": "code", "execution_count": null, "id": "6efad386-185b-4faf-be39-6c5a3f84ffe4", - "metadata": { - "id": "6efad386-185b-4faf-be39-6c5a3f84ffe4", - "outputId": "9554903e-f8d5-43fa-ffe9-f00ce836bf2d" - }, + "metadata": {}, "outputs": [], "source": [ "! ./nextflow run hello.nf --str 'Hello!'" @@ -480,10 +458,7 @@ { "cell_type": "markdown", "id": "7619875d-7f10-4699-b4d2-120d5d7d4cd7", - "metadata": { - "id": "7619875d-7f10-4699-b4d2-120d5d7d4cd7", - "tags": [] - }, + "metadata": {}, "source": [ "## Submit Nextflow Job to the Google Batch\n", "Create and modify your own config file to include a 'gbatch' profile block to tell Nextflow to submit the job to Google Batch instead of running locally" @@ -492,9 +467,7 @@ { "cell_type": "markdown", "id": "ec7abe9b-dca1-4ef6-87d6-39fcdd2e3c9b", - "metadata": { - "id": "ec7abe9b-dca1-4ef6-87d6-39fcdd2e3c9b" - }, + "metadata": {}, "source": [ "The config file allows nextflow to utilize excecuters like Google Batch. In this tutorial the config files is named __'nextflow.config'__. Make sure you open this file and update the `` that are account specific.\n", "- Make sure that your region is a region included in the Google Batch!\n", @@ -522,10 +495,7 @@ { "cell_type": "markdown", "id": "340f7300-449a-4a12-bbc5-073547d58cac", - "metadata": { - "id": "340f7300-449a-4a12-bbc5-073547d58cac", - "tags": [] - }, + "metadata": {}, "source": [ "### Optional: Listing nf-core tools with docker and viewing their commands\n", "Using the command below you can see all the tools that nfcore holds and their versions/lastes releases" @@ -535,12 +505,7 @@ "cell_type": "code", "execution_count": null, "id": "ca1ff164-cee2-446e-ab2e-a3ed984e0dc0", - "metadata": { - "id": "ca1ff164-cee2-446e-ab2e-a3ed984e0dc0", - "outputId": "0530644a-dd9a-4077-dbc8-d1e335788a01", - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! docker run nfcore/tools list" @@ -549,9 +514,7 @@ { "cell_type": "markdown", "id": "9e46373c-61d0-4c91-b001-e55568d9fa2d", - "metadata": { - "id": "9e46373c-61d0-4c91-b001-e55568d9fa2d" - }, + "metadata": {}, "source": [ "You can view commands for methylseq (or any other specified nf-core tool) by using the [--help] flag" ] @@ -560,12 +523,7 @@ "cell_type": "code", "execution_count": null, "id": "05ea2893-60b3-4934-ae86-b07d4bc59728", - "metadata": { - "id": "05ea2893-60b3-4934-ae86-b07d4bc59728", - "outputId": "1e6de26f-0433-4bbd-8a43-119097bb1f41", - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! ./nextflow run nf-core/methylseq -r 1.6.1 --help" @@ -574,10 +532,7 @@ { "cell_type": "markdown", "id": "b4dbef59-d619-4444-8870-18c1f0ba3b5c", - "metadata": { - "id": "b4dbef59-d619-4444-8870-18c1f0ba3b5c", - "tags": [] - }, + "metadata": {}, "source": [ "### Run Methylseq with the test profile" ] @@ -585,9 +540,7 @@ { "cell_type": "markdown", "id": "7238bd3e-1853-42c3-9d2d-c72e46975ff2", - "metadata": { - "id": "7238bd3e-1853-42c3-9d2d-c72e46975ff2" - }, + "metadata": {}, "source": [ "The 'test' profile uses a small dataset allowing you to ensure the workflow works with your config file without long runtimes. Ensure you include:\n", "- Version of the nf-core tool [-r]\n", @@ -598,12 +551,7 @@ "cell_type": "code", "execution_count": null, "id": "4b21f170-37fa-4fbc-ab83-3f6b4d386ef9", - "metadata": { - "id": "4b21f170-37fa-4fbc-ab83-3f6b4d386ef9", - "outputId": "0507c847-7f83-40af-ebf0-a1fdef27499b", - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! ./nextflow run nf-core/methylseq -r 1.6.1 -profile test,gbatch -c nextflow-methyseq.config" @@ -612,9 +560,7 @@ { "cell_type": "markdown", "id": "e386ccb3-aa6d-4a77-8d7d-c20ed0419f84", - "metadata": { - "id": "e386ccb3-aa6d-4a77-8d7d-c20ed0419f84" - }, + "metadata": {}, "source": [ "You will notice in the above that to the left of the process within the __[ ]__ is actually a __tag__ you can search in Google Batch and the text before the __/__ corresponds to the __temporary directories__ within your working directory. Feel free to delete the temporary directories once your workflow has succesfully completed.\n", "\n", @@ -649,6 +595,7 @@ }, { "cell_type": "markdown", + "id": "5e245825", "metadata": {}, "source": [ "## Conclusion\n", @@ -667,35 +614,7 @@ ] } ], - "metadata": { - "colab": { - "name": "Workshop_2_updated.ipynb", - "provenance": [] - }, - "environment": { - "kernel": "python3", - "name": "common-cpu.m93", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m93" - }, - "kernelspec": { - "display_name": "Python (Local)", - "language": "python", - "name": "local-base" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.12" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/GoogleBatch/nextflow/Part2_GBatch_Nextflow.ipynb b/notebooks/GoogleBatch/nextflow/Part2_GBatch_Nextflow.ipynb index 357bc00..5575330 100644 --- a/notebooks/GoogleBatch/nextflow/Part2_GBatch_Nextflow.ipynb +++ b/notebooks/GoogleBatch/nextflow/Part2_GBatch_Nextflow.ipynb @@ -10,6 +10,7 @@ }, { "cell_type": "markdown", + "id": "58608548", "metadata": {}, "source": [ "## Overview\n", @@ -18,6 +19,7 @@ }, { "cell_type": "markdown", + "id": "b5596e1f", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -26,6 +28,7 @@ }, { "cell_type": "markdown", + "id": "3385bf33", "metadata": {}, "source": [ "## Prerequisites\n", @@ -44,6 +47,7 @@ }, { "cell_type": "markdown", + "id": "0e0e841b", "metadata": {}, "source": [ "## Get Started" @@ -51,6 +55,7 @@ }, { "cell_type": "markdown", + "id": "88fe41f6", "metadata": {}, "source": [ "### Install packages and set up environment\n", @@ -367,6 +372,7 @@ }, { "cell_type": "markdown", + "id": "baeb0d0a", "metadata": {}, "source": [ "## Conclusion\n", @@ -392,31 +398,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m93", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m93" - }, - "kernelspec": { - "display_name": "Python (Local)", - "language": "python", - "name": "local-base" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.12" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/LifeSciencesAPI/nextflow/Part1_LS_API_Nextflow.ipynb b/notebooks/LifeSciencesAPI/nextflow/Part1_LS_API_Nextflow.ipynb index 2e3dee8..e9b3623 100644 --- a/notebooks/LifeSciencesAPI/nextflow/Part1_LS_API_Nextflow.ipynb +++ b/notebooks/LifeSciencesAPI/nextflow/Part1_LS_API_Nextflow.ipynb @@ -21,6 +21,7 @@ }, { "cell_type": "markdown", + "id": "ffe82256", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -30,6 +31,7 @@ }, { "cell_type": "markdown", + "id": "69387ba0", "metadata": {}, "source": [ "## Prerequisites\n", @@ -46,6 +48,7 @@ }, { "cell_type": "markdown", + "id": "538cfc56", "metadata": {}, "source": [ "## Get Started" @@ -54,10 +57,7 @@ { "cell_type": "markdown", "id": "0f8f4b85-9459-497d-97ec-5909e8aeacae", - "metadata": { - "id": "0f8f4b85-9459-497d-97ec-5909e8aeacae", - "tags": [] - }, + "metadata": {}, "source": [ "### Install packages and setup your environment" ] @@ -65,9 +65,7 @@ { "cell_type": "markdown", "id": "f2e4a5ca-8a2b-4156-b83e-c89f0c1ffc9c", - "metadata": { - "id": "f2e4a5ca-8a2b-4156-b83e-c89f0c1ffc9c" - }, + "metadata": {}, "source": [ "#### Create a bucket" ] @@ -154,10 +152,7 @@ "cell_type": "code", "execution_count": null, "id": "3cb5bd4b-032a-47f0-bee4-299a547c3b48", - "metadata": { - "id": "3cb5bd4b-032a-47f0-bee4-299a547c3b48", - "outputId": "b0e740fa-dabc-4d45-c95b-15f72d32bffa" - }, + "metadata": {}, "outputs": [], "source": [ "! gcloud beta lifesciences pipelines run \\\n", @@ -170,10 +165,7 @@ { "cell_type": "markdown", "id": "f4892a16-f4d9-4db9-a171-6e9245df2a72", - "metadata": { - "id": "f4892a16-f4d9-4db9-a171-6e9245df2a72", - "tags": [] - }, + "metadata": {}, "source": [ "### Check job status\n", "To check the job status enter operation ID from the gcloud output\n", @@ -197,12 +189,7 @@ "cell_type": "code", "execution_count": null, "id": "9cba7c4e-4b8c-4e4c-80e4-8de1f11b5790", - "metadata": { - "id": "9cba7c4e-4b8c-4e4c-80e4-8de1f11b5790", - "outputId": "47886ae8-869f-46d3-a6fa-a2b56242be9b", - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! gcloud beta lifesciences operations describe $ID" @@ -211,9 +198,7 @@ { "cell_type": "markdown", "id": "9f056585-6c10-41b6-b7b6-0c75bebed811", - "metadata": { - "id": "9f056585-6c10-41b6-b7b6-0c75bebed811" - }, + "metadata": {}, "source": [ "### View your output" ] @@ -232,10 +217,7 @@ "cell_type": "code", "execution_count": null, "id": "02faf944-0143-49c7-bf4c-6b8e377fcd81", - "metadata": { - "id": "02faf944-0143-49c7-bf4c-6b8e377fcd81", - "outputId": "251ad4db-dcea-4d72-ff9a-01b3080acc8e" - }, + "metadata": {}, "outputs": [], "source": [ "! gsutil cp gs://$BUCKET/hello_world.log .\n", @@ -245,9 +227,7 @@ { "cell_type": "markdown", "id": "33a142e0-bd9a-405d-91f9-827503ff5fb1", - "metadata": { - "id": "33a142e0-bd9a-405d-91f9-827503ff5fb1" - }, + "metadata": {}, "source": [ "## Run Nextflow Locally" ] @@ -263,9 +243,7 @@ { "cell_type": "markdown", "id": "b709c718-96d0-4925-99dd-525a7e7b6c76", - "metadata": { - "id": "b709c718-96d0-4925-99dd-525a7e7b6c76" - }, + "metadata": {}, "source": [ "Nextflow interacts with many different files to have a proper working workflow:\n", "\n", @@ -281,9 +259,7 @@ { "cell_type": "markdown", "id": "9bea3004-ff40-4918-ac16-83aad9427ad7", - "metadata": { - "id": "9bea3004-ff40-4918-ac16-83aad9427ad7" - }, + "metadata": {}, "source": [ "### Run a nextflow 'Hello World' process locally" ] @@ -291,9 +267,7 @@ { "cell_type": "markdown", "id": "4715ef92-e3a6-44cf-9b1e-50f247dd0daf", - "metadata": { - "id": "4715ef92-e3a6-44cf-9b1e-50f247dd0daf" - }, + "metadata": {}, "source": [ "We are going to first run Hello World locally using the config file called hello.nf. \n", "\n", @@ -327,10 +301,7 @@ "cell_type": "code", "execution_count": null, "id": "6efad386-185b-4faf-be39-6c5a3f84ffe4", - "metadata": { - "id": "6efad386-185b-4faf-be39-6c5a3f84ffe4", - "outputId": "9554903e-f8d5-43fa-ffe9-f00ce836bf2d" - }, + "metadata": {}, "outputs": [], "source": [ "! ./nextflow run hello.nf --str 'Hello!'" @@ -339,10 +310,7 @@ { "cell_type": "markdown", "id": "7619875d-7f10-4699-b4d2-120d5d7d4cd7", - "metadata": { - "id": "7619875d-7f10-4699-b4d2-120d5d7d4cd7", - "tags": [] - }, + "metadata": {}, "source": [ "## Submit Nextflow Job to the Life Sciences API\n", "Create and modify your own config file to include a 'gls' profile block to tell Nextflow to submit the job to the API instead of running locally" @@ -351,9 +319,7 @@ { "cell_type": "markdown", "id": "ec7abe9b-dca1-4ef6-87d6-39fcdd2e3c9b", - "metadata": { - "id": "ec7abe9b-dca1-4ef6-87d6-39fcdd2e3c9b" - }, + "metadata": {}, "source": [ "The config file allows nextflow to utilize excecuters like Life Science API. In this tutorial the config files is named __'nextflow.config'__. Make sure you open this file and update the `` that are account specific.\n", "- Make sure that your region is a region included in the LS API!\n", @@ -381,10 +347,7 @@ { "cell_type": "markdown", "id": "340f7300-449a-4a12-bbc5-073547d58cac", - "metadata": { - "id": "340f7300-449a-4a12-bbc5-073547d58cac", - "tags": [] - }, + "metadata": {}, "source": [ "### Optional: Listing nf-core tools with docker and viewing their commands\n", "Using the command below you can see all the tools that nfcore holds and their versions/lastes releases" @@ -394,12 +357,7 @@ "cell_type": "code", "execution_count": null, "id": "ca1ff164-cee2-446e-ab2e-a3ed984e0dc0", - "metadata": { - "id": "ca1ff164-cee2-446e-ab2e-a3ed984e0dc0", - "outputId": "0530644a-dd9a-4077-dbc8-d1e335788a01", - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! docker run nfcore/tools list" @@ -408,9 +366,7 @@ { "cell_type": "markdown", "id": "9e46373c-61d0-4c91-b001-e55568d9fa2d", - "metadata": { - "id": "9e46373c-61d0-4c91-b001-e55568d9fa2d" - }, + "metadata": {}, "source": [ "You can view commands for methylseq (or any other specified nf-core tool) by using the [--help] flag" ] @@ -419,12 +375,7 @@ "cell_type": "code", "execution_count": null, "id": "05ea2893-60b3-4934-ae86-b07d4bc59728", - "metadata": { - "id": "05ea2893-60b3-4934-ae86-b07d4bc59728", - "outputId": "1e6de26f-0433-4bbd-8a43-119097bb1f41", - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! ./nextflow run nf-core/methylseq -r 1.6.1 --help" @@ -433,10 +384,7 @@ { "cell_type": "markdown", "id": "b4dbef59-d619-4444-8870-18c1f0ba3b5c", - "metadata": { - "id": "b4dbef59-d619-4444-8870-18c1f0ba3b5c", - "tags": [] - }, + "metadata": {}, "source": [ "### Run Methylseq with the test profile" ] @@ -444,9 +392,7 @@ { "cell_type": "markdown", "id": "7238bd3e-1853-42c3-9d2d-c72e46975ff2", - "metadata": { - "id": "7238bd3e-1853-42c3-9d2d-c72e46975ff2" - }, + "metadata": {}, "source": [ "The 'test' profile uses a small dataset allowing you to ensure the workflow works with your config file without long runtimes. Ensure you include:\n", "- Version of the nf-core tool [-r]\n", @@ -457,12 +403,7 @@ "cell_type": "code", "execution_count": null, "id": "4b21f170-37fa-4fbc-ab83-3f6b4d386ef9", - "metadata": { - "id": "4b21f170-37fa-4fbc-ab83-3f6b4d386ef9", - "outputId": "0507c847-7f83-40af-ebf0-a1fdef27499b", - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! ./nextflow run nf-core/methylseq -r 1.6.1 -profile test,gls -c nextflow-methyseq.config" @@ -471,15 +412,14 @@ { "cell_type": "markdown", "id": "e386ccb3-aa6d-4a77-8d7d-c20ed0419f84", - "metadata": { - "id": "e386ccb3-aa6d-4a77-8d7d-c20ed0419f84" - }, + "metadata": {}, "source": [ "You will notice in the above that to the left of the process within the __[ ]__ is actually a __tag__ you can search in Life Sciences and the text before the __/__ corresponds to the __temporary directories__ within your working directory. Feel free to delete the temporary directories once your workflow has succesfully completed." ] }, { "cell_type": "markdown", + "id": "8686bbae", "metadata": {}, "source": [ "## Conclusion\n", @@ -498,35 +438,7 @@ ] } ], - "metadata": { - "colab": { - "name": "Workshop_2_updated.ipynb", - "provenance": [] - }, - "environment": { - "kernel": "python3", - "name": "common-cpu.m93", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m93" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.12" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/LifeSciencesAPI/nextflow/Part2_LS_API_Nextflow.ipynb b/notebooks/LifeSciencesAPI/nextflow/Part2_LS_API_Nextflow.ipynb index 2bde974..7e944f3 100644 --- a/notebooks/LifeSciencesAPI/nextflow/Part2_LS_API_Nextflow.ipynb +++ b/notebooks/LifeSciencesAPI/nextflow/Part2_LS_API_Nextflow.ipynb @@ -21,6 +21,7 @@ }, { "cell_type": "markdown", + "id": "95787f70", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -30,6 +31,7 @@ }, { "cell_type": "markdown", + "id": "09e5e6fd", "metadata": {}, "source": [ "## Prerequisites\n", @@ -46,6 +48,7 @@ }, { "cell_type": "markdown", + "id": "8d54ab3e", "metadata": {}, "source": [ "## Get Started" @@ -344,6 +347,7 @@ }, { "cell_type": "markdown", + "id": "0fea8823", "metadata": {}, "source": [ "## Conclusion\n", @@ -369,31 +373,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m93", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m93" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.12" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/LifeSciencesAPI/snakemake/LS_API_Snakemake.ipynb b/notebooks/LifeSciencesAPI/snakemake/LS_API_Snakemake.ipynb index fdb7db8..6dbd93f 100644 --- a/notebooks/LifeSciencesAPI/snakemake/LS_API_Snakemake.ipynb +++ b/notebooks/LifeSciencesAPI/snakemake/LS_API_Snakemake.ipynb @@ -28,6 +28,7 @@ }, { "cell_type": "markdown", + "id": "28c8d1b9", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -37,6 +38,7 @@ }, { "cell_type": "markdown", + "id": "310852bd", "metadata": {}, "source": [ "## Prerequisites\n", @@ -53,6 +55,7 @@ }, { "cell_type": "markdown", + "id": "4633decf", "metadata": {}, "source": [ "## Get Started" @@ -103,9 +106,7 @@ { "cell_type": "markdown", "id": "dd7ab630-955d-43d1-bc43-c7b3e701ed04", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "#### Install packages\n", "First install mambaforge, then use mamba to install snakemake." @@ -206,9 +207,7 @@ { "cell_type": "markdown", "id": "ea2d17cb-dff6-45d3-9aef-3ec6203508f6", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "### Copy data file for Trimmomatic" ] @@ -413,10 +412,7 @@ "cell_type": "code", "execution_count": null, "id": "bee32318-33df-43b2-98bc-5eb091ceae59", - "metadata": { - "scrolled": true, - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "%%time\n", @@ -550,6 +546,7 @@ }, { "cell_type": "markdown", + "id": "878da035", "metadata": {}, "source": [ "## Conclusions\n", @@ -558,6 +555,7 @@ }, { "cell_type": "markdown", + "id": "ecb597e4", "metadata": {}, "source": [ "## Clean up\n", @@ -567,31 +565,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "tf2-gpu.2-11.m110", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-11:m110" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/SRADownload/SRA-Download.ipynb b/notebooks/SRADownload/SRA-Download.ipynb index b20c5d3..6382112 100644 --- a/notebooks/SRADownload/SRA-Download.ipynb +++ b/notebooks/SRADownload/SRA-Download.ipynb @@ -20,6 +20,7 @@ }, { "cell_type": "markdown", + "id": "c5ee4e60", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -39,6 +40,7 @@ }, { "cell_type": "markdown", + "id": "30a4c94b", "metadata": {}, "source": [ "## Get Started" @@ -139,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "9aa93698-a082-4c11-9d48-0abe775fbcc5", "metadata": {}, "outputs": [], @@ -287,9 +289,7 @@ { "cell_type": "markdown", "id": "a53f13c3-6b62-4408-84d8-ebad27c2eedb", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "### Download FASTQ Files with fasterq dump" ] @@ -443,6 +443,7 @@ }, { "cell_type": "markdown", + "id": "6e9f441d", "metadata": {}, "source": [ "## Conclusions\n", @@ -459,31 +460,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "tf2-gpu.2-11.m110", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-11:m110" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/SpleenLiverSegmentation/SpleenSeg_Pretrained-4_27.ipynb b/notebooks/SpleenLiverSegmentation/SpleenSeg_Pretrained-4_27.ipynb index fc3b9a3..defde63 100644 --- a/notebooks/SpleenLiverSegmentation/SpleenSeg_Pretrained-4_27.ipynb +++ b/notebooks/SpleenLiverSegmentation/SpleenSeg_Pretrained-4_27.ipynb @@ -13,6 +13,7 @@ }, { "cell_type": "markdown", + "id": "139e005d", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -22,6 +23,7 @@ }, { "cell_type": "markdown", + "id": "a3698e08", "metadata": {}, "source": [ "## Prerequisites\n", @@ -30,6 +32,7 @@ }, { "cell_type": "markdown", + "id": "125a6cec", "metadata": {}, "source": [ "## Get Started" @@ -37,6 +40,7 @@ }, { "cell_type": "markdown", + "id": "f1ed036a", "metadata": {}, "source": [ "### Install packages" @@ -958,6 +962,7 @@ }, { "cell_type": "markdown", + "id": "bf745751", "metadata": {}, "source": [ "## Conclusions\n", @@ -966,6 +971,7 @@ }, { "cell_type": "markdown", + "id": "4f77d08e", "metadata": {}, "source": [ "## Clean Up\n", @@ -974,34 +980,12 @@ }, { "cell_type": "markdown", + "id": "9903aa80", "metadata": {}, "source": [] } ], - "metadata": { - "environment": { - "name": "pytorch-gpu.1-9.m75", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/pytorch-gpu.1-9:m75" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/elasticBLAST/run_elastic_blast.ipynb b/notebooks/elasticBLAST/run_elastic_blast.ipynb index 159c3af..ed083aa 100644 --- a/notebooks/elasticBLAST/run_elastic_blast.ipynb +++ b/notebooks/elasticBLAST/run_elastic_blast.ipynb @@ -18,6 +18,7 @@ }, { "cell_type": "markdown", + "id": "6f02c8f0", "metadata": {}, "source": [ "## Overview\n", @@ -26,6 +27,7 @@ }, { "cell_type": "markdown", + "id": "47b18f7e", "metadata": {}, "source": [ "## Prerequisites\n", @@ -34,6 +36,7 @@ }, { "cell_type": "markdown", + "id": "c30ecdab", "metadata": {}, "source": [ "## Learning objectives\n", @@ -43,6 +46,7 @@ }, { "cell_type": "markdown", + "id": "65c9e24a", "metadata": {}, "source": [ "## Get Started" @@ -206,6 +210,7 @@ }, { "cell_type": "markdown", + "id": "6684aca5", "metadata": {}, "source": [ "## Conclusion\n", @@ -231,31 +236,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "common-cpu.m93", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/base-cpu:m93" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.12" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/ncbi-stat-tutorial/STAT-tutorial.ipynb b/notebooks/ncbi-stat-tutorial/STAT-tutorial.ipynb index 364191f..2280df0 100644 --- a/notebooks/ncbi-stat-tutorial/STAT-tutorial.ipynb +++ b/notebooks/ncbi-stat-tutorial/STAT-tutorial.ipynb @@ -22,6 +22,7 @@ }, { "cell_type": "markdown", + "id": "53e5567b", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -31,6 +32,7 @@ }, { "cell_type": "markdown", + "id": "4c2262ee", "metadata": {}, "source": [ "## Prerequisites\n", @@ -39,6 +41,7 @@ }, { "cell_type": "markdown", + "id": "a8783be0", "metadata": {}, "source": [ "## Get Started" @@ -263,6 +266,7 @@ }, { "cell_type": "markdown", + "id": "2f8d42ae", "metadata": {}, "source": [ "## Conclusion\n", @@ -271,6 +275,7 @@ }, { "cell_type": "markdown", + "id": "e7080684", "metadata": {}, "source": [ "## Clean Up\n", @@ -278,31 +283,7 @@ ] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "tf2-gpu.2-11.m109", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-11:m109" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/notebooks/pangolin/pangolin_pipeline.ipynb b/notebooks/pangolin/pangolin_pipeline.ipynb index 34904ef..09460fc 100644 --- a/notebooks/pangolin/pangolin_pipeline.ipynb +++ b/notebooks/pangolin/pangolin_pipeline.ipynb @@ -10,6 +10,7 @@ }, { "cell_type": "markdown", + "id": "ca6ac8fb", "metadata": {}, "source": [ "## Overview" @@ -25,6 +26,7 @@ }, { "cell_type": "markdown", + "id": "ac70c296", "metadata": {}, "source": [ "## Learning Objectives\n", @@ -33,6 +35,7 @@ }, { "cell_type": "markdown", + "id": "bbf7275a", "metadata": {}, "source": [ "## Prerequisites\n", @@ -41,6 +44,7 @@ }, { "cell_type": "markdown", + "id": "42531d09", "metadata": {}, "source": [ "## Get Started" @@ -58,9 +62,7 @@ "cell_type": "code", "execution_count": null, "id": "f994b990", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#change this depending on how many threads are available in your notebook\n", @@ -71,9 +73,7 @@ "cell_type": "code", "execution_count": null, "id": "f421805e", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#install biopython to import packages below\n", @@ -116,9 +116,7 @@ "cell_type": "code", "execution_count": null, "id": "fd936fd6", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! mamba install -y -c conda-forge -c bioconda -c etetoolkit sra-tools pangolin ete3 minimap2 -y" @@ -128,9 +126,7 @@ "cell_type": "code", "execution_count": null, "id": "5a99cf0d", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#import libraries\n", @@ -151,9 +147,7 @@ "cell_type": "code", "execution_count": null, "id": "8f831fca", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "if not os.path.exists('pangolin_analysis'):\n", @@ -165,9 +159,7 @@ "cell_type": "code", "execution_count": null, "id": "6423ca5d", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "if os.path.exists('sarscov2_sequences.fasta'):\n", @@ -188,9 +180,7 @@ "cell_type": "code", "execution_count": null, "id": "16824bcf", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#give a list of accession number for sars sequences\n", @@ -210,9 +200,7 @@ "cell_type": "code", "execution_count": null, "id": "a28a7122", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#use the bio.entrez toolkit within biopython to download the accession numbers\n", @@ -236,9 +224,7 @@ "cell_type": "code", "execution_count": null, "id": "56acb7cc", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#make sure our fasta file has the same number of seqs as the acc_nums list\n", @@ -250,9 +236,7 @@ "cell_type": "code", "execution_count": null, "id": "8606c352", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "#let's peek at our new fasta file\n", @@ -262,9 +246,7 @@ { "cell_type": "markdown", "id": "2db37b4e", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "### Run pangolin to identify lineages and output alignment\n", "Here we call pangolin, give it our input sequences and the number of threads. We also tell it to output the alignment. The full list of pangolin parameters can be found in the [docs](https://cov-lineages.org/resources/pangolin/usage.html)." @@ -284,9 +266,7 @@ "cell_type": "code", "execution_count": null, "id": "f1a17a74", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "! pangolin sarscov2_seqs.fasta --threads $CPU" @@ -302,6 +282,7 @@ }, { "cell_type": "markdown", + "id": "8678c069", "metadata": {}, "source": [ "## Conclusions\n", @@ -310,6 +291,7 @@ }, { "cell_type": "markdown", + "id": "d2969976", "metadata": {}, "source": [ "## Clean Up\n", @@ -318,35 +300,12 @@ }, { "cell_type": "markdown", + "id": "b71cfeee", "metadata": {}, "source": [] } ], - "metadata": { - "environment": { - "kernel": "python3", - "name": "tf2-gpu.2-11.m110", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-11:m110" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 }