Update API to deploy separate endpoints by run ID #2
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| # Setup ------------------------------------------------------------------------ | ||
| library(arrow) | ||
| library(assertthat) | ||
| library(aws.s3) | ||
| library(ccao) | ||
| library(dplyr) | ||
|
|
@@ -11,91 +12,130 @@ source("generics.R") | |
| # Read AWS creds from Docker secrets | ||
| if (file.exists("/run/secrets/ENV_FILE")) { | ||
| readRenviron("/run/secrets/ENV_FILE") | ||
| } else if (file.exists("secrets/ENV_FILE")) { | ||
| readRenviron("secrets/ENV_FILE") | ||
| } else { | ||
| readRenviron(".env") | ||
| } | ||
|
|
||
| # Get the model run attributes at runtime from env vars | ||
| dvc_bucket <- Sys.getenv("AWS_S3_DVC_BUCKET") | ||
| run_bucket <- Sys.getenv("AWS_S3_MODEL_BUCKET") | ||
dfsnow marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| api_port <- as.numeric(Sys.getenv("API_PORT", unset = "3636")) | ||
| default_run_id_var_name <- "AWS_S3_MODEL_RUN_ID" | ||
|
There was a problem hiding this comment. It might be nice to change the name of this env var to something that more clearly marks it as the default (like There was a problem hiding this comment. Let's change it now and make a list of things we actually need to change when redeploying. There was a problem hiding this comment. Done in abe5403, with an updated list of deploy steps in the PR body. |
||
| default_run_id <- Sys.getenv(default_run_id_var_name) | ||
|
|
||
| # The list of run IDs that will be deployed as possible model endpoints | ||
| valid_run_ids <- c( | ||
| "2024-02-06-relaxed-tristan", | ||
| "2024-03-17-stupefied-maya" | ||
|
There was a problem hiding this comment. Any other run IDs that we should include in this vector for now? There was a problem hiding this comment. Let's actually include the final 2022 and 2023 models. We can just reproduce and replace the old workbooks for those years. There was a problem hiding this comment. Cool, done in abe5403. Are the old workbooks even still operational, given that the model version in the (currently static) API has changed since 2022 and 2023? |
||
| ) | ||
|
|
||
| assert_that( | ||
| default_run_id %in% valid_run_ids, | ||
| msg = sprintf( | ||
| "%s must be a valid run_id - got '%s', expected one of: %s", | ||
| default_run_id_var_name, | ||
| default_run_id, | ||
| paste(valid_run_ids, collapse = ", ") | ||
| ) | ||
| ) | ||
|
|
||
| # Given a run ID, return a model object that can be used to power a | ||
| # vetiver API endpoint | ||
| get_model_from_run_id <- function(run_id) { | ||
| run_year = substr(run_id, 1, 4) | ||
|
|
||
| # Download Files ------------------------------------------------------------- | ||
|
|
||
| # Grab model fit and recipe objects | ||
| temp_file_fit <- tempfile(fileext = ".zip") | ||
| aws.s3::save_object( | ||
| object = file.path( | ||
| run_bucket, "workflow/fit", | ||
| paste0("year=", run_year), | ||
| paste0(run_id, ".zip") | ||
| ), | ||
| file = temp_file_fit | ||
jeancochrane marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| ) | ||
|
|
||
| temp_file_recipe <- tempfile(fileext = ".rds") | ||
| aws.s3::save_object( | ||
| object = file.path( | ||
| run_bucket, "workflow/recipe", | ||
| paste0("year=", run_year), | ||
| paste0(run_id, ".rds") | ||
| ), | ||
| file = temp_file_recipe | ||
| ) | ||
|
|
||
| # Grab metadata file for the specified run | ||
| metadata <- read_parquet( | ||
| file.path( | ||
| run_bucket, "metadata", | ||
| paste0("year=", run_year), | ||
| paste0(run_id, ".parquet") | ||
| ) | ||
| ) | ||
|
|
||
| # Load the training data used for this model | ||
| training_data_md5 <- metadata$dvc_md5_training_data | ||
| training_data <- read_parquet( | ||
| file.path( | ||
| dvc_bucket, | ||
| substr(training_data_md5, 1, 2), | ||
| substr(training_data_md5, 3, nchar(training_data_md5)) | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| # Load Model ----------------------------------------------------------------- | ||
|
|
||
| # Load fit and recipe from file | ||
| fit <- lightsnip::lgbm_load(temp_file_fit) | ||
| recipe <- readRDS(temp_file_recipe) | ||
|
|
||
| # Extract a sample row of predictors to use for the API docs | ||
| predictors <- recipe$var_info %>% | ||
| filter(role == "predictor") %>% | ||
| pull(variable) | ||
| ptype_tbl <- training_data %>% | ||
| filter(meta_pin == "15251030220000") %>% | ||
| select(all_of(predictors)) | ||
| ptype <- vetiver_create_ptype(model = fit, save_prototype = ptype_tbl) | ||
|
|
||
|
|
||
| # Create API ----------------------------------------------------------------- | ||
|
|
||
| # Create model object and populate metadata | ||
| model <- vetiver_model(fit, "LightGBM", save_prototype = ptype) | ||
| model$recipe <- recipe | ||
| model$pv$round_type <- metadata$pv_round_type | ||
| model$pv$round_break <- metadata$pv_round_break[[1]] | ||
| model$pv$round_to_nearest <- metadata$pv_round_to_nearest[[1]] | ||
|
|
||
| return(model) | ||
| } | ||
|
|
||
| default_model <- get_model_from_run_id(default_run_id) | ||
|
|
||
| router <- pr() %>% | ||
| # Point the /predict endpoint to the default model | ||
| vetiver_api(default_model) | ||
|
|
||
| # Create endpoints for each model based on run ID and add them to the router | ||
| for (run_id in valid_run_ids) { | ||
| model <- get_model_from_run_id(run_id) | ||
| vetiver_api( | ||
| router, | ||
| model, | ||
| path = sprintf("/predict/%s", run_id) | ||
| ) | ||
| } | ||
|
|
||
| # Start API | ||
| pr_run( | ||
| router, | ||
| host = "0.0.0.0", | ||
| port = api_port | ||
| ) | ||
|
There was a problem hiding this comment. I don't remember why we're running There was a problem hiding this comment. Done in d8bf46f, let's see what happens! |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The logic here was a bit confusing for local development, where neither
/run/secrets/ENV_FILEnorsecrets/ENV_FILEexist. I think the new conditional structure should make local development easier, but let me know if I'm misinterpreting something.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I agree this is pretty confusing and underdocumented. If I recall, the file
secrets/ENV_FILEis mounted to/run/secrets/ENV_FILEwhen using compose. This file contains AWS creds specific to the API. During development, these creds would be unnecessary as the user would likely have active AWS creds viaaws-mfa.The
.envfile is separate and not related. It contains the rest of the setup vars used by compose (API_PORT,CCAO_REGISTRY_URL, etc.) and the final model ID and year. In other words, it's just config stuff, not actually secret. This file is necessary to load during development, but NOT when deployed via compose (compose adds all vars in a .env file to the deployed container). So, the logic here makes sense if you remove the change on line 15.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That makes sense, I refactored in abe5403 to always load this file and only load
secrets/ENV_FILEif it exists (similar to/run/secrets/ENV_FILE).