Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FIM Caching Fixes #608

Merged
merged 1 commit into from
Jan 3, 2024
Merged

FIM Caching Fixes #608

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Core/EC2/RDSBastion/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ variable "viz_redshift_name" {
type = string
}

variable "viz_redshift_iam_role" {
variable "viz_redshift_iam_role_arn" {
type = string
}

Expand Down Expand Up @@ -378,6 +378,7 @@ data "cloudinit_config" "startup" {
viz_redshift_master_password = local.dbs["viz_redshift"]["db_password"]
viz_redshift_user_username = jsondecode(var.viz_redshift_user_secret_string)["username"]
viz_redshift_user_password = jsondecode(var.viz_redshift_user_secret_string)["password"]
viz_redshift_iam_role_arn = var.viz_redshift_iam_role_arn
viz_db_name = local.dbs["viz"]["db_name"]
viz_db_host = local.dbs["viz"]["db_host"]
viz_db_port = local.dbs["viz"]["db_port"]
Expand Down
6 changes: 3 additions & 3 deletions Core/EC2/RDSBastion/scripts/viz/redshift_setup.sh.tftpl
View file
Open in desktop
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I set these usernames and passwords correctly? You may need to doublecheck.

Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@ echo "---- SETTING UP VIZ REDSHIFT DB ----"


# Setting up Viz Redshift DB
export PGPASSWORD=${viz_db_password}
export PGPASSWORD=${viz_redshift_master_password}

# Adding users to Viz Redshift DB
echo "Adding viz redshift user..."
psql -h "${viz_redshift_host}" -U "${viz_redshift_username}" -p ${viz_redshift_port} -d "${viz_redshift_name}" -qtAc "CREATE USER ${viz_redshift_user_username} WITH PASSWORD '${viz_redshift_user_password}';"
psql -h "${viz_redshift_host}" -U "${viz_redshift_master_username}" -p ${viz_redshift_port} -d "${viz_redshift_name}" -qtAc "CREATE USER ${viz_redshift_user_username} WITH PASSWORD '${viz_redshift_user_password}';"

# Setup external schemas - linked to viz processing rds datbase - this could be abstracted as done with the viz processing foreign schemas, but I'm not doing that now since it is only one schema we need.
echo "Adding external schema link to viz ingest ..."
psql -h "${viz_redshift_host}" -U "${viz_redshift_username}" -p ${viz_redshift_port} -d "${viz_redshift_name}" \
psql -h "${viz_redshift_host}" -U "${viz_redshift_master_username}" -p ${viz_redshift_port} -d "${viz_redshift_name}" \
-tAc "DROP SCHEMA IF EXISTS external_viz_ingest;
CREATE EXTERNAL SCHEMA external_viz_ingest
FROM POSTGRES
Expand Down
4 changes: 2 additions & 2 deletions Core/IAM/Roles/redshift.json.tftpl
View file
Open in desktop
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No underscored allowed in IAM Policies.

Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"Version": "2012-10-17",
"Statement": [
{
"Sid": "s3_access",
"Sid": "s3Access",
"Action": [
"s3:PutObject",
"s3:Get*",
Expand All @@ -14,7 +14,7 @@
]
},
{
"Sid": "secret_for_external_data_schema",
"Sid": "secretForExternalDataSchema",
"Action": [
"secretsmanager:DescribeSecret",
"secretsmanager:GetSecretValue"
Expand Down
10 changes: 9 additions & 1 deletion Core/LAMBDA/viz_functions/image_based/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ resource "null_resource" "viz_raster_processing_cluster" {
source_hash = data.archive_file.raster_processing_zip.output_md5
}

depends_on = [ aws_s3_object.raster_processing_zip_upload ]

provisioner "local-exec" {
command = "aws codebuild start-build --project-name ${aws_codebuild_project.viz_raster_processing_lambda.name} --profile ${var.environment} --region ${var.region}"
}
Expand Down Expand Up @@ -314,6 +316,8 @@ resource "null_resource" "viz_optimize_rasters_cluster" {
source_hash = data.archive_file.optimize_rasters_zip.output_md5
}

depends_on = [ aws_s3_object.optimize_rasters_zip_upload ]

provisioner "local-exec" {
command = "aws codebuild start-build --project-name ${aws_codebuild_project.viz_optimize_raster_lambda.name} --profile ${var.environment} --region ${var.region}"
}
Expand Down Expand Up @@ -455,6 +459,8 @@ resource "null_resource" "viz_hand_fim_processing_cluster" {
fim_version = var.fim_version
}

depends_on = [ aws_s3_object.hand_fim_processing_zip_upload ]

provisioner "local-exec" {
command = "aws codebuild start-build --project-name ${aws_codebuild_project.viz_hand_fim_processing_lambda.name} --profile ${var.environment} --region ${var.region}"
}
Expand Down Expand Up @@ -484,7 +490,7 @@ data "aws_lambda_function" "viz_hand_fim_processing" {

data "archive_file" "schism_processing_zip" {
type = "zip"
output_path = "${path.module}/temp/viz_schism_fim_processing__${var.environment}_${var.region}.zip"
output_path = "${path.module}/temp/viz_schism_fim_processing_${var.environment}_${var.region}.zip"

dynamic "source" {
for_each = fileset("${path.module}/viz_schism_fim_processing", "**")
Expand Down Expand Up @@ -593,6 +599,8 @@ resource "null_resource" "viz_schism_fim_processing_cluster" {
source_hash = data.archive_file.schism_processing_zip.output_md5
}

depends_on = [ aws_s3_object.schism_processing_zip_upload ]

provisioner "local-exec" {
command = "aws codebuild start-build --project-name ${aws_codebuild_project.viz_schism_fim_processing_lambda.name} --profile ${var.environment} --region ${var.region}"
}
Expand Down
6 changes: 3 additions & 3 deletions Core/LAMBDA/viz_functions/viz_publish_service/lambda_function.py
View file
Open in desktop
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Small fix for an edge-case where EGIS messes up and provides garbage data win the list of services.

Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ def lambda_handler(event, context):

# Check to see if the service already exists and a publish flag is present or not.
time.sleep(30)
matching_services = [service for service in publish_server.services.list(folder=folder) if service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish] # noqa: E501
matching_services = [service for service in publish_server.services.list(folder=folder) if 'serviceName' in service.properties and (service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish)] # noqa: E501
publish_flag = s3_file(publish_flag_bucket, publish_flag_key).check_existence()
if len(matching_services) > 0 and publish_flag is True:
print(f"{matching_services[0].properties['serviceName']} is already online.")
Expand Down Expand Up @@ -77,15 +77,15 @@ def lambda_handler(event, context):
print(f"---> Published {sd_s3_path}")

# Ensuring that the description for the service matches the iteminfo
matching_service = [service for service in publish_server.services.list(folder=folder) if service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish][0]
matching_service = [service for service in publish_server.services.list(folder=folder) if 'serviceName' in service.properties and (service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish)][0]
if not matching_service.properties['description']:
print("Updating service property description to match iteminfo")
service_properties = matching_service.properties
service_properties['description'] = matching_service.iteminformation.properties['description']
try:
matching_service.edit(dict(service_properties))
except:
matching_service = [service for service in publish_server.services.list(folder=folder) if service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish][0]
matching_service = [service for service in publish_server.services.list(folder=folder) if 'serviceName' in service.properties and (service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish)][0]
if not matching_service.properties['description']:
raise Exception("Failed to update the map service description")

Expand Down
6 changes: 5 additions & 1 deletion Core/Redshift/viz/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,13 @@ resource "aws_route53_record" "viz_redshift_data_warehouse" {
name = "redshift-viz.${var.private_route_53_zone.name}"
type = "CNAME"
ttl = 300
records = [aws_redshift_cluster.viz_redshift_data_warehouse.address]
records = [aws_redshift_cluster.viz_redshift_data_warehouse.dns_name]
}

output "dns_name" {
value = aws_route53_record.viz_redshift_data_warehouse.name
}

output "port" {
value = aws_redshift_cluster.viz_redshift_data_warehouse.port
}
2 changes: 1 addition & 1 deletion Core/SecretsManager/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@ output "secret_strings" {
value = { for name in keys(var.names_and_users) : name => module.secret[name].secret_string }
}

output "arns" {
output "secret_arns" {
value = { for name in keys(var.names_and_users) : name => module.secret[name].secret_arn }
}
2 changes: 1 addition & 1 deletion Core/SecurityGroups/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ resource "aws_security_group" "redshift" {
protocol = "tcp"
security_groups = []
self = false
to_port = 22
to_port = 5439
}
]

Expand Down
14 changes: 7 additions & 7 deletions Core/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ module "secrets-manager" {
"viz-processing-pg-rdssecret" = { "username" : "postgres" }
"viz-proc-admin-rw-user" = { "username" : "viz_proc_admin_rw_user" }
"viz-proc-dev-rw-user" = { "username" : "viz_proc_dev_rw_user" }
"viz_redshift_master" = { "username" : "viz_redshift_master" }
"viz_redshift_user" = { "username" : "viz_redshift_user" }
"viz-redshift-master" = { "username" : "viz_redshift_master" }
"viz-redshift-user" = { "username" : "viz_redshift_user" }
"ingest-pg-rdssecret" = { "username" : "postgres" }
"ingest-mqsecret" = { "username" : "rabbit_admin" }
"rds-rfc-fcst" = { "username" : "rfc_fcst" }
Expand Down Expand Up @@ -352,7 +352,7 @@ module "redshift-viz" {
db_viz_redshift_user_secret_string = module.secrets-manager.secret_strings["viz-redshift-user"]
db_viz_redshift_security_groups = [module.security-groups.redshift.id]
viz_redshift_db_name = local.env.viz_redshift_db_name
role_viz_redshift_arn = module.iam-roles.redshift.arn
role_viz_redshift_arn = module.iam-roles.role_redshift.arn

private_route_53_zone = module.private-route53.zone
}
Expand Down Expand Up @@ -416,12 +416,12 @@ module "rds-bastion" {
viz_db_address = module.rds-viz.instance.address
viz_db_port = module.rds-viz.instance.port
viz_db_name = local.env.viz_db_name
viz_redshift_master_secret_string = module.secrets-manager.secret_strings["viz_redshift_master"]
viz_redshift_user_secret_string = module.secrets-manager.secret_strings["viz_redshift_user"]
viz_redshift_master_secret_string = module.secrets-manager.secret_strings["viz-redshift-master"]
viz_redshift_user_secret_string = module.secrets-manager.secret_strings["viz-redshift-user"]
viz_redshift_address = module.redshift-viz.dns_name
viz_redshift_port = module.redshift-viz.port
viz_redshift_name = local.env.viz_redshift_db_name
viz_redshift_iam_role = module.iam-roles.redshift.arn
viz_redshift_iam_role_arn = module.iam-roles.role_redshift.arn
egis_db_master_secret_string = module.secrets-manager.secret_strings["egis-master-pg-rds-secret"]
egis_db_secret_string = module.secrets-manager.secret_strings["egis-pg-rds-secret"]
egis_db_address = module.rds-egis.dns_name
Expand Down Expand Up @@ -642,7 +642,7 @@ module "viz-lambda-functions" {
egis_portal_password = local.env.viz_ec2_hydrovis_egis_pass
viz_redshift_host = module.redshift-viz.dns_name
viz_redshift_db_name = local.env.viz_redshift_db_name
viz_redshift_user_secret_string = module.secrets-manager.secret_strings["viz_redshift_user"]
viz_redshift_user_secret_string = module.secrets-manager.secret_strings["viz-redshift-user"]
dataservices_host = module.data-services.dns_name
viz_pipeline_step_function_arn = module.step-functions.viz_pipeline_step_function.arn
default_tags = local.env.tags
Expand Down