Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move LAMP code to mbta-performance repo, add v3 alert saving #97

Merged
merged 17 commits into from
Apr 13, 2024
Merged

Move LAMP code to mbta-performance repo, add v3 alert saving #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
6aec26f
Move LAMP code to mbta-performance repo
devinmatte Apr 8, 2024
42221b9
Adding TODO comments
devinmatte Apr 8, 2024
d575265
Start storing alerts every 15 minutes during the service day
devinmatte Apr 9, 2024
2dc084c
Adding deprecation warnings
devinmatte Apr 9, 2024
e5bcfaa
Add service date tests
devinmatte Apr 9, 2024
e236c0f
Renaming test file
devinmatte Apr 9, 2024
bf7f7ac
Fix permissions
devinmatte Apr 9, 2024
d94e6c9
Improve alert job details
devinmatte Apr 9, 2024
f0e352b
Fixing alerts runs
devinmatte Apr 9, 2024
56e9175
Bump version
devinmatte Apr 9, 2024
77d7b05
Adding job descriptions to help people understand jobs
devinmatte Apr 10, 2024
5a149af
We need secretsmanager
devinmatte Apr 10, 2024
987db0e
Adding datadog layer
devinmatte Apr 10, 2024
08cfffe
Remove Datadog layer
devinmatte Apr 10, 2024
9e2cb1a
Increase memory limits and improve permissions
devinmatte Apr 10, 2024
00007aa
Updating CODEOWNERS
devinmatte Apr 10, 2024
746c363
Small package bumps
devinmatte Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/CODEOWNERS
View file
Open in desktop
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
* @austinjpaul @devinmatte
ingestor/chalicelib/lamp @devinmatte @hamima-halim
.github @devinmatte @nathan-weinberg
devops @devinmatte @nathan-weinberg
2 changes: 1 addition & 1 deletion devops/deploy.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ GIT_SHA=`git rev-parse HEAD`
echo "Deploying version $GIT_VERSION | $GIT_SHA"

# Adding some datadog tags to get better data
DD_TAGS="git.commit.sha:$GIT_SHA,git.repository_url:github.com/transitmatters/data-ingestion"
DD_TAGS="git.commit.sha:$GIT_SHA,git.repository_url:github.com/transitmatters/data-ingestion,version:$GIT_VERSION"
DD_GIT_REPOSITORY_URL="github.com/transitmatters/data-ingestion"
DD_GIT_COMMIT_SHA="$GIT_SHA"

Expand Down
2 changes: 0 additions & 2 deletions devops/helpers.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ function shrink {
zip -d -qq cfn/layer-deployment.zip '**/NOTICE*'
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/numpy*/tests/**/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/pandas*/tests/**/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/pyarrow*/tests/**/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/examples/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/a*/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/b*/*
Expand All @@ -35,7 +34,6 @@ function shrink {
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/q*/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/r*/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/sa*/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/se*/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/ss*/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/t*/*
zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/u*/*
Expand Down
19 changes: 12 additions & 7 deletions ingestor/.chalice/config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,14 @@
},
"lambda_functions": {
"store_yesterday_alerts": {
"iam_policy_file": "policy-alerts.json"
"iam_policy_file": "policy-alerts.json",
"lambda_memory_size": 256,
"lambda_timeout": 150
},
"store_current_alerts": {
"iam_policy_file": "policy-alerts.json",
"lambda_memory_size": 256,
"lambda_timeout": 90
},
"store_new_train_runs": {
"iam_policy_file": "policy-newtrains.json"
Expand Down Expand Up @@ -56,14 +63,17 @@
},
"update_delivered_trip_metrics": {
"iam_policy_file": "policy-delivered-trip-metrics-daily.json",
"lambda_memory_size": 192,
"lambda_timeout": 150
},
"update_delivered_trip_metrics_yesterday": {
"iam_policy_file": "policy-delivered-trip-metrics-daily.json",
"lambda_memory_size": 192,
"lambda_timeout": 150
},
"update_agg_trip_metrics": {
"iam_policy_file": "policy-agg-trip-metric-tables.json"
"iam_policy_file": "policy-agg-trip-metric-tables.json",
"lambda_memory_size": 192
},
"populate_agg_delivered_trip_metrics": {
"iam_policy_file": "policy-agg-trip-metric-tables.json"
Expand Down Expand Up @@ -106,11 +116,6 @@
"lambda_memory_size": 1024,
"lambda_timeout": 60,
"max_ebs_size_gb": 2
},
"process_daily_lamp": {
"iam_policy_file": "policy-lamp-ingest.json",
"lambda_timeout": 900,
"lambda_memory_size": 2048
}
}
}
Expand Down
13 changes: 9 additions & 4 deletions ingestor/.chalice/policy-alerts.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,17 @@
"Resource": "arn:*:logs:*:*:*"
},
{
"Action": [
"s3:PutObject"
],
"Action": "s3:ListBucket",
"Effect": "Allow",
"Resource": ["arn:aws:s3:::tm-mbta-performance"]
},
{
"Action": "s3:*",
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::tm-mbta-performance/Alerts/*"
"arn:aws:s3:::tm-mbta-performance/Alerts",
"arn:aws:s3:::tm-mbta-performance/Alerts/*",
"arn:aws:s3:::tm-mbta-performance/Alerts/v3/*"
]
}
]
Expand Down
19 changes: 0 additions & 19 deletions ingestor/.chalice/policy-lamp-ingest.json
View file
Open in desktop

This file was deleted.

99 changes: 82 additions & 17 deletions ingestor/.chalice/resources.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@
"GitVersion": {
"Type": "String",
"Description": "Current Git Id"
},
"DDGitRepositoryUrl": {
"Type": "String",
"Default": "github.com/transitmatters/data-ingestion",
"Description": "Git repository URL"
}
},
"Resources": {
Expand All @@ -35,6 +40,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -53,6 +61,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -61,6 +72,7 @@
"StoreYesterdayAlerts": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Description": "Stores all v2 API alerts from yesterday",
"Environment": {
"Variables": {
"MBTA_V2_API_KEY": {
Expand All @@ -74,6 +86,31 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
}
},
"StoreCurrentAlerts": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Description": "Stores all v3 current API alerts",
"Environment": {
"Variables": {
"DD_API_KEY": {
"Ref": "DDApiKey"
},
"DD_VERSION": {
"Ref": "GitVersion"
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you explain what this is for? To help us find stuff in DD?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will allow datadog to link stack traces to github lines of code, making it possible to click directly from a stack trace to the related lines of code and the commit it was introduced in

"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -82,6 +119,7 @@
"StoreNewTrainRuns": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Description": "Stores stats about new train runs",
"Environment": {
"Variables": {
"MBTA_V2_API_KEY": {
Expand All @@ -95,6 +133,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -103,6 +144,7 @@
"BbStoreStationStatus": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Description": "Stores status of all blue bike stations",
"Environment": {
"Variables": {
"DD_LAMBDA_HANDLER": "app.bb_store_station_status",
Expand All @@ -114,6 +156,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -132,6 +177,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -150,6 +198,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -169,6 +220,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -187,6 +241,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -205,6 +262,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -223,6 +283,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -231,6 +294,7 @@
"UpdateRidership": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Description": "Ingests and saves MBTA ridership numbers from the Box link",
"Environment": {
"Variables": {
"DD_API_KEY": {
Expand All @@ -241,6 +305,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -249,6 +316,7 @@
"UpdateSpeedRestrictions": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Description": "Ingests and saves MBTA official speed restrictions",
"Environment": {
"Variables": {
"DD_API_KEY": {
Expand All @@ -259,6 +327,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -267,6 +338,7 @@
"UpdateTimePredictions": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Description": "Ingests and saves time predictions from the MBTA",
"Environment": {
"Variables": {
"DD_API_KEY": {
Expand All @@ -277,6 +349,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -285,6 +360,7 @@
"StoreLandingData": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Description": "Caches landing page data to improve homepage loading times",
"Environment": {
"Variables": {
"DD_API_KEY": {
Expand All @@ -295,6 +371,9 @@
},
"DD_TAGS": {
"Ref": "DDTags"
},
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand All @@ -303,6 +382,7 @@
"UpdateYankeeShuttles": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Description": "Updates and stores Yankee shuttle positions",
"Environment": {
"Variables": {
"DD_API_KEY": {
Expand All @@ -316,24 +396,9 @@
},
"YANKEE_API_KEY": {
"Ref": "YankeeApiKey"
}
}
}
}
},
"ProcessDailyLamp": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Environment": {
"Variables": {
"DD_API_KEY": {
"Ref": "DDApiKey"
},
"DD_VERSION": {
"Ref": "GitVersion"
},
"DD_TAGS": {
"Ref": "DDTags"
"DD_GIT_REPOSITORY_URL": {
"Ref": "DDGitRepositoryUrl"
}
}
}
Expand Down
Loading