diff --git a/devops/deploy.sh b/devops/deploy.sh index 0742b67..9283ae0 100755 --- a/devops/deploy.sh +++ b/devops/deploy.sh @@ -14,7 +14,7 @@ GIT_SHA=`git rev-parse HEAD` echo "Deploying version $GIT_VERSION | $GIT_SHA" # Adding some datadog tags to get better data -DD_TAGS="git.commit.sha:$GIT_SHA,git.repository_url:github.com/transitmatters/data-ingestion" +DD_TAGS="git.commit.sha:$GIT_SHA,git.repository_url:github.com/transitmatters/data-ingestion,version:$GIT_VERSION" DD_GIT_REPOSITORY_URL="github.com/transitmatters/data-ingestion" DD_GIT_COMMIT_SHA="$GIT_SHA" diff --git a/ingestor/.chalice/policy-alerts.json b/ingestor/.chalice/policy-alerts.json index e32e98a..3859344 100644 --- a/ingestor/.chalice/policy-alerts.json +++ b/ingestor/.chalice/policy-alerts.json @@ -13,7 +13,10 @@ { "Action": ["s3:GetObject", "s3:PutObject"], "Effect": "Allow", - "Resource": ["arn:aws:s3:::tm-mbta-performance/Alerts/*"] + "Resource": [ + "arn:aws:s3:::tm-mbta-performance/Alerts/*", + "arn:aws:s3:::tm-mbta-performance/Alerts/v3/*" + ] } ] } diff --git a/ingestor/.chalice/resources.json b/ingestor/.chalice/resources.json index 7fc844c..43312b4 100644 --- a/ingestor/.chalice/resources.json +++ b/ingestor/.chalice/resources.json @@ -19,6 +19,11 @@ "GitVersion": { "Type": "String", "Description": "Current Git Id" + }, + "DDGitRepositoryUrl": { + "Type": "String", + "Default": "github.com/transitmatters/data-ingestion", + "Description": "Git repository URL" } }, "Resources": { @@ -35,6 +40,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -53,6 +61,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -74,6 +85,30 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" + } + } + } + } + }, + "StoreCurrentAlerts": { + "Type": "AWS::Serverless::Function", + "Properties": { + "Environment": { + "Variables": { + "DD_API_KEY": { + "Ref": "DDApiKey" + }, + "DD_VERSION": { + "Ref": "GitVersion" + }, + "DD_TAGS": { + "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -95,6 +130,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -114,6 +152,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -132,6 +173,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -150,6 +194,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -169,6 +216,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -187,6 +237,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -205,6 +258,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -223,6 +279,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -241,6 +300,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -259,6 +321,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -277,6 +342,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -295,6 +363,9 @@ }, "DD_TAGS": { "Ref": "DDTags" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } @@ -316,6 +387,9 @@ }, "YANKEE_API_KEY": { "Ref": "YankeeApiKey" + }, + "DD_GIT_REPOSITORY_URL": { + "Ref": "DDGitRepositoryUrl" } } } diff --git a/ingestor/chalicelib/alerts.py b/ingestor/chalicelib/alerts.py index c7b532e..5880ad2 100644 --- a/ingestor/chalicelib/alerts.py +++ b/ingestor/chalicelib/alerts.py @@ -30,15 +30,14 @@ def save_v3_alerts(): service_date = get_current_service_date() try: - all_alerts = s3.download(BUCKET, key(service_date), encoding="utf8", compressed=True) + all_alerts = s3.download(BUCKET, key(service_date, v3=True), encoding="utf8", compressed=True) except ClientError as ex: if ex.response["Error"]["Code"] != "NoSuchKey": raise all_alerts = dict() - all_alerts = dict() for alert in alerts["data"]: all_alerts[alert["id"]] = alert alert_json = json.dumps(all_alerts).encode("utf8") - s3.upload(BUCKET, key(service_date), alert_json, compress=True) + s3.upload(BUCKET, key(service_date, v3=True), alert_json, compress=True)