Merge pull request #43 from asfadmin/test

Release v0.2.0

.github/dependabot.yml

@@ -9,6 +9,8 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    ignore:
+      - dependency-name: "elasticsearch"
     labels:
       - "bumpless"
   - package-ecosystem: "github-actions"

.github/workflows/deploy.yml

@@ -6,6 +6,8 @@ on:
       - prod
       - test
 
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -41,10 +43,7 @@ jobs:
       - name: install dependencies
         if: github.ref == matrix.deploy_ref
         shell: bash
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install -r log-parse/requirements.txt -t log-parse/src/
-          python -m pip install -r ems-report/requirements.txt -t ems-report/src/
+        run: make install-lambda-deps
       - name: package and deploy
         if: github.ref == matrix.deploy_ref
         shell: bash

.github/workflows/test.yml

@@ -0,0 +1,16 @@
+name: Test with pytest
+
+on: push
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: mamba-org/provision-with-micromamba@v16
+
+      - shell: bash -l {0}
+        run: |
+          make pytest

CHANGELOG.md

@@ -6,6 +6,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.0]
+
+### Changed
+- For products with version >= `v3`:
+  - Browse image downloads are reported under the `ARIA_S1_GUNW_BROWSE` collection.
+  - All other product downloads are reported under the `ARIA_S1_GUNW` collection.
+- For older products:
+  - Browse image downloads are reported under the `SENTINEL-1_INTERFEROGRAMS_BROWSE` collection.
+  - All other product downloads are reported under the `SENTINEL-1_INTERFEROGRAMS` collection.
+- Upgraded lambda functions to Python 3.12 from Python 3.8.
+- Upgraded Elasticsearch to 7.10 from 7.4.
+- Upgraded instance type for Elasticsearch domain to `t3.small.elasticsearch` from `t2.small.elasticsearch`.
+
+### Removed
+- Downloads of `.unw_geo.zip` products are no longer reported under the `SENTINEL-1_INSAR_UNWRAPPED_INTERFEROGRAM_AND_COHERENCE_MAP` collection. This collection was removed within the last year.
+
 ## [0.1.0]
 
 This is the initial release after replacing the AWS CodePipeline workflow with a GitHub Actions deployment workflow.

Makefile

@@ -0,0 +1,16 @@
+EMS_REPORT = ${PWD}/ems-report/src/
+LOG_PARSE = ${PWD}/log-parse/src/
+export PYTHONPATH = ${EMS_REPORT}:${LOG_PARSE}
+
+install:
+	python -m pip install --upgrade pip && \
+	python -m pip install -r requirements-all.txt
+
+install-lambda-deps:
+	python -m pip install --upgrade pip && \
+	python -m pip install -r requirements-ems-report.txt -t ems-report/src/ && \
+	python -m pip install -r requirements-log-parse.txt -t log-parse/src/
+
+test_file ?= tests/
+pytest:
+	pytest $(test_file)

cloudformation.yaml

@@ -27,9 +27,11 @@ Resources:
 
   Domain:
     Type: AWS::Elasticsearch::Domain
+    UpdatePolicy:
+      EnableVersionUpgrade: true
     Properties:
       DomainName: !Ref AWS::StackName
-      ElasticsearchVersion: 7.4
+      ElasticsearchVersion: "7.10"
       AccessPolicies: !Sub |-
         {
           "Version": "2012-10-17",
@@ -58,7 +60,7 @@ Resources:
       ElasticsearchClusterConfig:
         DedicatedMasterEnabled: false
         InstanceCount: 1
-        InstanceType: t2.small.elasticsearch
+        InstanceType: t3.small.elasticsearch
         ZoneAwarenessEnabled: false
       EBSOptions:
         EBSEnabled: true

ems-report/cloudformation.yaml

@@ -63,11 +63,11 @@ Resources:
             Resource: !Sub "arn:aws:s3:::${OutputBucket}/${OutputPrefix}*"
           - Effect: Allow
             Action:
-            - es:ESHttpGet
+            - es:ESHttpPost
             Resource: !Sub "${DomainArn}/${IndexName}/_search*"
           - Effect: Allow
             Action:
-            - es:ESHttpGet
+            - es:ESHttpPost
             - es:ESHttpDelete
             Resource: !Sub "${DomainArn}/_search/scroll*"
 
@@ -91,10 +91,10 @@ Resources:
                 "prefix": "${OutputPrefix}"
               }
             }
-      Handler: main.lambda_handler
+      Handler: ems_report.lambda_handler
       MemorySize: 1024
       Role: !GetAtt Role.Arn
-      Runtime: python3.8
+      Runtime: python3.12
       Timeout: 900
 
   Schedule:

ems-report/src/main.py → ems-report/src/ems_report.py

@@ -35,14 +35,16 @@ def get_elasticsearch_connection(host):
     return es
 
 
-def get_category(file_name):
+def get_category(file_name: str) -> str:
+    version = file_name.split('-')[-1].split('.')[0]
+    if version < 'v3':
+        if file_name.endswith('.png'):
+            return 'SENTINEL-1_INTERFEROGRAMS_BROWSE'
+        return 'SENTINEL-1_INTERFEROGRAMS'
+
     if file_name.endswith('.png'):
-        category = 'SENTINEL-1_INTERFEROGRAMS_BROWSE'
-    elif file_name.endswith('.unw_geo.zip'):
-        category = 'SENTINEL-1_INSAR_UNWRAPPED_INTERFEROGRAM_AND_COHERENCE_MAP'
-    else:
-        category = 'SENTINEL-1_INTERFEROGRAMS'
-    return category
+        return 'ARIA_S1_GUNW_BROWSE'
+    return 'ARIA_S1_GUNW'
 
 
 def get_records(report_date, config):

environment.yml

@@ -0,0 +1,9 @@
+name: grfn-logging
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - python=3.12
+  - pip
+  - pip:
+    - -r requirements-all.txt

log-parse/cloudformation.yaml

@@ -86,10 +86,10 @@ Resources:
               "host": "${DomainEndpoint}",
               "index": "${IndexName}"
             }
-      Handler: main.lambda_handler
+      Handler: log_parse.lambda_handler
       MemorySize: 128
       Role: !GetAtt Role.Arn
-      Runtime: python3.8
+      Runtime: python3.12
       Timeout: 300
 
   EventPermission:

requirements-all.txt

@@ -0,0 +1,3 @@
+-r requirements-ems-report.txt
+-r requirements-log-parse.txt
+pytest==8.1.1

requirements-ems-report.txt

@@ -0,0 +1,3 @@
+aws-requests-auth==0.4.3
+boto3==1.34.80
+elasticsearch==7.10.1

requirements-log-parse.txt

@@ -0,0 +1,3 @@
+aws-requests-auth==0.4.3
+boto3==1.34.80
+elasticsearch==7.10.1

tests/test_ems_report.py

@@ -0,0 +1,20 @@
+import ems_report
+
+
+def test_get_category():
+    assert ems_report.get_category('foo-v1.png') \
+           == 'SENTINEL-1_INTERFEROGRAMS_BROWSE'
+    assert ems_report.get_category('foo-v1') \
+           == 'SENTINEL-1_INTERFEROGRAMS'
+    assert ems_report.get_category('S1-GUNW-A-R-018-tops-20230528_20230504-232801-00072W_00037S-PP-efd5-v2_0_6.png') \
+           == 'SENTINEL-1_INTERFEROGRAMS_BROWSE'
+    assert ems_report.get_category('S1-GUNW-A-R-018-tops-20230528_20230504-232801-00072W_00037S-PP-efd5-v2_0_6') \
+           == 'SENTINEL-1_INTERFEROGRAMS'
+    assert ems_report.get_category('S1-GUNW-A-R-091-tops-20240316_20240304-233600-00074W_00037S-PP-4b33-v3_0_1.png') \
+           == 'ARIA_S1_GUNW_BROWSE'
+    assert ems_report.get_category('S1-GUNW-A-R-091-tops-20240316_20240304-233600-00074W_00037S-PP-4b33-v3_0_1') \
+           == 'ARIA_S1_GUNW'
+    assert ems_report.get_category('foo-v4.png') \
+           == 'ARIA_S1_GUNW_BROWSE'
+    assert ems_report.get_category('foo-v4') \
+           == 'ARIA_S1_GUNW'