create gh actions using reusable actions

.github/get_latest_index.py

@@ -0,0 +1,95 @@
+import os
+import re
+import requests
+import sys
+import uuid
+from google.cloud import bigquery
+
+class IDCIndexManager:
+    def __init__(self, project_id):
+        print("Initializing IDCIndexManager...")
+        self.project_id = project_id
+        self.client = bigquery.Client(project=project_id)
+
+    def get_latest_idc_release_version(self, view_id):
+        print("Getting latest IDC release version...")
+        view = self.client.get_table(view_id)
+        latest_idc_release_version=int(re.search(r"idc_v(\d+)", view.view_query).group(1))
+        return latest_idc_release_version
+
+    def extract_index_version(self, file_path):
+        print(f"Extracting index version from {file_path}...")
+        with open(file_path, "r") as file:
+            for line in file:
+                if "def get_idc_version(self):" in line:
+                    return int(re.findall(r"v(\d+)", next(file))[0])
+
+    def update_index_version(self, file_path, latest_idc_release_version):
+        print(f"Updating index version in {file_path}...")
+        with open(file_path, "r") as file:
+            lines = file.readlines()
+        with open(file_path, "w") as file:
+            for i in range(len(lines)):
+                if "def get_idc_version(self):" in lines[i]:
+                    lines[i + 1] = re.sub(
+                        r"v(\d+)", f"v{latest_idc_release_version}", lines[i + 1]
+                    )
+                file.write(lines[i])
+
+    def update_sql_queries_folder(
+        self, dir, current_index_version, latest_idc_release_version
+    ):
+        print(f"Updating SQL queries from {dir}...")
+        for file_name in os.listdir(dir):
+            if file_name.endswith(".sql"):
+                file_path = os.path.join(dir, file_name)
+                with open(file_path, "r") as file:
+                    sql_query = file.read()
+                modified_sql_query = sql_query.replace(
+                    f"idc_v{current_index_version}", f"idc_v{latest_idc_release_version}"
+                )
+                with open(file_path, "w") as file:
+                    file.write(modified_sql_query)
+                return modified_sql_query
+
+    def execute_sql_query(self, file_path):
+        print(f"Executing SQL query from {file_path}...")
+        with open(file_path, "r") as file:
+            sql_query = file.read()
+        df = self.client.query(sql_query).to_dataframe()
+        csv_file_name = f"{os.path.basename(file_path).split('.')[0]}.csv.zip"
+        return df, csv_file_name
+
+    def create_csv_zip_from_df(self, df, csv_file_name):
+        print(f"Creating CSV zip file {csv_file_name}...")
+        df.to_csv(csv_file_name, compression={'method': 'zip'}, escapechar="\\")
+
+    def run_queries_folder(self, dir):
+        print(f"Running queries from {dir}...")
+        for file_name in os.listdir(dir):
+            if file_name.endswith(".sql"):
+                file_path = os.path.join(dir, file_name)
+                df, csv_file_name = self.execute_sql_query(file_path)
+                self.create_csv_zip_from_df(df, csv_file_name)
+
+    def set_multiline_output(self, name, value):
+        print(f"Setting multiline output {name}...")
+        with open(os.environ["GITHUB_OUTPUT"], "a") as fh:
+            delimiter = uuid.uuid1()
+            print(f"{name}<<{delimiter}", file=fh)
+            print(value, file=fh)
+            print(delimiter, file=fh)
+
+    def run(self):
+        print("Running IDCIndexManager...")
+        latest_idc_release_version = self.get_latest_idc_release_version("bigquery-public-data.idc_current.dicom_all_view")
+        print(f"Latest IDC release version: {latest_idc_release_version}")
+        current_index_version = self.extract_index_version("idc_index/index.py")
+        print(f"Current index version: {current_index_version}")
+        self.set_multiline_output("current_index_version", int(current_index_version))
+        self.set_multiline_output("latest_idc_release_version", int(latest_idc_release_version))
+
+
+if __name__ == "__main__":
+    manager = IDCIndexManager("gcp-project-id")
+    manager.run()

.github/workflows/get-latest-index.yml

@@ -0,0 +1,92 @@
+name: idc-index release manager
+on:
+  workflow_dispatch: 
+  schedule:
+    - cron: 0 12 */1 * *
+
+jobs:
+  update_idc_index:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.11
+
+    - name: Install dependencies
+      run: pip install requests==2.31.0 pandas==2.1.1 google-cloud-bigquery==3.12.0 pyarrow==13.0.0 db-dtypes==1.1.1 PyGithub==2.1.1
+
+    - name: Authorize Google Cloud
+      uses: google-github-actions/auth@v1
+      with:
+        credentials_json: ${{ secrets.SERVICE_ACCOUNT_KEY }}
+        create_credentials_file: true
+        export_environment_variables: true 
+
+    - name: Run script to get the latest idc index
+      id: initialize_idc_manager_class
+      shell: python
+      run: |
+        import sys
+        import os
+        sys.path.append(".github")
+
+        from get_latest_index import IDCIndexManager
+
+        project_id = os.environ['GCP_PROJECT_ID']
+        manager = IDCIndexManager(project_id)
+
+        current_index_version = manager.extract_index_version("idc_index/index.py")
+        latest_idc_release_version = manager.get_latest_idc_release_version("bigquery-public-data.idc_current.dicom_all_view")
+
+        if current_index_version < latest_idc_release_version:
+            manager.update_index_version("idc_index/index.py", latest_idc_release_version)
+            manager.update_sql_queries_folder("queries/", current_index_version, latest_idc_release_version)
+            manager.run_queries_folder("queries/")
+
+        manager.set_multiline_output("current_index_version", int(current_index_version))
+        manager.set_multiline_output("latest_idc_release_version", int(latest_idc_release_version))
+      env: 
+        GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} 
+
+    - name: Update latest tag's source code
+      if: ${{ steps.initialize_idc_manager_class.outputs.current_index_version  != steps.initialize_idc_manager_class.outputs.latest_idc_release_version }}
+      uses: richardsimko/update-tag@v1
+      with:
+        tag_name: latest
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Create latest Release
+      if:  ${{ steps.initialize_idc_manager_class.outputs.current_index_version  != steps.initialize_idc_manager_class.outputs.latest_idc_release_version }}
+      uses: crowbarmaster/GH-Automatic-Releases@latest
+      with:
+        repo_token: "${{ secrets.GITHUB_TOKEN }}"
+        automatic_release_tag: "latest"
+        #generate_notes: false
+        body: "Latest idc-index"
+        prerelease: true
+        title: "Latest idc-index"
+        files: |
+            *.zip
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v5
+      with:
+        title: Update to v${{ steps.initialize_idc_manager_class.outputs.latest_idc_release_version }}
+        body:  Update sql queries and/or index.py to v${{ steps.initialize_idc_manager_class.outputs.latest_idc_release_version }}
+        base:  main
+        branch: update-sql-queries-and-or-index
+        add-paths: |
+          queries/*.sql
+          idc_index/index.py
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+
+
+