sentinel_1_isce3_rtc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentinel-1 RTC with ISCE3\n",
    "\n",
    "**Overview**\n",
    "- Notebook for creating Radiometrically Terrain Corrected (RTC) data for Sentinel-1 using the ISCE3 software\n",
    "- Based on - https://github.com/ASFOpenSARlab/opensarlab-notebooks/blob/master/SAR_Training/English/Master/ISCE3_Sentinel1_RTC.ipynb\n",
    "- Designed to run on the NCI but should be suitable for use elsewhere\n",
    "- This notebook will search and download the following data:\n",
    "    - Sentinel-1 SLC \n",
    "    - Precise Orbit Files\n",
    "    - Copernicus 30m DEM covering region of Interest\n",
    "- A .yaml file will then be created for the ISCE3 software \n",
    "- RTC backscatter will be created with the ISCE3 software using the OPERA-adt/RTC project - https://github.com/opera-adt/RTC \n",
    "\n",
    "**Requirements**\n",
    "- A conda environment setup as descined in the *README.md*\n",
    "- Appropriate storage space\n",
    "- Credentials for usgs (https://urs.earthdata.nasa.gov/users/new) \n",
    "- Credentials for copernicus CDSE ( https://dataspace.copernicus.eu/)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initial setup\n",
    "import os\n",
    "import asf_search as asf\n",
    "from eof.download import download_eofs\n",
    "import yaml\n",
    "import boto3\n",
    "from botocore import UNSIGNED\n",
    "from botocore.config import Config\n",
    "import shutil\n",
    "from shapely import Polygon, box\n",
    "import numpy as np\n",
    "import pyproj\n",
    "import rasterio\n",
    "import rasterio.merge\n",
    "from rasterio.transform import from_origin\n",
    "from rasterio.warp import calculate_default_transform, reproject, Resampling\n",
    "from dem_stitcher.geoid import remove_geoid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# paths\n",
    "data_dir = '/home/547/ab7271/s1-rtc-isce3-notebook/data' # set this as the absolute path\n",
    "download_folder = os.path.join(data_dir,'scenes')\n",
    "earthdata_credentials_path = \"credentials/credentials_earthdata.yaml\"\n",
    "copernicus_credentials_path = \"credentials/credentials_copernicus.yaml\"\n",
    "precise_orb_download_folder = os.path.join(data_dir,'orbits','POEORB')\n",
    "restituted_orb_download_folder = os.path.join(data_dir,'orbits','RESORB')\n",
    "rtc_outpath = os.path.join(data_dir,'results')\n",
    "rtc_scratch_dir = os.path.join(data_dir,'scratch')\n",
    "dem_download_folder = os.path.join(data_dir,'dem')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_folders = True\n",
    "if create_folders:\n",
    "    for f in [\n",
    "        download_folder, \n",
    "        precise_orb_download_folder, \n",
    "        restituted_orb_download_folder,\n",
    "        dem_download_folder,\n",
    "        rtc_outpath,\n",
    "        rtc_scratch_dir\n",
    "        ]:\n",
    "        os.makedirs(f, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Credentials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(earthdata_credentials_path, \"r\", encoding='utf8') as f:\n",
    "        earthdata_credentials = yaml.safe_load(f.read())\n",
    "with open(copernicus_credentials_path, \"r\", encoding='utf8') as f:\n",
    "        copernicus_credentials = yaml.safe_load(f.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Search and Download Scene of Interest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 1 - Search based on a geometry and daterange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wkt = \"POINT (130.7085216 -16.0044816)\" # South Australia\n",
    "wkt = \"POINT (110.526792 -66.282343)\" # Casey station Antarctica\n",
    "print(f'Searching over point: {wkt}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# product type https://github.com/asfadmin/Discovery-asf_search/blob/master/asf_search/constants/PRODUCT_TYPE.py\n",
    "# prod = 'GRD_HS' # IW\n",
    "# prod = 'GRD_MD' # EW\n",
    "\n",
    "# prod = 'GRD_HD'\n",
    "# prod = 'GRD_MS'\n",
    "# prod = 'GRD_FD'\n",
    "prod = 'SLC'\n",
    "\n",
    "# acquisition mode\n",
    "mode = 'IW'\n",
    "#mode = 'EW'\n",
    "\n",
    "results = asf.search(platform=[asf.PLATFORM.SENTINEL1], \n",
    "                     intersectsWith=wkt, \n",
    "                     maxResults=10, \n",
    "                     processingLevel=prod,\n",
    "                     beamMode=mode,\n",
    "                     start='2021-12-31T11:59:59Z',\n",
    "                     end='2022-01-19T11:59:59Z')\n",
    "print(len(results))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 2 - Search for known products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_list = True\n",
    "if search_list:\n",
    "\n",
    "    prod = 'SLC'\n",
    "    mode = 'IW'\n",
    "    \n",
    "    scene_list = ['S1B_IW_SLC__1SSH_20190223T222639_20190223T222706_015079_01C2E9_1D63', \n",
    "   ]\n",
    "\n",
    "    # search the scene list with the specified product type\n",
    "    results = asf.granule_search(scene_list, asf.ASFSearchOptions(processingLevel=prod))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, r in enumerate(results):\n",
    "    print(f\"{i+1} - {r.properties['sceneName']}\")\n",
    "    print(r.properties)\n",
    "    print(r.geometry)\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Scenes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download\n",
    "session = asf.ASFSession()\n",
    "session.auth_with_creds(earthdata_credentials['login'],earthdata_credentials['password'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# limit to the first scene\n",
    "results = results[0:1]\n",
    "geometry = results[0].geometry\n",
    "scene = results[0].properties['sceneName']\n",
    "pol = results[0].properties['polarization']\n",
    "name = str(results[0].properties['sceneName'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Downloding {len(results)} scenes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download all results\n",
    "scene_paths = []\n",
    "scene_names = []\n",
    "for s in results:\n",
    "    name = s.properties['sceneName']\n",
    "    scene_names.append(name)\n",
    "    print(name)\n",
    "    path = os.path.join(download_folder, name)\n",
    "    s.download(path=download_folder, session=session)\n",
    "    scene_paths.append(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Precise Orbit Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for scene in  scene_paths:\n",
    "    # download precise orbits\n",
    "    download_eofs(\n",
    "        sentinel_file=scene, \n",
    "        save_dir=precise_orb_download_folder, \n",
    "        orbit_type='precise', \n",
    "        cdse_user=copernicus_credentials['login'], \n",
    "        cdse_password=copernicus_credentials['password'],\n",
    "        asf_user=earthdata_credentials['login'], \n",
    "        asf_password=earthdata_credentials['password']\n",
    "        )\n",
    "    # download restituted orbits\n",
    "    download_eofs(\n",
    "        sentinel_file=scene, \n",
    "        save_dir=restituted_orb_download_folder, \n",
    "        orbit_type='restituted', \n",
    "        cdse_user=copernicus_credentials['login'], \n",
    "        cdse_password=copernicus_credentials['password'],\n",
    "        asf_user=earthdata_credentials['login'], \n",
    "        asf_password=earthdata_credentials['password']\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('precise orbits')\n",
    "for p in os.listdir(precise_orb_download_folder):\n",
    "    prec_orb_path = os.path.join(precise_orb_download_folder,p)\n",
    "    print(prec_orb_path)\n",
    "\n",
    "print('restited orbits')\n",
    "for p in os.listdir(restituted_orb_download_folder):\n",
    "    res_orb_path = os.path.join(restituted_orb_download_folder,p)\n",
    "    print(res_orb_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download and process DEMs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_polygon(src_crs, dst_crs, geometry, always_xy=True):\n",
    "    src_crs = pyproj.CRS(f\"EPSG:{src_crs}\")\n",
    "    dst_crs = pyproj.CRS(f\"EPSG:{dst_crs}\") \n",
    "    transformer = pyproj.Transformer.from_crs(src_crs, dst_crs, always_xy=always_xy)\n",
    "     # Transform the polygon's coordinates\n",
    "    transformed_exterior = [\n",
    "        transformer.transform(x, y) for x, y in geometry.exterior.coords\n",
    "    ]\n",
    "    # Create a new Shapely polygon with the transformed coordinates\n",
    "    transformed_polygon = Polygon(transformed_exterior)\n",
    "    return transformed_polygon\n",
    "\n",
    "def adjust_scene_poly_at_extreme_lat(bbox, src_crs, ref_crs, delta=0.1):\n",
    "    \"\"\"\n",
    "    Adjust the bounding box around a scene in src_crs (4326) due to warping at high\n",
    "    Latitudes. For example, the min and max bounding values for an antarctic scene in\n",
    "    4326 may not actually be the true min and max due to distortions at high latitudes. \n",
    "\n",
    "    Parameters:\n",
    "    - bbox: Tuple of four coordinates (x_min, y_min, x_max, y_max).\n",
    "    - src_crs: Source EPSG. e.g. 4326\n",
    "    - ref_crs: reference crs to create the true bbox. i.e. 3031 in southern \n",
    "                hemisphere and 3995 in northern (polar stereographic)\n",
    "    - delta: distance between generation points along the bounding box sides in\n",
    "            src_crs. e.g. 0.1 degrees in lat/lon \n",
    "\n",
    "    Returns:\n",
    "    - A polygon bounding box expanded to the true min max\n",
    "    \"\"\"\n",
    "    x_min, y_min, x_max, y_max = bbox\n",
    "    # Generate points along the top side\n",
    "    top_side = [(x, y_max) for x in list(np.arange(x_min, x_max, delta)) + [x_max]]    \n",
    "    # Generate points along the right side\n",
    "    right_side = [(x_max, y) for y in list(np.arange(y_max - delta, y_min-delta, -delta)) + [y_min]]\n",
    "    # Generate points along the bottom side\n",
    "    bottom_side = [(x, y_min) for x in list(np.arange(x_max - delta, x_min-delta, -delta)) + [x_min]]\n",
    "    list(np.arange(y_min + delta, y_max, delta)) + [y_max]\n",
    "    # Generate points along the left side\n",
    "    left_side = [(x_min, y) for y in list(np.arange(y_min + delta, y_max, delta)) + [y_max]]\n",
    "    # Combine all sides' points\n",
    "    all_points = top_side + right_side + bottom_side + left_side\n",
    "    # convert to a polygon \n",
    "    polygon = Polygon(all_points)\n",
    "    # convert polygon to desired crs and get bounds in those coordinates\n",
    "    trans_bounds = transform_polygon(src_crs, ref_crs, polygon).bounds\n",
    "    trans_poly = Polygon(\n",
    "        [(trans_bounds[0], trans_bounds[1]), \n",
    "         (trans_bounds[2], trans_bounds[1]), \n",
    "         (trans_bounds[2], trans_bounds[3]), \n",
    "         (trans_bounds[0], trans_bounds[3])]\n",
    "        )\n",
    "    corrected_poly = transform_polygon(ref_crs, src_crs, trans_poly)\n",
    "    return corrected_poly\n",
    "\n",
    "def expand_raster_with_bounds(input_raster, output_raster, old_bounds, new_bounds, fill_value=None):\n",
    "    \"\"\"Expand the raster to the desired bounds. Resolution and Location are preserved.\n",
    "\n",
    "    Args:\n",
    "        input_raster (str): input raster path\n",
    "        output_raster (str): out raster path\n",
    "        old_bounds (tuple): current bounds\n",
    "        new_bounds (tuple): new bounds\n",
    "        fill_value (float, int, optional): Fill value to pad with. Defaults to None and nodata is used.\n",
    "    \"\"\"\n",
    "    # Open the raster dataset\n",
    "    with rasterio.open(input_raster, 'r') as src:\n",
    "        # get old bounds\n",
    "        old_left, old_bottom, old_right, old_top = old_bounds\n",
    "        # Define the new bounds\n",
    "        new_left, new_bottom, new_right, new_top = new_bounds\n",
    "        # adjust the new bounds with even pixel multiples of existing\n",
    "        # this will stop small offsets\n",
    "        print(f'Making new raster with target bounds: {new_bounds}')\n",
    "        new_left = old_left - int(abs(new_left-old_left)/src.res[0])*src.res[0]\n",
    "        new_right = old_right + int(abs(new_right-old_right)/src.res[0])*src.res[0]\n",
    "        new_bottom = old_bottom - int(abs(new_bottom-old_bottom)/src.res[1])*src.res[1]\n",
    "        new_top = old_top + int(abs(new_top-old_top)/src.res[1])*src.res[1]\n",
    "        print(f'New raster bounds: {(new_left, new_bottom, new_right, new_top)}')\n",
    "        # Calculate the new width and height, should be integer values\n",
    "        new_width = int((new_right - new_left) / src.res[0])\n",
    "        new_height = int((new_top - new_bottom) / src.res[1])\n",
    "        # Define the new transformation matrix\n",
    "        transform = from_origin(new_left, new_top, src.res[0], src.res[1])\n",
    "        # Create a new raster dataset with expanded bounds\n",
    "        profile = src.profile\n",
    "        profile.update({\n",
    "            'width': new_width,\n",
    "            'height': new_height,\n",
    "            'transform': transform\n",
    "        })\n",
    "        # make a temp file\n",
    "        tmp = output_raster.replace('.tif','_tmp.tif')\n",
    "        print(f'Making temp file: {tmp}')\n",
    "        with rasterio.open(tmp, 'w', **profile) as dst:\n",
    "            # Read the data from the source and write it to the destination\n",
    "            fill_value = profile['nodata'] if fill_value is None else fill_value\n",
    "            print(f'Padding new raster extent with value: {fill_value}')\n",
    "            data = np.full((new_height, new_width), fill_value=fill_value, dtype=profile['dtype'])\n",
    "            dst.write(data, 1)\n",
    "        # merge the old raster into the new raster with expanded bounds \n",
    "        print(f'Merging original raster and expanding bounds...')\n",
    "    del data\n",
    "    rasterio.merge.merge(\n",
    "        datasets=[tmp, input_raster],\n",
    "        method='max',\n",
    "        dst_path=output_raster)\n",
    "    os.remove(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "geometry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# buffer the sceme nounda to ensure coverage\n",
    "scene_bounds = Polygon(geometry['coordinates'][0]).bounds\n",
    "print(f'original scene bounds : {scene_bounds}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adjust DEM bounds at extreme latitudes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if we are at high latitudes we need to correct the bounds due to the skewed box shape\n",
    "if (scene_bounds[1] < -50) or (scene_bounds[3] < -50):\n",
    "    # Southern Hemisphere\n",
    "    print(f'Adjusting scene bounds due to warping at high latitude')\n",
    "    scene_poly = adjust_scene_poly_at_extreme_lat(scene_bounds, 4326, 3031)\n",
    "    scene_bounds = scene_poly.bounds \n",
    "    print(f'Adjusted scene bounds : {scene_bounds}')\n",
    "if (scene_bounds[1] > 50) or (scene_bounds[3] > 50):\n",
    "    # Northern Hemisphere\n",
    "    print(f'Adjusting scene bounds due to warping at high latitude')\n",
    "    scene_poly = adjust_scene_poly_at_extreme_lat(scene_bounds, 4326, 3995)\n",
    "    scene_bounds = scene_poly.bounds \n",
    "    print(f'Adjusted scene bounds : {scene_bounds}')\n",
    "print('adjusted scene bounds')\n",
    "scene_bounds\n",
    "# Create the polygon\n",
    "geom = Polygon([(scene_bounds[0], scene_bounds[1]), \n",
    "                (scene_bounds[2], scene_bounds[1]), \n",
    "                (scene_bounds[2], scene_bounds[3]), \n",
    "                (scene_bounds[0], scene_bounds[3])])\n",
    "geom = list(geom.exterior.coords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_lat, max_lat  = min([c[1] for c in geom]), max([c[1] for c in geom])\n",
    "min_lon, max_lon  = min([c[0] for c in geom]), max([c[0] for c in geom])\n",
    "lats = list(range(int(np.floor(min_lat)), int(np.ceil(max_lat+1))))\n",
    "longs = list(range(int(np.floor(min_lon)), int(np.ceil(max_lon+1))))\n",
    "print(geom)\n",
    "print(lats)\n",
    "print(longs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find and download Copernicus 30m DEM from AWS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dem_s3_paths = []\n",
    "\n",
    "for lat in lats:\n",
    "    for long in longs:\n",
    "        if lat >= 0:\n",
    "            lat_dir = \"N\"\n",
    "        else:\n",
    "            lat_dir = \"S\"\n",
    "        if long >= 0:\n",
    "            long_dir = \"E\"\n",
    "        else:\n",
    "            long_dir = \"W\"\n",
    "        \n",
    "        dem_s3_paths.append(f\"Copernicus_DSM_COG_10_{lat_dir}{int(abs(lat)):02d}_00_{long_dir}{int(abs(long)):03d}_00_DEM/Copernicus_DSM_COG_10_{lat_dir}{int(abs(lat)):02d}_00_{long_dir}{int(abs(long)):03d}_00_DEM.tif\")\n",
    "dem_s3_paths = list(set(dem_s3_paths))\n",
    "dem_s3_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3 = boto3.resource('s3', config=Config(signature_version=UNSIGNED))\n",
    "bucket_name = \"copernicus-dem-30m\"\n",
    "bucket = s3.Bucket(bucket_name)\n",
    "dems_to_merge = []\n",
    "\n",
    "for s3_path in dem_s3_paths:\n",
    "    try:\n",
    "        dl_path = os.path.join(dem_download_folder,s3_path.split('/')[1])\n",
    "        if not os.path.exists(dl_path):\n",
    "            bucket.download_file(s3_path, dl_path)\n",
    "            print(f'downloaded : {s3_path}')\n",
    "            dems_to_merge.append(dl_path)\n",
    "        else:\n",
    "            print(f'file exists : {s3_path}')\n",
    "            dems_to_merge.append(dl_path)\n",
    "    except:\n",
    "        print(f'not found : {s3_path}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dem_paths = ' '.join(dems_to_merge)\n",
    "dem_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge the DEM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_dem_path = os.path.join(dem_download_folder,f\"{name}_dem.tif\")\n",
    "!gdal_merge.py -n 0 -o $merged_dem_path $dem_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensure the DEM covers the scene ROI for ISCE3\n",
    "- In areas such as Antarctica, there may be missing DEM data at the coast line that is required for the scene\n",
    "- ISCE requires full coverage of the scene, zero padded where data is missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with rasterio.open(merged_dem_path) as src:\n",
    "    dem_meta = src.meta.copy()\n",
    "\n",
    "# get the bounds of the downloaded DEM\n",
    "# the full area requested may not be covered\n",
    "dem_bounds = rasterio.transform.array_bounds(\n",
    "    dem_meta['height'], \n",
    "    dem_meta['width'], \n",
    "    dem_meta['transform'])\n",
    "print(f'Downloaded DEM bounds: {dem_bounds}')\n",
    "# the DEM not covering the full extent of the scene is an issue\n",
    "if not box(*dem_bounds).contains_properly(box(*scene_bounds)):\n",
    "    print(f'Downloaded DEM does not cover scene bounds : {scene_bounds}')\n",
    "    print('Expanding the bounds of the downloaded DEM')\n",
    "    adjusted_merged_dem_path = merged_dem_path.replace('.tif','_adj.tif') #adjusted DEM path\n",
    "    expand_raster_with_bounds(merged_dem_path, adjusted_merged_dem_path, dem_bounds, scene_bounds, fill_value=0)\n",
    "    print(f'Replacing DEM: {merged_dem_path}')\n",
    "    os.remove(merged_dem_path)\n",
    "    os.rename(adjusted_merged_dem_path, merged_dem_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove the GEOID\n",
    "- ISCE3 requires the DEM to be referenced to the WGS84 Ellipsoid\n",
    "- To do this we must remove the egm_08 geoid heights from the DEM using the dem stitcher software"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "REFERENCE_ELIPSOID = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if REFERENCE_ELIPSOID == True:\n",
    "    # NOTE - potential memory issues killing process on the NCI\n",
    "    with rasterio.open(merged_dem_path) as src:\n",
    "        dem_arr = src.read(1)\n",
    "        dem_profile = src.profile\n",
    "\n",
    "    print('Removing geoid, referencing DEM to WGS84 ellipsoid heights')\n",
    "    geoid_name = 'egm_08'\n",
    "    dst_area_or_point = 'Point'\n",
    "    dem_arr = remove_geoid(\n",
    "                dem_arr,\n",
    "                dem_profile,\n",
    "                geoid_name,\n",
    "                dem_area_or_point=dst_area_or_point,\n",
    "            )\n",
    "    \n",
    "    ellisoid_merged_dem_path = os.path.join(dem_download_folder,f\"{name}_dem_ellipsoid.tif\")\n",
    "    print(f'saving ellipsoid referenced dem to : {ellisoid_merged_dem_path}')\n",
    "    with rasterio.open(ellisoid_merged_dem_path, 'w', **dem_profile) as ds:\n",
    "        ds.write(dem_arr)\n",
    "        ds.update_tags(AREA_OR_POINT='Point')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set the final dem path\n",
    "if REFERENCE_ELIPSOID == True:\n",
    "    dem_path = ellisoid_merged_dem_path\n",
    "else:\n",
    "    dem_path = merged_dem_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Produce ISCE3 RTC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Copy and edit a template rtc_s1.yaml \n",
    "\n",
    "This is the config file we will use to define parameters for RTC processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rtc_config_template_path = \"RTC/src/rtc/defaults/rtc_s1.yaml\"\n",
    "rtc_config_path = os.path.join(data_dir,\"rtc_s1.yaml\")\n",
    "shutil.copyfile(rtc_config_template_path, rtc_config_path)\n",
    "with open(rtc_config_path, 'r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "cfg = config['runconfig']['groups']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Change some parameters of the config to point to our files\n",
    "\n",
    "- The run is based off of this .yaml file, so there may be additonal changes you want to make\n",
    "\n",
    "- For example, additional metadata, projections and output resolutions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cfg['input_file_group']['safe_file_path'] = [os.path.join(data_dir,f\"{scene}.zip\")]\n",
    "cfg['input_file_group']['orbit_file_path'] = [str(prec_orb_path)]\n",
    "cfg['dynamic_ancillary_file_group']['dem_file'] = str(dem_path)\n",
    "cfg['product_group']['output_dir'] = rtc_outpath\n",
    "cfg['product_group']['save_mosaics'] = True\n",
    "cfg['product_group']['product_path'] = '.'\n",
    "cfg['product_group']['scratch_path'] = rtc_scratch_dir\n",
    "cfg['product_group']['product_id'] = f'OPERA_L2_RTC-S1_{name}'\n",
    "cfg['processing']['polarization'] = 'dual-pol' if len(pol) > 2 else 'co-pol' # string for template value\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(rtc_config_path, 'w') as f:\n",
    "    yaml.dump(config, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run rtc_s1_single_job.py\n",
    "- Open rtc_s1.yaml and adjust any parameters you like\n",
    "- Note you maye need to change the envrionment path below based on conda install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! ~/miniconda3/envs/isce3_rtc/bin/rtc_s1_single_job.py $rtc_config_path"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pyroSAR",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}