lidar_SCB_copy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "797f557e-435f-49cd-8002-2135758b8c67",
   "metadata": {},
   "source": [
    "# Lidar Processing Workflow\n",
    "**Cara Piske, Graduate Program of Hydrologic Sciences, 2022; Advisor: Dr. Adrian Harpold**<br>\n",
    "This code processes raw lidar point clouds in order to calculate snow depth using PDAL. <br>\n",
    "Lidar data were provided by the Airborne Snow Observatory (ASO), the National Center for Airborne Laser Mapping (NCALM), and Watershed Sciences Inc. (WSI). <br>\n",
    "\n",
    "The goal of this project is to process snow depth to the one-meter spatial scale while maintaining conservative under-canopy estimates. Therefore, little interpolation occurs under-canopy. We follow these protocols in order to obtain a 1-m rasterized product (as opposed to the 3-m rasterized product provided by ASO on the NSIDC data portal). NCALM and WSI flights were obtained through OpenTopography."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01e7346b-ed91-48fe-912a-4dc9d7b20be4",
   "metadata": {},
   "source": [
    "Start by importing necessary files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ef19ecac-1150-469a-9a3f-3eb6f8b1c531",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import necessary files\n",
    "import numpy as np \n",
    "import matplotlib.pyplot as plt\n",
    "import json # where we will save the json files to run a pipeline\n",
    "import os # for file management\n",
    "import subprocess # allows us to run command line commands\n",
    "import pdal # lidar processing package\n",
    "import rasterio\n",
    "import pyproj\n",
    "from osgeo import gdal\n",
    "\n",
    "import time\n",
    "# packages to copy files\n",
    "from pathlib import Path\n",
    "import shutil\n",
    "import glob\n",
    "# packages to extract wkt from polygon\n",
    "import shapefile\n",
    "import pygeoif\n",
    "# for parallel processing\n",
    "import concurrent.futures\n",
    "from multiprocessing import Pool\n",
    "import csv\n",
    "\n",
    "import re\n",
    "import seaborn as sns\n",
    "import scipy\n",
    "#import seaborn as sns\n",
    "from scipy.interpolate import interp1d\n",
    "from scipy.stats import kde\n",
    "from scipy.stats import gaussian_kde\n",
    "# from sklearn import linear_model\n",
    "# from sklearn.metrics import mean_squared_error, r2_score\n",
    "# See lidar_functions.py\n",
    "import lidar_functions\n",
    "pdal_pipeline = 'C:\\\\Users\\cpiske\\.conda\\envs\\lidar\\Lib\\site-packages\\pdal\\pipeline.py'\n",
    "gdal_edit = os.path.join('C:\\\\','Users','cpiske','.conda','envs','lidar','Lib','site-packages','osgeo_utils','gdal_edit.py')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afec494a-5f52-4fc5-8ced-a5886f938892",
   "metadata": {},
   "source": [
    "Note that many functions are dependent on specific directory structures. See README"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "11eb2762-4c68-4a8e-a2e0-3c4434606bbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_base_path = 'piske_processing/PDAL_workflow/JSON/' # set so that we can redefine json across operating systems"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1933dbf5-603d-4eac-a4df-8ec90e6278d4",
   "metadata": {},
   "source": [
    "Applied"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e7a6c326-633d-4550-b011-0f012e2b2825",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'G:\\\\'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Set current working directory\n",
    "#path = '/Volumes/Piske_lidar'\n",
    "path = '/'\n",
    "os.chdir(path)\n",
    "os.getcwd() # print to ensure we're in the right directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "87f30faa-fe09-4f19-b736-214b2aad11c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_base_path = 'piske_processing/PDAL_workflow/JSON/' # set so that we can redefine json across operating systems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "600a92ec-7986-41ff-99d1-28daea0535a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define all json files\n",
    "filterMergeRasterize_json = json_base_path + 'filterMergeRasterize.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c9edcfd-2013-4858-90a5-d9ca09c35507",
   "metadata": {},
   "outputs": [],
   "source": [
    "# All json paths\n",
    "DEM_json = json_base_path+'DEM_from_las.json' \n",
    "HAG_json = json_base_path + 'HAG_dem.json'\n",
    "clip_json = json_base_path +'clip_to_polygon.json'\n",
    "extract_las_atPoint = json_base_path +'extract_las_atPoint.json'\n",
    "extract_las_atPolygon = json_base_path +'extract_las_atPolygon.json'\n",
    "correct_merge_rasterize_json = json_base_path + 'correct_merge_rasterize.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cda00195-88f1-48ea-a2fe-addbc08c7eef",
   "metadata": {},
   "outputs": [],
   "source": [
    "writers_bounds = '([730235.96, 738826.45], [4364741.66, 4372273.16])'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c371e470-c770-4b4d-93a4-52d882661c02",
   "metadata": {},
   "source": [
    "# Pre-Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "019afb1c-f0f0-4e99-8072-cdabfaee5be6",
   "metadata": {},
   "source": [
    "## Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "caff1251-3a50-4fec-9e91-4837b39954f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CompletedProcess(args=['pdal', 'info', 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/HAG/NCALM_SCB_2014_730000_4366000.las', '--metadata'], returncode=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define input file\n",
    "input_lid = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/HAG/NCALM_SCB_2014_730000_4366000.las'\n",
    "pdal_info_cmd = ['pdal','info',input_lid] # general info\n",
    "pdal_metadata_cmd = ['pdal','info',input_lid,'--metadata'] # full file metadata, including details crs\n",
    "subprocess.run(pdal_metadata_cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "47ecd2f6-067f-4c0a-bba7-86442f1d6c7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "pdal_info_results = subprocess.run(pdal_metadata_cmd, stdout = subprocess.PIPE) # stout (standard out), PIPE indicates that a new pipe to the child should be created\n",
    "pdal_info_dict = json.loads(pdal_info_results.stdout.decode()) # create dict with metadata info\n",
    "#pdal_info_dict # print results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25fb2bc9-743a-480a-9ce6-7838c8ca8be6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pdal_info_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bd7cd5d-0ac8-419c-9d36-fa0589d65e04",
   "metadata": {},
   "source": [
    "Check extents (in order to create bounds for rasterization below)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcfa978a-0e82-4d69-81f3-fcc1a59e258c",
   "metadata": {},
   "outputs": [],
   "source": [
    "lidar_folder = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20220321/las/'\n",
    "# lidar_folder = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/NAD83_NAD83_epoch2010/'\n",
    "onlyfiles = [f for f in os.listdir(lidar_folder) if os.path.isfile(os.path.join(lidar_folder, f))]\n",
    "full_paths = [lidar_folder +f for f in onlyfiles]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0990454-2c2d-4b1b-906a-0561933b7954",
   "metadata": {},
   "outputs": [],
   "source": [
    "# NCALM_2014_ext = lidar_functions.check_flight_extent(full_paths)\n",
    "# ASO_SCB_20160326_ext = lidar_functions.check_flight_extent(full_paths)\n",
    "# ASO_SCB_20160417_ext = lidar_functions.check_flight_extent(full_paths)\n",
    "# ASO_SCB_20160518_ext = lidar_functions.check_flight_extent(full_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8516d352-8076-4ecd-89bd-2ca235b8ba93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(NCALM_2014_ext)\n",
    "# print(ASO_SCB_20160326_ext)\n",
    "# print(ASO_SCB_20160417_ext)\n",
    "print(ASO_SCB_20160518_ext)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff1066ad-9023-43c1-a6a9-199681166dec",
   "metadata": {},
   "source": [
    "05/18/2016 has the most limited extent, so we'll use those bounds for rasterization."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4751d94d-d53d-40ce-a8d4-d62adad47b08",
   "metadata": {},
   "source": [
    "## Retile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea4032e7-3646-4f0e-9d5e-acc98c7ea589",
   "metadata": {},
   "outputs": [],
   "source": [
    "lidar_folder = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/NAD83_NAD83_epoch2010/'\n",
    "retile_folder = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/retile_uo/'\n",
    "onlyfiles = [f for f in os.listdir(lidar_folder) if os.path.isfile(os.path.join(lidar_folder, f))]\n",
    "for files in onlyfiles:\n",
    "    full_path = os.path.join(lidar_folder, files)\n",
    "    output_path = retile_folder+'#' + files\n",
    "    retile_command = ['pdal', 'tile', full_path, output_path, '--length=1000','--buffer=50']\n",
    "    subprocess.run(retile_command)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26292a6b-d7e1-4671-8a2c-81b27432469c",
   "metadata": {},
   "source": [
    "## Rename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd725789-60cb-4aa6-b79f-dfca7e1ff3f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "lidar_folder = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20220321/laz/'\n",
    "tic = time.perf_counter()\n",
    "if __name__ == '__main__':\n",
    "    with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor:\n",
    "        onlyfiles = [f for f in os.listdir(lidar_folder) if os.path.isfile(os.path.join(lidar_folder, f))]\n",
    "        full_path = [os.path.join(lidar_folder, s) for s in onlyfiles]\n",
    "        executor.map(lidar_functions.rename_llx_lly, full_path) \n",
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa1b2a67-2212-4cd4-94e1-41f66634a893",
   "metadata": {},
   "outputs": [],
   "source": [
    "lidar_folder = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20220321/laz/'\n",
    "tic = time.perf_counter()\n",
    "if __name__ == '__main__':\n",
    "    with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor:\n",
    "        onlyfiles = [f for f in os.listdir(lidar_folder) if os.path.isfile(os.path.join(lidar_folder, f))]\n",
    "        full_path = [os.path.join(lidar_folder, s) for s in onlyfiles]\n",
    "        executor.map(lidar_functions.add_str_to_filename, full_path) \n",
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a97d2d1-e28e-4bed-a4ba-b0363d4434a4",
   "metadata": {},
   "source": [
    "## Save Tile Boundaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbace8bd-7da0-4913-9960-c65f1eca84cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# one folder\n",
    "tic = time.perf_counter()\n",
    "lidar_folder = 'MRB/Merced_lidar/ASO/ASO_MRB_20210429/laz/'\n",
    "output_folder = 'MRB/Merced_lidar/ASO/ASO_MRB_20210429/tindex/original/'\n",
    "if __name__ == '__main__':\n",
    "    with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:\n",
    "        onlyfiles = [f for f in os.listdir(lidar_folder) if os.path.isfile(os.path.join(lidar_folder, f))]\n",
    "        full_path = [os.path.join(lidar_folder, s) for s in onlyfiles]\n",
    "        # change depending on directory formats\n",
    "        #output_path = [os.path.join(os.path.basename(lidar_folder),'tindex/tiles/' + s[:-3] + 'sqlite') for s in onlyfiles]\n",
    "        output_path = [output_folder + s[:-3] + 'sqlite' for s in onlyfiles]\n",
    "        executor.map(lidar_functions.create_tindex, full_path, output_path) #running 10 times\n",
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a59151d2-faa6-445f-b84a-86ecb4fa60fe",
   "metadata": {},
   "source": [
    "## Copy Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4528ffe9-6116-491e-967c-7fbfb1fdbade",
   "metadata": {},
   "outputs": [],
   "source": [
    "tic = time.perf_counter()\n",
    "lidar_folder = 'MRB/Merced_lidar/ASO/ASO_MRB_20210429/laz/'\n",
    "output_path = 'MRB/Merced_lidar/ASO/ASO_MRB_20210429/ICB_tiles/'\n",
    "onlyfiles = [f for f in os.listdir(lidar_folder) if os.path.isfile(os.path.join(lidar_folder, f))]\n",
    "full_paths = [os.path.join(lidar_folder, s) for s in onlyfiles] \n",
    "output_paths = np.repeat(output_path, len(full_paths))\n",
    "if __name__ == '__main__':\n",
    "    with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:\n",
    "        executor.map(lidar_functions.copy_lid_by_ext_ICB, full_paths, output_paths)\n",
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6df8c3d4-0865-494a-915e-3e0a3d4051ed",
   "metadata": {},
   "source": [
    "**Fix Metadata**\n",
    "\n",
    "Due to an issue with earlier vertical datum conversions (using VDATUM version before update in 05/2022), some files were repreojected into the incorrect CRS. Others were simply assigned the wrong metadata. We'll deal with this issue within the rasterization pipelines. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a8934cc-5009-45a1-9b5b-852df2c3a908",
   "metadata": {},
   "source": [
    "# Create DEM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c05e3051-45c4-4ab1-8076-68fe36d1e098",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "DEM_json = json_base_path+'DEM_from_las.json' # define path to json files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cd7b5912-904f-4064-9a78-32ac0ad932aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/NAD83_NAD83_epoch2010/'\n",
    "output_tif = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/DEM/SCB/NCALM_2014_BE.tif'\n",
    "onlyfiles = [f for f in os.listdir(input_path) if os.path.isfile(os.path.join(input_path, f))]\n",
    "input_list = [input_path + s for s in onlyfiles]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0850ec42-97c2-4944-88b4-256f8d4dbe51",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_dict = {}\n",
    "tags = ['']*len(input_list)\n",
    "filenames = ['']*len(input_list)\n",
    "for i in range(len(input_list)):\n",
    "    filename_dict['filename_'+str(i)] = {'filename':input_list[i], 'tag':'A_'+str(i)}\n",
    "    tags[i] = 'A_'+str(i)\n",
    "    filenames[i] = filename_dict[list(filename_dict)[i]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6f364afe-c052-49b4-a645-25e33b360247",
   "metadata": {},
   "outputs": [],
   "source": [
    "#writers_bounds = '(['+ str(ASO_SCB_20160518_ext[0])+','+ str(ASO_SCB_20160518_ext[1])+'],['+str(ASO_SCB_20160518_ext[2])+','+str(ASO_SCB_20160518_ext[3])+'])'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b114ffa3-9aef-4a9c-91cb-0528a5a27d9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter out the ground points of the tiles\n",
    "filter_range = {\"type\":\"filters.range\",\n",
    "                \"limits\":\"Classification[2:2]\"}\n",
    "# merge all las files or stages\n",
    "filter_merge = {\"type\":\"filters.merge\",\n",
    "               \"tag\": \"merged\",\n",
    "               \"inputs\": tags}\n",
    "# write merged las to raster\n",
    "writers_gdal= {\"type\": \"writers.gdal\",\n",
    "               'output_type': 'mean', \n",
    "              'resolution': '1.0',\n",
    "               'override_srs':'EPSG:6339',\n",
    "               'bounds': writers_bounds,\n",
    "              'radius': '0.7',\n",
    "               'window_size':3,# we want more of a wall-to-wall product here so we use a secondary algorithm to increase calculation distance\n",
    "               'filename': output_tif}\n",
    "# Append each stage to a list prior to saving to json \n",
    "pipeline_list = filenames.copy()\n",
    "pipeline_list.append(filter_merge)\n",
    "pipeline_list.append(filter_range)\n",
    "pipeline_list.append(writers_gdal)\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "# save to json\n",
    "with open(DEM_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0c1324e2-3971-44fc-a2f8-10f60ab76ab0",
   "metadata": {},
   "outputs": [],
   "source": [
    "tic = time.perf_counter()\n",
    "pdal_cmd = ['pdal','pipeline', DEM_json]\n",
    "subprocess.run(pdal_cmd)\n",
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bf0d2cf8-94ad-49b5-ad98-938347a60cce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16.608580006666667"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(toc-tic)/60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3bb028f-69b8-4ebd-8c38-0e66b294858a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "13399307-5a71-472d-9039-262c703e83e7",
   "metadata": {},
   "source": [
    "## Heigh Above Ground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf501c4d-8952-462c-9c9a-138e0b7e52eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define json path \n",
    "HAG_json = json_base_path + 'HAG_dem.json'\n",
    "# convert all z values to the height above ground \n",
    "target_dem = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/DEM/NCALM_2014_BE.tif'\n",
    "filter_hag = {\"type\":\"filters.hag_dem\",\n",
    "              \"raster\":target_dem, # full file path of target DEM (.tif)\n",
    "              \"zero_ground\":\"false\"} # Do not assign 0 to ground classified points\n",
    "filter_ferry = {\"type\":\"filters.ferry\",\n",
    "                \"dimensions\":\"HeightAboveGround=>Z\"} # replace all Z dimensions with HAG instead of elevation\n",
    "filter_range = {\"type\":\"filters.range\",\n",
    "                \"limits\":\"Z[-0.2:70]\"} # apply a noise filter\n",
    "pipeline_list = [filter_hag, filter_ferry,filter_range]\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "with open(HAG_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8c1511e-856a-426e-9794-75b704b146bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parallel processing\n",
    "# time = 1.1 min\n",
    "tic = time.perf_counter()\n",
    "input_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/NAD83_NAD83_epoch2010/'\n",
    "output_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/HAG/'\n",
    "if __name__ == '__main__':\n",
    "    with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:\n",
    "        onlyfiles = [f for f in os.listdir(input_path) if os.path.isfile(os.path.join(input_path, f))]\n",
    "        full_path = [input_path + s for s in onlyfiles]\n",
    "        output_path_full = [output_path  + s for s in onlyfiles]\n",
    "        executor.map(lidar_functions.HAG_dem, full_path, output_path_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8be62742-e0d1-4f5b-b0ff-0632b0286250",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parallel processing\n",
    "# time = 1.1 min\n",
    "tic = time.perf_counter()\n",
    "input_path = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160417/NAD83_NAD83_epoch2010/'\n",
    "output_path = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160417/HAG/'\n",
    "if __name__ == '__main__':\n",
    "    with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:\n",
    "        onlyfiles = [f for f in os.listdir(input_path) if os.path.isfile(os.path.join(input_path, f))]\n",
    "        full_path = [input_path + s for s in onlyfiles]\n",
    "        output_path_full = [output_path  + s for s in onlyfiles]\n",
    "        executor.map(lidar_functions.HAG_dem, full_path, output_path_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6b9822c-9db8-4132-b641-06605deb1ba3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parallel processing\n",
    "# time = 1.1 min\n",
    "tic = time.perf_counter()\n",
    "input_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/NAD83_NAD83_epoch2010/'\n",
    "output_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/HAG/'\n",
    "if __name__ == '__main__':\n",
    "    with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:\n",
    "        onlyfiles = [f for f in os.listdir(input_path) if os.path.isfile(os.path.join(input_path, f))]\n",
    "        full_path = [input_path + s for s in onlyfiles]\n",
    "        output_path_full = [output_path  + s for s in onlyfiles]\n",
    "        executor.map(lidar_functions.HAG_dem, full_path, output_path_full)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a22bb40-5dd2-4a68-986c-582474a5cd67",
   "metadata": {},
   "source": [
    "RASTERIZE TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab2dae61-1ed9-44df-9705-93a6a238626b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parallel processing\n",
    "# time = 1.1 min\n",
    "tic = time.perf_counter()\n",
    "input_path = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160326/HAG/'\n",
    "output_path = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160326/rasterize_test_count/'\n",
    "if __name__ == '__main__':\n",
    "    with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:\n",
    "        onlyfiles = [f for f in os.listdir(input_path) if os.path.isfile(os.path.join(input_path, f))]\n",
    "        full_path = [input_path + s for s in onlyfiles]\n",
    "        output_path_full = [output_path  + s for s in onlyfiles]\n",
    "        executor.map(lidar_functions.rasterize_count, full_path, output_path_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7178fbeb-cbe4-439e-aab9-57e2ec3084ef",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "7ad2a53a-72cd-4fe7-8c9f-fe68102cdbc5",
   "metadata": {},
   "source": [
    "# Vertical Bias Correction"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "926a5b5b-e820-473b-8016-c6844b1d40a0",
   "metadata": {},
   "source": [
    "## Move Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5818571-a968-43f1-9514-6ef591da593a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining source and destination\n",
    "# paths\n",
    "src = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160417/HAG/*.la*'\n",
    "trg = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160417/hwy89_vertical_bias/target_lid'\n",
    "file_paths = glob.glob(src)\n",
    "\n",
    "for files in file_paths:\n",
    "    filename = os.path.basename(files)\n",
    "    if int(files[-18:-12]) > 737000 and int(files[-11:-4]) > 4370000:\n",
    "    # copying the files to the\n",
    "    # destination directory\n",
    "        shutil.copy2(files, trg)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8451906-bed0-43fb-be02-7a1f242dc0e0",
   "metadata": {},
   "source": [
    "### Merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78b38fda-766e-48b1-847f-94ae5a1c585c",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_path = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160326/hwy89_vertical_bias/target_lid/*.la*' # define path of input files\n",
    "input_fname = glob.glob(input_path) # save to list\n",
    "output_fname = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160326/hwy89_vertical_bias/target_lid/ASO_SCB_20160326_hwy89_merge.'+input_fname[0][-3:]# set output filename\n",
    "pdal_merge_command = input_fname\n",
    "pdal_merge_command.insert(len(pdal_merge_command),output_fname) # insert output file to list\n",
    "#pdal_merge_command.insert(0,'-f')\n",
    "pdal_merge_command.insert(0,'merge')\n",
    "pdal_merge_command.insert(0,'pdal')\n",
    "subprocess.run(pdal_merge_command)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebdd7829-317e-42c3-8f2d-9a882d93c793",
   "metadata": {},
   "source": [
    "### Clip"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f35d800a-8876-447f-86a7-89e8c4997fc8",
   "metadata": {},
   "source": [
    "#### ASO 2016\n",
    "The 2016 data uses points overlapping a hwy89 polygon for vertical bias correction because it should have no snow. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d916c772-c2e7-4aba-82d8-997cd8ba09ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# name JSON file\n",
    "clip_json = json_base_path +'clip_to_geometries.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5863300c-fc3f-4fc5-a875-b38f72ae50f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_shapefile = 'SCB/supporting_files/bounding_box/hwy89_poly.shp'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69d954e8-927c-460e-8c87-38577deb244c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract wkt from hwy 89 polygon\n",
    "hwy89 = shapefile.Reader(path_to_shapefile)\n",
    "geom=[]\n",
    "for s in hwy89.shapes():\n",
    "    geom.append(pygeoif.geometry.as_shape(s)) \n",
    "poly_base = pygeoif.MultiPolygon(geom)\n",
    "target_poly = poly_base.wkt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "039a00c7-1565-4dca-9471-21c7b787ee37",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_las_atPolygon(filenames, tags, target_poly, output_txt):\n",
    "    # merge all las files or stages\n",
    "    filter_merge = {\"type\":\"filters.merge\",\n",
    "                    \"tag\": \"merged\",\n",
    "                    \"inputs\": tags}\n",
    "    # crop\n",
    "    filter_crop = {'type':'filters.crop',\n",
    "                   'polygon':target_poly,\n",
    "                   'distance':1,\n",
    "                   'inputs':'merged',\n",
    "                   'tag': 'cropped'}\n",
    "    # write merged las to raster\n",
    "    writers_gdal = {\"type\":\"writers.text\",\n",
    "                    \"format\":\"csv\",\n",
    "                    \"order\":\"Z\",\n",
    "                    'keep_unspecified':False,\n",
    "                    'filename':output_txt}\n",
    "    # Append each stage to a list prior to saving to json \n",
    "    pipeline_list = filenames.copy()\n",
    "    pipeline_list.append(filter_merge)\n",
    "    pipeline_list.append(filter_crop)\n",
    "    pipeline_list.append(writers_gdal)\n",
    "\n",
    "    pipeline_dict = {'pipeline' : pipeline_list}\n",
    "    # save to json\n",
    "    with open(extract_las_atPolygon_json, 'w') as out:\n",
    "        json.dump(pipeline_dict, out, indent=4)\n",
    "    pdal_cmd = ['pdal','pipeline', extract_las_atPolygon_json]\n",
    "    subprocess.run(pdal_cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71402413-e728-42b6-91cb-a000371da216",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_las = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160518/hwy89_vertical_bias/target_lid/ASO_SCB_20160518_hwy89_merge.las'\n",
    "output_txt = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160518/hwy89_vertical_bias/clipped/ASO_SCB_20160518_hwy89_clip.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d09b842-a504-4bab-8c81-8af8d202f3c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "pdal_commands = ['pdal', 'translate', input_las, output_txt, '--json', clip_json]\n",
    "subprocess.run(pdal_commands)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5974506f-0990-414e-98db-ad79a9eb29e4",
   "metadata": {},
   "source": [
    "#### NCALM 2008\n",
    "The 2008 data uses snow-on field data extracted at points for vertical bias correction. <br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7551b5d-a8a1-40eb-84f5-25e940002bc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# name JSON file\n",
    "extract_las_atPoint_json = json_base_path +'extract_las_atPoint.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56467508-027c-4fda-b448-bcbed112e845",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in ground_truthed data\n",
    "def csv_to_list(input_csv):\n",
    "    src = open(input_csv)\n",
    "    csvreader = csv.reader(src)\n",
    "    header = next(csvreader)\n",
    "    output_list = []\n",
    "    for row in csvreader:\n",
    "        output_list.append(row)\n",
    "    src.close()\n",
    "    return(header, output_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fee3fb6-292f-4727-9bb2-f691722e7e72",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_las_atPoint(filenames, tags, target_point, output_txt,distance):\n",
    "    # merge all las files or stages\n",
    "    filter_merge = {\"type\":\"filters.merge\",\n",
    "                    \"tag\": \"merged\",\n",
    "                    \"inputs\": tags}\n",
    "    # crop\n",
    "    filter_crop = {'type':'filters.crop',\n",
    "                   'point':target_point,\n",
    "                   'distance':distance,\n",
    "                   'inputs':'merged',\n",
    "                   'tag': 'cropped'}\n",
    "    # crop\n",
    "    filter_range = {'type':'filters.range',\n",
    "                   'limits':'Classification[2:2]',\n",
    "                   'inputs':'cropped',\n",
    "                   'tag': 'range'}\n",
    "    \n",
    "    # write merged las to raster\n",
    "    writers_gdal = {\"type\":\"writers.text\",\n",
    "                    \"format\":\"csv\",\n",
    "                    \"order\":\"Z\",\n",
    "                    \"write_header\":False,\n",
    "                    'keep_unspecified':False,\n",
    "                    'filename':output_txt}\n",
    "    # Append each stage to a list prior to saving to json \n",
    "    pipeline_list = filenames.copy()\n",
    "    pipeline_list.append(filter_merge)\n",
    "    pipeline_list.append(filter_crop)\n",
    "    pipeline_list.append(filter_range)\n",
    "    pipeline_list.append(writers_gdal)\n",
    "\n",
    "    pipeline_dict = {'pipeline' : pipeline_list}\n",
    "    # save to json\n",
    "    with open(extract_las_atPoint_json, 'w') as out:\n",
    "        json.dump(pipeline_dict, out, indent=4)\n",
    "    pdal_cmd = ['pdal','pipeline', extract_las_atPoint_json]\n",
    "    subprocess.run(pdal_cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df4742ce-3bb6-45ad-ae59-855fa60c362d",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/HAG/'\n",
    "output_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/vertical_bias_correction/'\n",
    "filenames, tags = lidar_functions.create_command_template(input_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc0026e0-8c6f-44fe-b339-aa695bba261e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract data from field data\n",
    "SNOTEL_src = \"SCB/supporting_files/combined_field_snow/Harpold_data_paper/snowdepth_filtered.csv\"\n",
    "snotel_header, snotel_data = csv_to_list(SNOTEL_src)\n",
    "hunt_src = 'SCB/supporting_files/combined_field_snow/Huntingon_2008_snow/Hungington_2008_snow.csv'\n",
    "hunt_header, hunt_data = csv_to_list(hunt_src)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9efb5d7d-a0d8-4c35-bb52-b7723ad71e0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(len(hunt_data)):\n",
    "#     for j in range(1,6):\n",
    "#         output_txt = output_path+'huntington_'+str(i)+'_dist'+str(j)+'.csv'\n",
    "#         target_point = 'POINT('+hunt_data[i][7]+' '+hunt_data[i][8]+')'\n",
    "#         extract_las_atPoint(filenames, tags, target_point, output_txt,j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63dde43b-606f-4d94-a50b-fc7135ae6f93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# time = 4 min\n",
    "for i in range(len(snotel_data)):\n",
    "    for j in range(1,6):\n",
    "        output_txt = output_path+'snotel_'+str(i)+'_dist'+str(j)+'.csv'\n",
    "        target_point = 'POINT('+snotel_data[i][5]+' '+snotel_data[i][6]+')'\n",
    "        extract_las_atPoint(filenames, tags, target_point, output_txt,j)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bbc45e7-d895-43f8-9aab-af78f1328c3a",
   "metadata": {},
   "source": [
    "## Calculate Stats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8b75ae5-8299-468b-a120-67249d868a86",
   "metadata": {},
   "source": [
    "Applied\n",
    "Using 2014 NCALM Flight"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dca5384c-26c6-41bf-9688-1295e44054f6",
   "metadata": {},
   "source": [
    "### ASO 2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2960560b-0428-448b-9030-e199946193f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "ASO_20160326_hwy89_arr = np.loadtxt('SCB/Sagehen_lidar/ASO/ASO_SCB_20160326/hwy89_vertical_bias/clipped/ASO_SCB_20160326_hwy89_clip.csv',skiprows=1)\n",
    "ASO_20160417_hwy89_arr = np.loadtxt('SCB/Sagehen_lidar/ASO/ASO_SCB_20160417/hwy89_vertical_bias/clipped/ASO_SCB_20160417_hwy89_clip.csv',skiprows=1)\n",
    "ASO_20160518_hwy89_arr = np.loadtxt('SCB/Sagehen_lidar/ASO/ASO_SCB_20160518/hwy89_vertical_bias/clipped/ASO_SCB_20160518_hwy89_clip.csv',skiprows=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bbe0cd43-447f-40c8-9bd1-73711bfbe4cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ASO_20160326_hwy89_stats = lidar_functions.calculate_vertical_bias_over_snowOff('SCB/Sagehen_lidar/ASO/ASO_SCB_20160326/hwy89_vertical_bias/clipped/ASO_SCB_20160326_hwy89_clip.laz')\n",
    "# ASO_20160417_hwy89_stats = lidar_functions.calculate_vertical_bias_over_snowOff('SCB/Sagehen_lidar/ASO/ASO_SCB_20160417/hwy89_vertical_bias/clipped/ASO_SCB_20160417_hwy89_clip.laz')\n",
    "# ASO_20160518_hwy89_stats = lidar_functions.calculate_vertical_bias_over_snowOff('SCB/Sagehen_lidar/ASO/ASO_SCB_20160518/hwy89_vertical_bias/clipped/ASO_SCB_20160518_hwy89_clip.las')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c724eab8-e366-4c29-a3d0-33d69b753df8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# NCALM_2014_hwy89_stats = lidar_functions.calculate_vertical_bias('SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/hwy89_vertical_bias/clipped/NCALM_SCB_2014_hwy89_clip.las')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "37e85b56-2fbf-4b1d-a18d-f0c36bd90ab7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.24\n",
      "0.31\n"
     ]
    }
   ],
   "source": [
    "print(np.nanpercentile((ASO_20160326_hwy89_arr),10))\n",
    "print(np.nanpercentile((ASO_20160326_hwy89_arr),50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aeaaa728-27ac-4ded-974b-f9d88fa65fcd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.27\n",
      "0.35\n"
     ]
    }
   ],
   "source": [
    "print(np.nanpercentile((ASO_20160417_hwy89_arr),10))\n",
    "print(np.nanpercentile((ASO_20160417_hwy89_arr),50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4bd95dde-4dd2-4cef-a705-921f1403e148",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.37\n",
      "0.44\n"
     ]
    }
   ],
   "source": [
    "print(np.nanpercentile((ASO_20160518_hwy89_arr),10))\n",
    "print(np.nanpercentile((ASO_20160518_hwy89_arr),50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c00b8cd-d1e1-4f4d-baa0-cb7d58b4c083",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(ASO_20160326_hwy89_arr-0.31, bins=40)\n",
    "plt.axvline(0, color='k')\n",
    "#plt.xlim([-.2, 0.75])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6670984d-73d8-462e-b036-31e6d3dd2015",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(ASO_20160326_hwy89_stats)\n",
    "print(ASO_20160417_hwy89_stats)\n",
    "print(ASO_20160518_hwy89_stats)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f456a359-3004-44c3-b3a5-e2cea8e8d92f",
   "metadata": {},
   "source": [
    "### **NCALM 2008**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a31fa74-0489-4f75-9ddc-2801b3bfa425",
   "metadata": {},
   "outputs": [],
   "source": [
    "# input_las_txt = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/SNOTEL_vertical_bias/clipped/snotel_541.csv'\n",
    "input_las_txt = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/SNOTEL_vertical_bias/clipped/snotel_539.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b32fe203-316a-4976-81e5-6d41d5e8730b",
   "metadata": {},
   "outputs": [],
   "source": [
    "hag_arr = np.loadtxt(input_las_txt,skiprows=1)\n",
    "lowest_10th_per = np.nanpercentile(hag_arr, 10)\n",
    "mean_hag = np.nanmean(hag_arr)\n",
    "median_hag = np.nanmedian(hag_arr)\n",
    "stats_539 = [\"lowest_10th\",lowest_10th_per, \"mean\",mean_hag, \"median\", median_hag]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d0b2900-2d7f-41d6-a065-3cdaeb8e8bed",
   "metadata": {},
   "outputs": [],
   "source": [
    "NCALM_2008_vbc_dict = {}\n",
    "input_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/vertical_bias_correction/ground_pts/'\n",
    "onlyfiles = [f for f in os.listdir(input_path) if os.path.isfile(os.path.join(input_path, f))] # make a list of all filenames in directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf21b891-2b16-4b8b-b0a5-209888375c3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "NCALM_2008_vbc_dict = {}\n",
    "all_files = glob.glob('SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/vertical_bias_correction/ground_pts/*')\n",
    "for files in onlyfiles:\n",
    "    txt_path = input_path + files\n",
    "    txt_arr = np.loadtxt(txt_path)\n",
    "    NCALM_2008_vbc_dict[files[:-4]] = txt_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03e1b4d9-54b2-4c4c-9a71-8fc47f317a83",
   "metadata": {},
   "outputs": [],
   "source": [
    "NCALM_2008_error_dict = {}\n",
    "keys_list = list(NCALM_2008_vbc_dict.keys())\n",
    "for i in range(len(keys_list)):\n",
    "    index = int(re.findall(r'\\d+',  keys_list[i])[0])\n",
    "    if keys_list[i][0:3]=='hun':\n",
    "        NCALM_2008_error_dict[keys_list[i]] = float(hunt_data[index][4]) - NCALM_2008_vbc_dict[keys_list[i]]\n",
    "    elif keys_list[i][0:3]=='sno':\n",
    "        NCALM_2008_error_dict[keys_list[i]] = float(snotel_data[index][4])/100 - NCALM_2008_vbc_dict[keys_list[i]]\n",
    "    else:\n",
    "        print('error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b6dc373-2439-4cb9-98ac-0d1532af3d99",
   "metadata": {},
   "outputs": [],
   "source": [
    "NCALM_2008_error_plot, axs = plt.subplots(5, 11, figsize =(30,10));\n",
    "#NCALM_2008_error_plot.suptitle('NCALM 2008 Lidar Depth Distributions Around Ground Truth') # add title to entire figure\n",
    "\n",
    "k = 0\n",
    "for i in range(11):\n",
    "    for j in range(5):\n",
    "        keys_list = list(NCALM_2008_vbc_dict.keys())\n",
    "        index = int(re.findall(r'\\d+',  keys_list[k])[0])\n",
    "        true_val = float(hunt_data[index][4])\n",
    "        g = sns.histplot(ax = axs[j,i], data = NCALM_2008_vbc_dict[keys_list[k]])\n",
    "        med = np.median(NCALM_2008_vbc_dict[keys_list[k]])\n",
    "        g.axvline(true_val, color='red')\n",
    "        g.axvline(med, color = 'green')\n",
    "        y_lab = str(j+1)+'m'\n",
    "        x_lab = \"Point \" +str(i)\n",
    "        axs[j,i].set(yticklabels = [], title=' ',xlabel = '',ylabel ='')\n",
    "        axs[0,0].set_ylabel('Radius = 1m'); axs[1,0].set_ylabel('Radius = 2m');axs[2,0].set_ylabel('Radius = 3m');axs[3,0].set_ylabel('Radius = 4m');axs[4,0].set_ylabel('Radius = 5m')\n",
    "        k = k+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11b62a20-03a4-4e5a-be51-1e36ea255b67",
   "metadata": {},
   "outputs": [],
   "source": [
    "NCALM_2008_error_plot, axs = plt.subplots(5, 11, figsize =(30,10));\n",
    "#NCALM_2008_error_plot.suptitle('NCALM 2008 Lidar Depth Distributions Around Ground Truth') # add title to entire figure\n",
    "\n",
    "k = 0\n",
    "for i in range(11):\n",
    "    for j in range(5):\n",
    "        keys_list = list(NCALM_2008_error_dict.keys())\n",
    "        index = int(re.findall(r'\\d+',  keys_list[k])[0])\n",
    "        #true_val = float(hunt_data[index][4])\n",
    "        g = sns.histplot(ax = axs[j,i], data = NCALM_2008_error_dict[keys_list[k]])\n",
    "        med = np.median(NCALM_2008_error_dict[keys_list[k]])\n",
    "        # g.axvline(true_val, color='red')\n",
    "        g.axvline(med, color = 'green')\n",
    "        y_lab = str(j+1)+'m'\n",
    "        x_lab = \"Point \" +str(i)\n",
    "        axs[j,i].set(yticklabels = [], title=' ',xlabel = '',ylabel ='')\n",
    "        axs[0,0].set_ylabel('Radius = 1m'); axs[1,0].set_ylabel('Radius = 2m');axs[2,0].set_ylabel('Radius = 3m');axs[3,0].set_ylabel('Radius = 4m');axs[4,0].set_ylabel('Radius = 5m')\n",
    "        k = k+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24815514-66ef-4dc9-9fc3-0e2506db0bff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_error(dist):\n",
    "    error_med = []\n",
    "    keys_list = list(NCALM_2008_error_dict.keys())\n",
    "    for k in range(int(len(NCALM_2008_error_dict))):\n",
    "        if int(re.findall(r'\\d+',  keys_list[k])[1])==dist:\n",
    "            error_med.extend(NCALM_2008_error_dict[keys_list[k]])\n",
    "    return(error_med)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26b9cadc-b444-4478-850f-216c1bbadc44",
   "metadata": {},
   "outputs": [],
   "source": [
    "error_med_2 = calculate_error(2)\n",
    "error_med_3 = calculate_error(3)\n",
    "error_med_4 = calculate_error(4)\n",
    "error_med_5 = calculate_error(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5acebaa-af2b-4fb8-a063-2407bf1c3083",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Radius = 2m, median error = \",np.median(error_med_2),'stdev = ',np.std(error_med_2))\n",
    "print(\"Radius = 3m, median error = \",np.median(error_med_3),'stdev = ',np.std(error_med_3))\n",
    "print(\"Radius = 4m, median error = \",np.median(error_med_4),'stdev = ',np.std(error_med_4))\n",
    "print(\"Radius = 5m, median error = \",np.median(error_med_5),'stdev = ',np.std(error_med_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c8d404e-911a-431b-82b4-a3c3c84d70b4",
   "metadata": {},
   "source": [
    "hold off on corrections for now"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91e5bcbb-8274-4847-b129-d1c92bb3cb58",
   "metadata": {},
   "source": [
    "## Correct and Rasterize\n",
    "In this case we know that the May flight has the most limited extent. One way to check for this would be to use a similar code to "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a60dcefd-7390-4041-844c-da5c58049da4",
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_merge_rasterize_json = json_base_path + 'correct_merge_rasterize_Zlim_5.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea3c9439-5269-4964-94e3-40a61bb02147",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/HAG/'\n",
    "filenames, tags = lidar_functions.create_command_template(input_path)\n",
    "output_tif = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_20080210/corrected_tif/NCALM_SCB_20080210_vbc_5.tif'\n",
    "target_assign = 'Z-0.03'#'Z-'+str(abs(ASO_20160518_hwy89_stats[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62d93f1b-301e-4406-b58f-c8dbcf62cf2a",
   "metadata": {},
   "source": [
    "Create the reader stages of the pipeline. Each file is read as an individual reader stage here, and we cheat here by copying the formatting of json files and creating a dictionary with the values as the correctly formatted inputs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28fec394-70ca-404f-9fb5-67b32e934ed7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# [730235.96, 738826.45, 4364741.66, 4372273.16, 8590.48999999999, 7531.5]\n",
    "writers_bounds = '(['+ str(ASO_SCB_20160518_ext[0])+','+ str(ASO_SCB_20160518_ext[1])+'],['+str(ASO_SCB_20160518_ext[2])+','+str(ASO_SCB_20160518_ext[3])+'])'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "822e3cdd-58ac-48a0-b748-663ea519090d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge all las files or stages\n",
    "filter_merge = {\"type\":\"filters.merge\",\n",
    "               \"tag\": \"merged\",\n",
    "               \"inputs\": tags}\n",
    "# filter out the ground points of the tiles\n",
    "filter_assign = {'type': 'filters.assign',\n",
    "                 'value':\"Z=\"+target_assign,\n",
    "                 'inputs':'merged',\n",
    "                 'tag':'corrected'}\n",
    "# filter out the ground points of the tiles\n",
    "# filter_range = {'type': 'filters.range',\n",
    "#                  'limits':\"Z[0.15:5]\",\n",
    "#                  'inputs':'corrected',\n",
    "#                  'tag':'ranged'}\n",
    "filter_range = {'type': 'filters.range',\n",
    "                 'limits':\"Z[:5]\",\n",
    "                 'inputs':'corrected',\n",
    "                 'tag':'ranged'}\n",
    "# write merged las to raster\n",
    "writers_gdal= {\"type\": \"writers.gdal\",\n",
    "               'output_type': 'mean',\n",
    "              'resolution': '1.0',\n",
    "              'radius': '0.7',\n",
    "               'bounds': writers_bounds,\n",
    "               'inputs': 'ranged',\n",
    "               'filename':output_tif}\n",
    "# Append each stage to a list prior to saving to json \n",
    "pipeline_list = filenames.copy()\n",
    "pipeline_list.append(filter_merge)\n",
    "pipeline_list.append(filter_assign)\n",
    "pipeline_list.append(filter_range)\n",
    "pipeline_list.append(writers_gdal)\n",
    "\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "# save to json\n",
    "with open(correct_merge_rasterize_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb980f62-9d55-40a9-99cc-1136f519df41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# time = ~3 min\n",
    "pdal_cmd = ['pdal','pipeline', correct_merge_rasterize_json]\n",
    "subprocess.run(pdal_cmd)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b466837-460a-4f8b-9202-f81a226fa651",
   "metadata": {},
   "source": [
    "## Snow-Off Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa814a07-3c70-4bc8-b6d3-3b35f3336c30",
   "metadata": {},
   "source": [
    "### Vegetation Strata"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7cba995-877a-4a96-a828-784ec57aade0",
   "metadata": {},
   "source": [
    "**Combined Pipeline Method**\n",
    "We'll use the same bounds as above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "106d03d8-7a0f-4f32-9f9d-6848384eca22",
   "metadata": {},
   "outputs": [],
   "source": [
    "#filterMergeRasterize_json = json_base_path+'filterMergeRasterize.json'\n",
    "# filterMergeRasterize_json = json_base_path+'filterMergeRasterize_neg0pt15_0pt15.json'\n",
    "# filterMergeRasterize_json = json_base_path+'filterMergeRasterize_0pt15_2.json'\n",
    "# filterMergeRasterize_json = json_base_path+'filterMergeRasterize_2.json'\n",
    "filterMergeRasterize_json = json_base_path+'filterMergeRasterize_0pt15_0pt15.json'\n",
    "# filterMergeRasterize_json = json_base_path+'filterMergeRasterize_2_nonground.json'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af094e83-88cd-46f4-801d-7f31c2bbbd05",
   "metadata": {},
   "outputs": [],
   "source": [
    "#writers_bounds = '(['+ str(ASO_SCB_20160518_ext[0])+','+ str(ASO_SCB_20160518_ext[1])+'],['+str(ASO_SCB_20160518_ext[2])+','+str(ASO_SCB_20160518_ext[3])+'])'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06a4bd3a-451f-4066-b6f2-4fe6bbfebc13",
   "metadata": {},
   "outputs": [],
   "source": [
    "# '([730235.96,738826.45],[4364741.66,4372273.16])'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa124ce7-908c-4981-947b-57d296be799a",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/HAG/'\n",
    "output_tif = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/veg_strata/veg_classes/vegStrata_0pt15_1pt5.tif'\n",
    "filenames, tags = lidar_functions.create_command_template(input_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24f04675-a114-4c79-b9a6-8bd0ec645a21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge all las files or stages\n",
    "filter_merge = {\"type\":\"filters.merge\",\n",
    "               \"tag\": \"merged\",\n",
    "               \"inputs\": tags}\n",
    "# filter out the ground points of the tiles\n",
    "filter_range = {\"type\":\"filters.range\",\n",
    "                \"limits\":\"Z[0.15:1.5)\",\n",
    "               'inputs':'merged',\n",
    "               'tag':'filtered'}\n",
    "# write merged las to raster\n",
    "writers_gdal= {\"type\": \"writers.gdal\",\n",
    "               'output_type': 'count',\n",
    "              'resolution': '1.0',\n",
    "              'radius': '0.7',\n",
    "               'bounds': writers_bounds,\n",
    "               'inputs': 'filtered',\n",
    "               'filename':output_tif}\n",
    "# Append each stage to a list prior to saving to json \n",
    "pipeline_list = filenames.copy()\n",
    "pipeline_list.append(filter_merge)\n",
    "pipeline_list.append(filter_range)\n",
    "pipeline_list.append(writers_gdal)\n",
    "\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "# save to json\n",
    "with open(filterMergeRasterize_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "850dd26e-cc34-4731-9c2a-1b27b0d9c723",
   "metadata": {},
   "outputs": [],
   "source": [
    "tic = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea7b3ba5-0b1c-4f7c-a068-180d263aba09",
   "metadata": {},
   "outputs": [],
   "source": [
    "pdal_cmd = ['pdal','pipeline', filterMergeRasterize_json]\n",
    "subprocess.run(pdal_cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4464b547-4114-43ad-8c6c-7ac3256b62ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efb4c3fe-af8d-4b65-b53f-04338df3d9e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "(toc-tic) / 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c86f8358-6e91-4889-b3bc-e1300bea86f0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6eee961-41c0-47e1-b6b3-f7a492816548",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c4a0bf40-98c0-423a-8b6a-a1a587782508",
   "metadata": {},
   "source": [
    "### Create CHM"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "747e3961-714c-49fd-bbd5-022c07a0b8d6",
   "metadata": {},
   "source": [
    "Note that there are two CHMs for these data. One is based off all points, another is limited to possible short/understory vegetation (<1.5m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3107c9e-e6db-46b8-96f6-d10b3a5d49ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "CHM_json_LT1pt5 = json_base_path+'CHM_from_las_LT1pt5.json' # define path to json files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b1a911e-a30c-47d0-b961-4d5b8818a145",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/HAG/'\n",
    "output_tif = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/CHM/NCALM_2014_CHM_LT1pt5.tif'\n",
    "onlyfiles = [f for f in os.listdir(input_path) if os.path.isfile(os.path.join(input_path, f))]\n",
    "input_list = [input_path + s for s in onlyfiles]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae874a5e-2129-4105-98da-fae42d42026a",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_dict = {}\n",
    "tags = ['']*len(input_list)\n",
    "filenames = ['']*len(input_list)\n",
    "for i in range(len(input_list)):\n",
    "    filename_dict['filename_'+str(i)] = {'filename':input_list[i], 'tag':'A_'+str(i)}\n",
    "    tags[i] = 'A_'+str(i)\n",
    "    filenames[i] = filename_dict[list(filename_dict)[i]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93c5a436-3444-4b72-8fd0-bb837b12087d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge all las files or stages\n",
    "filter_merge = {\"type\":\"filters.merge\",\n",
    "               \"tag\": \"merged\",\n",
    "               \"inputs\": tags}\n",
    "filter_return = {'type': 'filters.returns',\n",
    "                'groups': 'first',\n",
    "                'inputs':'merged',\n",
    "                'tag':'filtered'}\n",
    "filter_range = {'type': 'filters.range',\n",
    "                'limits': 'Z[:1.5)',\n",
    "                'inputs':'merged',\n",
    "                'tag':'ranged'}\n",
    "# write merged las to raster\n",
    "writers_gdal= {\"type\": \"writers.gdal\",\n",
    "               'output_type': 'max', \n",
    "              'resolution': '1.0',\n",
    "               'bounds': writers_bounds,\n",
    "               'inputs':'ranged',\n",
    "              'radius': '0.7',\n",
    "               'filename': output_tif}\n",
    "# Append each stage to a list prior to saving to json \n",
    "pipeline_list = filenames.copy()\n",
    "pipeline_list.append(filter_merge)\n",
    "pipeline_list.append(filter_range)\n",
    "pipeline_list.append(writers_gdal)\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "# save to json\n",
    "with open(CHM_json_LT1pt5, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "762b779c-8d65-4b9d-ab1b-240eb55fcbeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "tic = time.perf_counter()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c666f24-f4ec-419d-b056-b8e3c3557ab3",
   "metadata": {},
   "outputs": [],
   "source": [
    "pdal_cmd = ['pdal','pipeline', CHM_json_LT1pt5]\n",
    "subprocess.run(pdal_cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "872d681c-e0ab-4278-863d-04927bb403e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fa43794-5cd5-4afd-8a32-a06882b7d96a",
   "metadata": {},
   "outputs": [],
   "source": [
    "(toc-tic)/60"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8cf4ef0-37c4-4b64-bd85-1b7a448426e5",
   "metadata": {},
   "source": [
    "### Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b46a254-f10a-4643-9078-a15a67ac7d50",
   "metadata": {},
   "outputs": [],
   "source": [
    "litree_json = json_base_path+'litree.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c73688b4-dc87-440a-8e8d-4eccd64e3889",
   "metadata": {},
   "outputs": [],
   "source": [
    "readers_las = {'type':\"readers.las\",\n",
    "              \"filename\":'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/NAD83_NAD83_epoch2010/NCALM_SCB_2014_733000_4365000.las'}\n",
    "filter_ferry = {\"type\":\"filters.ferry\", \n",
    "                \"dimensions\":\"Z => HeightAboveGround\"}\n",
    "filter_sort = {\"type\":\"filters.sort\", \n",
    "               \"dimension\":\"HeightAboveGround\", \n",
    "               \"order\":\"DESC\"}\n",
    "filter_litree = {\"type\":\"filters.litree\"}\n",
    "writers_gdal= {\"type\": \"writers.gdal\",\n",
    "               'output_type': 'mean', \n",
    "              'resolution': '1.0',\n",
    "               'dimension':'ClusterID',\n",
    "              'radius': '0.7',\n",
    "               'filename': 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/canopy_metrics/veg_structure/cluster/test.tif'}\n",
    "pipeline_list = [readers_las, filter_ferry, filter_sort, filter_litree, writers_gdal]\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "# save to json\n",
    "with open(litree_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e231b35b-d820-4c9a-88f8-c1021e563d6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b304de03-0837-4a24-8847-82d093bd05b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "pdal_cmd = ['pdal','pipeline', litree_json]\n",
    "subprocess.run(pdal_cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fe69ddb-f958-430a-82a4-3c6005ea36fd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "385081df-634d-493f-8772-cdd0831c436e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster_json = json_base_path+'cluster_from_las.json' # define path to json files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d98de70-acbf-4676-b0cf-5b55a69b04e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_path = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/HAG/'\n",
    "output_tif = 'SCB/Sagehen_lidar/NCALM/NCALM_SCB_2014/canopy_metrics/veg_structure/NCALM_SCB_2014_cluster_test.tif'\n",
    "onlyfiles = [f for f in os.listdir(input_path) if os.path.isfile(os.path.join(input_path, f))]\n",
    "input_list = [input_path + s for s in onlyfiles]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23f18635-a4ec-4667-bcbc-bb904a6558bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_dict = {}\n",
    "tags = ['']*len(input_list)\n",
    "filenames = ['']*len(input_list)\n",
    "for i in range(len(input_list)):\n",
    "    filename_dict['filename_'+str(i)] = {'filename':input_list[i], 'tag':'A_'+str(i)}\n",
    "    tags[i] = 'A_'+str(i)\n",
    "    filenames[i] = filename_dict[list(filename_dict)[i]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "774e20ed-6499-428e-bcf5-9ca0af3fead6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge all las files or stages\n",
    "filter_merge = {\"type\":\"filters.merge\",\n",
    "               \"tag\": \"merged\",\n",
    "               \"inputs\": tags}\n",
    "filter_cluster = {'type': 'filters.cluster',\n",
    "                'inputs':'merged',\n",
    "                'tag':'cluster'}\n",
    "# write merged las to raster\n",
    "writers_gdal= {\"type\": \"writers.gdal\",\n",
    "               'output_type': 'mean', \n",
    "              'resolution': '1.0',\n",
    "               'bounds': writers_bounds,\n",
    "               'dimension':'ClusterID',\n",
    "              'radius': '0.7',\n",
    "               'filename': output_tif}\n",
    "# Append each stage to a list prior to saving to json \n",
    "pipeline_list = filenames.copy()\n",
    "pipeline_list.append(filter_merge)\n",
    "pipeline_list.append(filter_cluster)\n",
    "pipeline_list.append(writers_gdal)\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "# save to json\n",
    "with open(cluster_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db5b57d2-da70-452c-9893-5e3da6579dbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "pdal_cmd = ['pdal','pipeline', cluster_json]\n",
    "subprocess.run(pdal_cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9583ff2-6d95-449e-8b2e-c5680ccb3ac3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc8d819d-5251-4328-af62-30b8f41ca767",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "efe8463e-2c0e-4d0d-b2a7-1b132a8e1f4e",
   "metadata": {},
   "source": [
    "**Move to Raster workflow**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a658118-ad67-41a7-97e2-7d64913b8d80",
   "metadata": {},
   "source": [
    "# ------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f221a885-025a-44c5-a762-a9fdafc5a11f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "706a4c5e-29e3-4dce-93b0-3639e61925d6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c740714-5881-4763-bcb2-b2387741f477",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abbd18c8-82e0-4a86-b878-9a53710554f2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26833f00-aa67-426b-8e20-c1c3eff46862",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56715656-b594-4285-9b2a-5a875aaabd01",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5842dd22-6cf8-4215-9c3a-d548908e818f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "33e66bb1-b42b-445d-af25-de4b376c9652",
   "metadata": {},
   "source": [
    "# Extras"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0db123a-b3ee-4d86-b524-b6ead9a35931",
   "metadata": {},
   "source": [
    "## Rasterize\n",
    "Basic outline for rasterizing single file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c7e682f2-1de4-4c85-9f71-d3c524a4bfae",
   "metadata": {},
   "outputs": [],
   "source": [
    "rasterize_json = json_base_path+'rasterize_mean.json' # define path to json files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "53913992-ea01-43bc-8148-2d02d0dc635c",
   "metadata": {},
   "outputs": [],
   "source": [
    "readers_las = {\"type\":\"readers.las\"}\n",
    "# write merged las to raster\n",
    "writers_gdal= {\"type\": \"writers.gdal\",\n",
    "               'output_type': 'mean', \n",
    "              'resolution': '1.0',\n",
    "               'radius': '0.7',\n",
    "               'window_size':3}\n",
    "# Append each stage to a list prior to saving to json \n",
    "pipeline_list = [readers_las, writers_gdal]\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "# save to json\n",
    "with open(rasterize_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b164a1ab-0e58-40cb-bb6b-5a8b11536603",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_lid = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160326/vdatum_Test_2022/oldVDATUM_ASO_SCB_20160326_731777_4365426.laz'\n",
    "output_tif = 'SCB/Sagehen_lidar/ASO/ASO_SCB_20160326/vdatum_Test_2022/oldVDATUM_ASO_SCB_20160326_731777_4365426.tif'\n",
    "lidar_functions.rasterize_mean(input_lid,output_tif)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bf738b4-4eae-41dc-9cc8-898b177ac071",
   "metadata": {},
   "source": [
    "## File format conversions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be10eab7-1dc5-4561-9ff8-dbabb55abf57",
   "metadata": {},
   "source": [
    "### Convert .las to .txt\n",
    "see: https://pdal.io/stages/writers.text.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "486c745d-b984-41bc-a04a-b54dcfbbaf18",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "069497d0-a99b-4952-bfa2-3af3d3c0ef5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up json file commands\n",
    "output_txt = '/Volumes/cpiske/lidar_processing/python_scripts/PDAL/test_las/mcc_part_b_tile_004_000Test.asc'\n",
    "output_json = 'lidar_processing/python_scripts/PDAL_workflow/JSON/las_to_txt.json'\n",
    "\n",
    "# create a pipeline and save to a json file \n",
    "\n",
    "filter_dict = {'type':'readers.las',\n",
    "               'override_srs': \"EPSG:4326\",\n",
    "              'filename': input_las} # we are reading in a las file\n",
    "rasterize_dict = {'type':'writers.las',\n",
    "'format':'geojson',\n",
    "'order':'X,Y,Z',\n",
    "'keep_unspecified':'false',\n",
    "'filename':output_txt}\n",
    "\n",
    "\n",
    "pipeline_list = [filter_dict, rasterize_dict]\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "with open(output_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f570972-2f18-40d1-b4d2-06f0ed5ec7bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_path = 'lidar_processing/python_scripts/PDAL/JSON/las_to_txt.json'\n",
    "pdal_commands = ['pdal', 'pipeline', json_path]\n",
    "subprocess.run(pdal_commands)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "142a3a30-1ae6-481b-96ac-f95a1f6f4ddc",
   "metadata": {},
   "source": [
    "### .laz to .las"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efe33f9e-84de-49ea-a538-0cd37b7f7877",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# # set up json file commands\n",
    "# input_laz = 'lidar_processing/python_scripts/PDAL/test_las/ASO_USCAMB20180425f1a1_180425_1_dem_filter.laz'\n",
    "# output_las = 'lidar_processing/python_scripts/PDAL/test_las/ASO_USCAMB20180425f1a1_180425_1_dem_filter.las'\n",
    "# output_json = 'lidar_processing/python_scripts/PDAL/JSON/laz_to_las.json'\n",
    "\n",
    "# # create a pipeline and save to a json file \n",
    "\n",
    "# filter_dict = {'type':'readers.las',\n",
    "#                'filename': input_las} # we are reading in a las file\n",
    "# translate_dict = {'type':'writers.las',\n",
    "#                   \"a_srs\": \"EPSG:4326\",\n",
    "#                   'filename':output_las}\n",
    "\n",
    "\n",
    "# pipeline_list = [filter_dict, translate_dict]\n",
    "# pipeline_dict = {'pipeline' : pipeline_list}\n",
    "# with open(output_json, 'w') as out:\n",
    "#     json.dump(pipeline_dict, out, indent=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbb37c88-c5d9-4859-91ab-5c35e53bd514",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efe40151-4f3c-4fc8-acc0-ff2e092d788b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# json_path = 'lidar_processing/python_scripts/PDAL/JSON/laz_to_las.json'\n",
    "# pdal_commands = ['pdal', 'pipeline', json_path]\n",
    "# subprocess.run(pdal_commands)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea292cbb-156e-4be0-9fa8-62560bd55feb",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "# Raster Caluclations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66b90302-4ff2-461e-84b4-28f68e90575f",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in [0,1,2,3,4,5]:\n",
    "    apr_elev = 'lidar_processing/PDAL_testFiles_tutorials/test_las/SCB/rasterize/ASO_20160417/mcc_part_b_tile_004_00'+str(i)+'.tif'\n",
    "    may_elev = 'lidar_processing/PDAL_testFiles_tutorials/test_las/SCB/rasterize/ASO_20160518/mcc_part_b_tile_004_00'+str(i)+'.tif'\n",
    "    output = 'lidar_processing/PDAL_testFiles_tutorials/test_las/SCB/raster_subtract/mcc_part_b_tile_004_00'+str(i)+'.tif'\n",
    "    raster_sub = ['gdal_calc.py', '-a', apr_elev, '-b', may_elev, '--calc=\"a - b\"', '--outfile', output]\n",
    "    subprocess.run(raster_sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef95aabd-bd70-47a0-be17-993b89088961",
   "metadata": {},
   "source": [
    "# Theo Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "784a85fc-2834-4d0d-81a9-19f790b08678",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_las = 'SCB/kost_lidar_data/ASO_2016/2016_05_18/WGS84_G1762_to_NAD83_NAVD88/mcc_part_b_tile_004_000.las'\n",
    "\n",
    "z_min = 0.15\n",
    "z_max = 2\n",
    "z_range = 'Z[' + str(z_min) + ':' + str(z_max) + ']'\n",
    "output_raster = 'lidar_processing/python_scripts/PDAL/test_file/mcc_part_b_tile_004_000.tif'\n",
    "output_json = 'lidar_processing/python_scripts/PDAL/JSON/las_to_tif.json'\n",
    "resolution = 0.01\n",
    "\n",
    "filter_dict = {'type':'filters.range', 'limits':z_range}\n",
    "rasterize_dict = {'filename':output_raster,\n",
    "'gdaldriver':'GTiff',\n",
    "'output_type':'count',\n",
    "'resolution':resolution,\n",
    "'type': 'writers.gdal'}\n",
    "\n",
    "\n",
    "pipeline_list = [input_las, filter_dict, rasterize_dict]\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "with open(output_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1929f364-00c7-482e-8d5f-537652eed3cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_las = 'SCB/kost_lidar_data/ASO_2016/2016_05_18/WGS84_G1762_to_NAD83_NAVD88/mcc_part_b_tile_004_000.las'\n",
    "\n",
    "# z_min = 0.15\n",
    "# z_max = 2\n",
    "# z_range = 'Z[' + str(z_min) + ':' + str(z_max) + ']'\n",
    "output_raster = 'lidar_processing/python_scripts/PDAL/test_file/mcc_part_b_tile_004_000.asc'\n",
    "output_json = 'lidar_processing/python_scripts/PDAL/JSON/las_to_asc.json'\n",
    "resolution = 0.01\n",
    "\n",
    "filter_dict = {'type':'filters.range', 'limits':z_range}\n",
    "rasterize_dict = {'filename':output_raster,\n",
    "'gdaldriver':'XYZ',\n",
    "'output_type':'count',\n",
    "'resolution':resolution,\n",
    "'type': 'writers.gdal'}\n",
    "\n",
    "\n",
    "pipeline_list = [input_las, filter_dict, rasterize_dict]\n",
    "pipeline_dict = {'pipeline' : pipeline_list}\n",
    "with open(output_json, 'w') as out:\n",
    "    json.dump(pipeline_dict, out, indent=4)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a4f6338-edab-4c85-8941-b05d63584dd8",
   "metadata": {},
   "source": [
    "sudo docker run -v path_to_laz_folder:/input 0b pdal info /input/test.laz\n",
    "\n",
    "where path_to_laz_folder is the path to the LAS/LAZ file (you just need the folder path, not the file path).\n",
    "\n",
    ":/input is the new folder that will be created in your Docker container that will hold your point cloud.\n",
    "\n",
    "0b is just the image id of pdal\n",
    "\n",
    "/input/test.laz is the path to the point cloud in the Docker container.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a941783e-828f-4226-a25a-b772a51d12d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_laz_folder = 'lidar_processing/python_scripts/PDAL/test_las'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aee8f9b-8d85-490a-a3cd-22d4b06bb646",
   "metadata": {},
   "outputs": [],
   "source": [
    "sudo docker run -v path_to_laz_folder:/input 0b pdal info /input/test.laz\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac352849-3336-4cd7-b7d4-2f0c2a2c79fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import subprocess\n",
    "import pdal\n",
    "\n",
    "def assemblePipeline(input_las, list_of_dicts):\n",
    "    pipeline_list = [input_las]\n",
    "    pipeline_list.extend(list_of_dicts)\n",
    "    pipeline_dict = {'pipeline' : pipeline_list}\n",
    "    return pipeline_dict\n",
    "\n",
    "def makeHeightFilter(height, buffer):\n",
    "    z_min = height - buffer/2\n",
    "    z_max = height + buffer/2\n",
    "    z_range = 'Z[' + str(z_min) + ':' + str(z_max) + ']'\n",
    "    heightDict = {'type':'filters.range', 'limits':z_range}\n",
    "    return heightDict\n",
    "\n",
    "def makeRasterizeFilter(output_raster, resolution, epsg):\n",
    "    rasterize_dict = {'filename':output_raster,\n",
    "                      'gdaldriver':'GTiff',\n",
    "                      'output_type':'count',\n",
    "                      'resolution':resolution,\n",
    "                      'override_srs' : epsg,\n",
    "                      'type': 'writers.gdal'}\n",
    "    return rasterize_dict\n",
    "\n",
    "def convertTifForPIL(input_raster, output_raster, epsg):\n",
    "    ''' GDAL bindings are an alien concept to me, so I gave up and used\n",
    "        subprocess.'''\n",
    "    commands = ['gdal_translate', input_raster, output_raster, '-ot', 'Byte', '-a_srs', epsg]\n",
    "    subprocess.run(commands)\n",
    "\n",
    "\n",
    "def buildHeightSlice(input_las, height, buffer, output_raster, resolution, epsg, json_path=None):\n",
    "    filter_dict = makeHeightFilter(height, buffer)\n",
    "    rasterize_dict = makeRasterizeFilter(output_raster, resolution, epsg)\n",
    "    filter_list = [filter_dict, rasterize_dict]\n",
    "    pipeline_dict = assemblePipeline(input_las, filter_list)\n",
    "\n",
    "    if json_path is not None:\n",
    "        with open(json_path, 'w') as out:\n",
    "            json.dump(pipeline_dict, out, indent=4)\n",
    "        pdal_commands = ['pdal', 'pipeline', json_path]\n",
    "        subprocess.run(pdal_commands)\n",
    "    else:\n",
    "        pdal_commands = json.dumps(pipeline_dict)\n",
    "        pipeline = pdal.Pipeline(pdal_commands)\n",
    "        pipeline.execute()\n",
    "\n",
    "input_las = '/Users/theo/data/las/TLS_0244_20180612_01_v003_30m_clip_height_norm.las'\n",
    "height = 1.37\n",
    "buffer = 0.05\n",
    "z_min = height - buffer/2\n",
    "z_max = height + buffer/2\n",
    "z_range = 'Z[' + str(z_min) + ':' + str(z_max) + ']'\n",
    "temp_raster = '/Users/theo/Pictures/almost_cool.tif'\n",
    "final_raster = '/Users/theo/Pictures/cool.tif'\n",
    "resolution = 0.01\n",
    "epsg = 'EPSG:3310'\n",
    "\n",
    "buildHeightSlice(input_las, height, buffer, temp_raster, resolution, epsg)\n",
    "convertTifForPIL(temp_raster, final_raster, epsg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d188910-a9a7-4c6c-b164-8a163c1d5445",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import subprocess\n",
    "import pdal\n",
    "import argparse\n",
    "\n",
    "# Create flags for the user to utilize.\n",
    "parser = argparse.ArgumentParser(description=\"Generate JSON pipeline to generate DTM from a point cloud.\")\n",
    "      \n",
    "required = parser.add_argument_group('Required arguments')\n",
    "required.add_argument('-crs', '--coordinate_system', required=True, action='store', help=\"EPSG code.\")\n",
    "required.add_argument('-i', '--infile', required=True, action='store', help=\"Input path to point cloud\")\n",
    "required.add_argument('-o', '--outfile', required=True, action='store', help=\"Output path.\")\n",
    "args = parser.parse_args()\n",
    "\n",
    "def generateJSON(infile, list_of_dicts):\n",
    "    pipeline_list = [infile]\n",
    "    pipeline_list.extend(list_of_dicts)\n",
    "    pipeline_dict = {'pipeline': pipeline_list}\n",
    "    with open(\"pipeline.json\", 'w') as out:\n",
    "        json.dump(pipeline_dict, out, indent=4)\n",
    "\n",
    "def generateDTM(epsg, infile, outfile):\n",
    "    reproject_dict = {\"type\": \"filters.reprojection\",\n",
    "                      \"out_srs\": \"EPSG:{}\".format(epsg)}\n",
    "    reclassify_zero_dict = {\"type\": \"filters.assign\",\n",
    "                       \"assignment\": \"Classification[:]=0\"}\n",
    "    elm_dict = {\"type\": \"filters.elm\"}\n",
    "    outlier_dict = {\"type\": \"filters.outlier\"}\n",
    "    smrf_dict = {\"type\": \"filters.smrf\", \"ignore\": \"Classification[7:7]\",\n",
    "                 \"slope\": 0.2, \"window\": 16, \"threshold\": 0.45, \"scalar\": 1.2}\n",
    "    range_dict = {\"type\":\"filters.range\", \"limits\":\"Classification[2:2]\"}\n",
    "    output_dict = {\"filename\": outfile, \"gdaldriver\": \"GTiff\", \"output_type\": \"all\", \"resolution\": 0.01, \"type\": \"writers.gdal\"}\n",
    "    list_of_dicts = list([reproject_dict, reclassify_zero_dict, elm_dict, outlier_dict, smrf_dict, range_dict, output_dict])\n",
    "    generateJSON(infile, list_of_dicts)\n",
    "    pdal_cmds = ['pdal', 'pipeline', 'pipeline.json']\n",
    "    subprocess.run(pdal_cmds)\n",
    "    \n",
    "generateDTM(args.coordinate_system, args.infile, args.outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0efcfb16-ecfc-4ce2-8a5e-6bf2974c4510",
   "metadata": {},
   "source": [
    "# More helpful things"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84929bbc-a6c6-49ad-bfc5-9789fe5dd577",
   "metadata": {},
   "source": [
    "### Get stats of a dataset\n",
    "see: https://www.spatialised.net/lidar-qa-with-pdal-part-1/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa7713f5-fbfd-412c-8488-04489fa868db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# name JSON file\n",
    "\n",
    "stats_json = 'lidar_processing/python_scripts/PDAL_workflow/JSON/stats.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cc3f4a2-adf9-4d2e-a62e-a2a538437396",
   "metadata": {},
   "outputs": [],
   "source": [
    "reader_las = {\"type\":\"readers.las\",\n",
    "              \"filename\": input_las_stats}\n",
    "filter_stats = {\"type\":\"filters.stats\",\n",
    "                \"dimensions\":\"Z\",\n",
    "                \"global\":\"Z\",\n",
    "                \"advanced\":\"true\"}\n",
    "pipeline_list = [reader_las, filter_stats]\n",
    "#pipeline_dict = {reader_las, filter_stats}\n",
    "# with open(stats_json, 'w') as out:\n",
    "#     json.dump(pipeline_dict, out, indent=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ff5cfca-34a5-471d-8b40-bef477ef6438",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94a58e53-92c7-47c2-b75b-f9c9466b9e2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = pdal.Pipeline(json.dumps(pipeline_list))\n",
    "pipeline.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "722c7605-47f2-4698-bec0-f2aa20a26f4a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "json.loads(pipeline.metadata)[\"metadata\"][\"filters.stats\"][\"statistic\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c67b19a-0d9f-4b10-ad08-ccf9593fa659",
   "metadata": {},
   "source": [
    "## Navigating folders/files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67984138-2046-46f9-9187-26121ac09dca",
   "metadata": {},
   "source": [
    "**create list of files/folders with a wildcard (*)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8b23dd1-01e1-46d4-81e7-b9e5be2df38b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ex. list all files in folder4 that end in .laz = folder1/folder2/folder3/folder4/*.laz\n",
    "# ex. list all folders named folder3 in folder 1 = foler1/*/folder3\n",
    "# ex. list all list all contents in folder2 = folder2/* - note just folder 2, no subdirectories\n",
    "glob_cmd = 'path'\n",
    "glob_exe = glob.glob(file_glob_cmd)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "124af7f3-340e-4a3d-8b74-fbff724635f2",
   "metadata": {},
   "source": [
    "**create a list with all directories/subdirectories on a path**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "316d4080-2f26-48b8-b908-992955d0200f",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_folders = [x[0] for x in os.walk('path')]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ddee80f-1c7f-46e1-a8e8-2cee82ac6f02",
   "metadata": {},
   "source": [
    "**create a list with all directories/subdirectories on a path with specific folders**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fb7ef3c-3534-4566-9a4d-63ce9bcd074d",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_folders = [x[0] for x in os.walk('path')]\n",
    "index_pos_list = [ i for i in range(len(all_folders)) if all_folders[i][-6:] == 'retile' ] # must change this to meet requirements\n",
    "full_list = [all_folders[i] for i in index_pos_list]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "507fb6c4-a80d-4b97-8533-1852b3a8e992",
   "metadata": {},
   "source": [
    "**get name of the directory just above one listed**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea8cc0c9-4dcb-44e7-a72f-8586c6828636",
   "metadata": {},
   "outputs": [],
   "source": [
    "subdirname = os.path.basename(os.path.dirname('path'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "141720ac-a112-4891-b727-3e4999ed4298",
   "metadata": {},
   "source": [
    "**create a list with only filenames**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5412acaf-9f08-430c-8ce2-ca0027d4d129",
   "metadata": {},
   "outputs": [],
   "source": [
    "onlyfiles = [f for f in os.listdir('path') if os.path.isfile(os.path.join('path', f))]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d80f4b2-ebb0-4eb2-9a39-58d554be97b0",
   "metadata": {},
   "source": [
    "**create a list with full file paths**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfc96133-4087-4aca-a90b-302487a803e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "full_path = ['path' + '/' + s for s in onlyfiles]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b8a6064-f348-405d-bf7d-5e8c8f7bfd81",
   "metadata": {},
   "source": [
    "## Parallel Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "215beb4e-46bb-4271-b452-6849510f3a36",
   "metadata": {},
   "outputs": [],
   "source": [
    "# time = 142.5\n",
    "tic = time.perf_counter()\n",
    "list(map(function, args));\n",
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a4ca6cc-9fb5-48ce-8146-b41150dd9674",
   "metadata": {},
   "outputs": [],
   "source": [
    "# time = 53 s\n",
    "tic = time.perf_counter()\n",
    "if __name__ == \"__main__\":\n",
    "    pool = Pool(3)\n",
    "    pool.map(function, arg)\n",
    "    pool.close()\n",
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "920e3ba6-7d5f-4a00-b563-ee2a53074b87",
   "metadata": {},
   "outputs": [],
   "source": [
    "# time = very fast? .06s\n",
    "tic = time.perf_counter()\n",
    "if __name__ == \"__main__\":\n",
    "    executor = concurrent.futures.ThreadPoolExecutor(max_workers=3)\n",
    "    executor.map(function, arg)\n",
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb4c6986-5a20-472a-a5bd-4a854161ff30",
   "metadata": {},
   "outputs": [],
   "source": [
    "# time = 0.22\n",
    "tic = time.perf_counter()\n",
    "if __name__ == \"__main__\":\n",
    "    executor = concurrent.futures.ProcessPoolExecutor(max_workers=3)\n",
    "    executor.map(function, arg)\n",
    "toc = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1f97306-b0c1-4b9d-8b1e-5bd95d1732c6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}