2/24/2019

OpticalWaveGauging · Feb 25, 2019 · 6f89913 · 6f89913
1 parent 2b48131
commit 6f89913
Show file tree

Hide file tree

Showing 17 changed files with 4,482 additions and 50 deletions.
diff --git a/README.md b/README.md
@@ -1,46 +1,90 @@
-### About
+## About
 
-Data and code to implement Buscombe et al (2019) optical wave gauging using deep neural networks, detailed in the paper
+Data and code to implement Buscombe et al (2019) optical wave gauging (OWG) using deep neural networks, detailed in the paper:
 
 > Buscombe, Carini, Harrison, Chickadel, and Warrick (in review) Optical wave gauging with deep neural networks. Submitted to Coastal Engineering 
 
 
-Software and data for training deep convolutional neural network models to estimate wave height and wave period from the same imagery
+Software and data for training deep convolutional neural network models to estimate wave height and wave period from surf zone imagery
 
 ### Folder structure
 
 * \conda_env contains yml files for setting up a conda environment
 * \conf contains the configuration file with user-definable settings
 * \train contains files using for training models 
+* \im128 is a file structure that will contain results from model training
 
-### Setting up computing environments
+## Setting up computing environments
 
-First, some conda housekeeping
+### Install Anaconda pyton distribution
 
+Install the latest version of Anaconda (https://www.anaconda.com/distribution/)
+
+When installed, open an anaconda command prompt
+
+### Get this github repository
+
+Use git to clone the github directory
+
+``
+git clone [email protected]:dbuscombe-usgs/OpticalWaveGauging_DNN.git
 ```
-conda clean --packages
-conda update -n base conda
+
+navigate to the ```OpticalWaveGauging_DNN``` directory
+
+```
+cd OpticalWaveGauging_DNN
 ```
 
 It is strongly recommended that you use a GPU-enabled tensorflow installation. CPU training of a model can take several hours to several days (more likely the latter). However, the following instructions are for a CPU install. To use gpu, replace ```tensorflow``` with ```tensorflow-gpu``` in ```conda_env/owg.yml```
 
+### Create a conda virtual environment
+
 Create a new conda environment called ```owg```
 
 ```
 conda env create -f conda_env/owg.yml
 ```
 
-Activate environment:
+This takes a few minutes. When it is done, activate environment:
 
 ```
 conda activate owg
 ```
 
-Install ```pillow``` using ```pip``` (because the conda version was incompatible with conda-installed ```tensorflow```, at least at time of writing)
+### Keras source code modifications
+
+1. go to the keras_applications folder. On Windows anaconda builds this is typically located here:
+
+```C:\Users\yourusername\AppData\Local\Continuum\anaconda3\envs\owg\Lib\site-packages\keras_applications```
+
+
+2. Zip up all the .py files. Call the zipped file 'orig.zip'. Delete the .py files.
+
+3. Then copy the .py files in conda_env\keras_applications to this directory
+
+
+Finally, add the following code to the ```_init_.py``` file in the keras\applications folder. 
 
 ```
-pip install pillow
-```
+def keras_modules_injection(base_fun):
+
+    def wrapper(*args, **kwargs):
+        if hasattr(keras_applications, 'get_submodules_from_kwargs'):
+            kwargs['backend'] = backend
+            kwargs['layers'] = layers
+            kwargs['models'] = models
+            kwargs['utils'] = utils
+        return base_fun(*args, **kwargs)
+
+    return wrapper
+```	
+
+(if it is not already there)
+
+On Windows anaconda builds this is typically located here:
+
+```C:\Users\yourusername\AppData\Local\Continuum\anaconda3\envs\owg\Lib\site-packages\keras\applications```
 
 
 ## Setting up the model 
@@ -131,7 +175,7 @@ To train OWGs for wave period, change the category in the config file to 'T' and
 
 ## Tidying up
 
-Organize model result files (*.hdf5 format) in the following file structure
+Model result files (*.hdf5 format) are organized in the following file structure
 
 im128
 

diff --git a/compile_results.py b/compile_results.py
@@ -12,7 +12,7 @@
 import matplotlib.pyplot as plt 
 from glob import glob
 import shutil
-
+import json
 from keras.applications.mobilenet import MobileNet
 from keras.applications.mobilenetv2 import MobileNetV2
 from keras.applications.inception_v3 import InceptionV3
@@ -157,12 +157,12 @@
 	out['div'] = div
 	out['text_X'] = test_X
 	out['text_Y'] = test_Y
-	out['df_T'] = df['waveperiod']
-	out['df_H'] = df['waveheight']
+	out['df_T'] = df['T']
+	out['df_H'] = df['H']
 	out['df_zscore'] = df['zscore']
 	out['df_category'] = [] 
 
-	root = os.getcwd()+os.sep+'im128'+os.sep+'res_snap'
+	root = os.getcwd()+os.sep+'im128'+os.sep+'res'
 
 
 	for epics in [num_epochs]: ##[20, 50, 100]:
@@ -208,7 +208,6 @@
 
 				out['yhat_epochs'+str(epics)+'_batchsize'+str(batch_size)+'_model'+arch] = div*owg.predict(test_X, batch_size = batch_size, verbose = True)+mean
 				counter+=1
-				os.remove(weight_path)
 
 	E = []			
 	for epics in [num_epochs]: ##20, 50, 100]:
@@ -246,12 +245,12 @@
 				axs[counter1, counter2].plot(test_Y, test_Y, 'r--', label = 'observed', lw=0.5)
 				if counter1==3:
 				   if counter2==0:
-					  if category=='H':
-						axs[counter1, counter2].set_xlabel('Observed H (m)', fontsize=6)
-						axs[counter1, counter2].set_ylabel(r'Estimated H (m)', fontsize=6)				  
-					  else:
-						axs[counter1, counter2].set_xlabel('Observed T (s)', fontsize=6)
-						axs[counter1, counter2].set_ylabel(r'Estimated T (s)', fontsize=6)
+				      if category=='H':
+				         axs[counter1, counter2].set_xlabel('Observed H (m)', fontsize=6)
+				         axs[counter1, counter2].set_ylabel(r'Estimated H (m)', fontsize=6)				  
+				      else:
+				         axs[counter1, counter2].set_xlabel('Observed T (s)', fontsize=6)
+				         axs[counter1, counter2].set_ylabel(r'Estimated T (s)', fontsize=6)
 				if category=='H':
 					string = r'RMS (m): '+str(np.sqrt(np.nanmean((pred_Y - test_Y)**2)))[:4] + '  R$^2$: '+str(np.min(np.corrcoef(test_Y, pred_Y))**2)[:4]
 					axs[counter1, counter2].set_title(letters[counter2]+') '+string, fontsize=4, loc='left')

diff --git a/conda_env/keras_applications/__init__.py b/conda_env/keras_applications/__init__.py
@@ -0,0 +1,106 @@
+"""Enables dynamic setting of underlying Keras module.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+_KERAS_BACKEND = None
+_KERAS_LAYERS = None
+_KERAS_MODELS = None
+_KERAS_UTILS = None
+
+def keras_modules_injection(base_fun):
+
+    def wrapper(*args, **kwargs):
+        if hasattr(keras_applications, 'get_submodules_from_kwargs'):
+            kwargs['backend'] = backend
+            kwargs['layers'] = layers
+            kwargs['models'] = models
+            kwargs['utils'] = utils
+        return base_fun(*args, **kwargs)
+
+    return wrapper
+
+def set_keras_submodules(backend=None,
+                         layers=None,
+                         models=None,
+                         utils=None,
+                         engine=None):
+    # Deprecated, will be removed in the future.
+    global _KERAS_BACKEND
+    global _KERAS_LAYERS
+    global _KERAS_MODELS
+    global _KERAS_UTILS
+    _KERAS_BACKEND = backend
+    _KERAS_LAYERS = layers
+    _KERAS_MODELS = models
+    _KERAS_UTILS = utils
+
+
+def get_keras_submodule(name):
+    # Deprecated, will be removed in the future.
+    if name not in {'backend', 'layers', 'models', 'utils'}:
+        raise ImportError(
+            'Can only retrieve one of "backend", '
+            '"layers", "models", or "utils". '
+            'Requested: %s' % name)
+    if _KERAS_BACKEND is None:
+        raise ImportError('You need to first `import keras` '
+                          'in order to use `keras_applications`. '
+                          'For instance, you can do:\n\n'
+                          '```\n'
+                          'import keras\n'
+                          'from keras_applications import vgg16\n'
+                          '```\n\n'
+                          'Or, preferably, this equivalent formulation:\n\n'
+                          '```\n'
+                          'from keras import applications\n'
+                          '```\n')
+    if name == 'backend':
+        return _KERAS_BACKEND
+    elif name == 'layers':
+        return _KERAS_LAYERS
+    elif name == 'models':
+        return _KERAS_MODELS
+    elif name == 'utils':
+        return _KERAS_UTILS
+
+
+def get_submodules_from_kwargs(kwargs):
+    backend = kwargs.get('backend', _KERAS_BACKEND)
+    layers = kwargs.get('layers', _KERAS_LAYERS)
+    models = kwargs.get('models', _KERAS_MODELS)
+    utils = kwargs.get('utils', _KERAS_UTILS)
+    for key in kwargs.keys():
+        if key not in ['backend', 'layers', 'models', 'utils']:
+            raise TypeError('Invalid keyword argument: %s', key)
+    return backend, layers, models, utils
+
+
+def correct_pad(backend, inputs, kernel_size):
+    """Returns a tuple for zero-padding for 2D convolution with downsampling.
+
+    # Arguments
+        input_size: An integer or tuple/list of 2 integers.
+        kernel_size: An integer or tuple/list of 2 integers.
+
+    # Returns
+        A tuple.
+    """
+    img_dim = 2 if backend.image_data_format() == 'channels_first' else 1
+    input_size = backend.int_shape(inputs)[img_dim:(img_dim + 2)]
+
+    if isinstance(kernel_size, int):
+        kernel_size = (kernel_size, kernel_size)
+
+    if input_size[0] is None:
+        adjust = (1, 1)
+    else:
+        adjust = (1 - input_size[0] % 2, 1 - input_size[1] % 2)
+
+    correct = (kernel_size[0] // 2, kernel_size[1] // 2)
+
+    return ((correct[0] - adjust[0], correct[0]),
+            (correct[1] - adjust[1], correct[1]))
+
+__version__ = '1.0.6'