updates prepping for next version

lauetoolsnn/end_to_end_scripts/LaueNN_pyScript_1_or_2phase.py

@@ -159,9 +159,9 @@
                     }
 
     generate_dataset = True
-    train_network = True
-    write_config_GUI = True
-    run_prediction = True
+    train_network = False
+    write_config_GUI = False
+    run_prediction = False
     prediction_GUI = False
     # =============================================================================
     # END OF USER INPUT
@@ -314,11 +314,20 @@
         SG1 = input_params["SG"][1]
 
         ## read hkl information from a fit file in case too large HKLs
-        manual_hkl_list=False
+        manual_hkl_list=True
         if manual_hkl_list:
+            ##read from a text file (fit file)
             import numpy as np
-            temp = np.loadtxt(r"img_0000_LT_1.fit")
-            hkl_array = temp[:,2:5]
+            # temp = np.loadtxt(r"img_0000_LT_1.fit")
+            # hkl_array = temp[:,2:5]
+            # hkl_array1 = None
+
+            ## or define your own list of hkls
+            import lauetoolsnn.lauetools.generaltools as GT
+            hkl_array = GT.threeindices_up_to(5)
+            hkl_max_index = np.max(np.abs(hkl_array), axis=1)
+            filter_cond = (hkl_max_index >= 3) * (hkl_max_index <= 4)
+            hkl_array = hkl_array[filter_cond]
             hkl_array1 = None
         else:
             hkl_array = None

lauetoolsnn/lauetools/dict_LaueTools.py

@@ -77,8 +77,8 @@ def get_extinct_mat():
 dict_CCD = {
     "MARCCD165": ((2048, 2048), 0.079142, 65535, "no", 4096, "uint16", "MAR Research 165 mm now rayonix", "mccd", ),
     "sCMOS": [(2018, 2016), 0.0734, 65535, "no", 3828, "uint16", "file as produced by sCMOS camera with checked fliplr transform. CCD parameters read from tif header by fabio", "tif"],
-    "cor1": [(2018, 2016), 0.0734, 65535, "no", 3828, "uint16", "file as produced by sCMOS camera with checked fliplr transform. CCD parameters read from tif header by fabio", "tif"],
-    "cor": [(2 * 2018, 2 * 2016), 0.0734 / 2.0, 65535, "no", 3828, "uint16", "binned 1x1, CCD parameters binned 1x1 read from tif header by fabio ", "tif"],
+    "cor": [(2018, 2016), 0.0734, 65535, "no", 3828, "uint16", "file as produced by sCMOS camera with checked fliplr transform. CCD parameters read from tif header by fabio", "cor"],
+    "cor_16M": [(2 * 2018, 2 * 2016), 0.0734 / 2.0, 65535, "no", 3828, "uint16", "binned 1x1, CCD parameters binned 1x1 read from tif header by fabio ", "cor"],
     "sCMOS_fliplr": [(2018, 2016), 0.0734, 65535, "sCMOS_fliplr", 3828, "uint16", "binned 2x2, CCD parameters read from tif header by fabio", "tif"],
     "sCMOS_fliplr_16M": [(2 * 2018, 2 * 2016), 0.0734 / 2.0, 65535, "sCMOS_fliplr", 3828, "uint16", "binned 1x1, CCD parameters binned 1x1 read from tif header by fabio ", "tif"],
     "sCMOS_16M": [(2 * 2018, 2 * 2016), 0.0734 / 2.0, 65535, "no", 3828, "uint16", "binned 1x1, CCD parameters binned 1x1 read from tif header by fabio ", "tif"],

lauetoolsnn/lauetoolsneuralnetwork.py

@@ -149,7 +149,6 @@ def resource_path(relative_path, verbose=0):
         print("Base path of the library: ",base_path)
     return os.path.join(base_path, relative_path)
 
-
 try:
     import pkg_resources  # part of setuptools
     version_package = pkg_resources.require("lauetoolsnn")[0].version
@@ -159,7 +158,6 @@ def resource_path(relative_path, verbose=0):
     latest_version = versions("lauetoolsnn")[0]
 
 frame_title = "Laue Neural-Network model- v3 @Ravi @Jean-Sebastien \n@author: Ravi raj purohit PURUSHOTTAM RAJ PUROHIT ([email protected]) \n@guide: Jean-Sebastien MICHA ([email protected])"
-
 Logo = resource_path("lauetoolsnn_logo_bXM_2.png",  verbose=0)
 Logo_splash = resource_path("lauetoolsnn_splash_bXM_2.png",  verbose=0)
 gif_path = resource_path("frames_medres.gif",  verbose=0)
@@ -171,7 +169,7 @@ def resource_path(relative_path, verbose=0):
     material1_global = "Si" ## same key as used in LaueTools
     symmetry1_global = "cubic"
     prefix_global = ""
-    detectorparameters_global = [79.50900, 977.9000, 931.8900, 0.3570000, 0.4370000]
+    detectorparameters_global = [79.51, 977.9, 931.9, 0.36, 0.44]
     pixelsize_global = 0.0734 # 0.079142 #
     ccd_label_global = "sCMOS" #"MARCCD165" #"Cor"#
     dim1_global = 2018 #2048 #

lauetoolsnn/misc_scripts/check_classHKL_function.py

@@ -0,0 +1,99 @@
+# coding: utf-8
+"""
+Created on June 18 06:54:04 2021
+
+Check generate_classHKL function
+@author: Ravi raj purohit PURUSHOTTAM RAJ PUROHIT ([email protected])
+
+"""
+__author__ = "Ravi raj purohit PURUSHOTTAM RAJ PUROHIT, CRG-IF BM32 @ ESRF"
+
+if __name__ == '__main__':     #enclosing required because of multiprocessing
+
+    ## Import modules used for this Notebook
+    import os
+
+    # =============================================================================
+    # Step 0: Define the dictionary with all parameters 
+    # =============================================================================
+    ## User Input dictionary with parameters
+    ## In case of only one phase/material, keep same value for material_ and material1_ key
+    input_params = {
+                    "global_path" : r"C:\Users\purushot\Desktop\LaueNN_script",
+                    "prefix" : "_2phase",                 ## prefix for the folder to be created for training dataset
+
+                    "material_": ["GaN", "Si"],             ## same key as used in dict_LaueTools
+                    "symmetry": ["hexagonal", "cubic"],           ## crystal symmetry of material_
+                    "SG": [184, 227], #186                    ## Space group of material_ (None if not known)
+                    "hkl_max_identify" : [6,5],        ## Maximum hkl index to classify in a Laue pattern
+                    "nb_grains_per_lp_mat" : [2,1],        ## max grains to be generated in a Laue Image
+
+                    ## hkl_max_identify : can be "auto" or integer: Maximum index of HKL to build output classes
+
+                    # =============================================================================
+                    # ## Data generation settings
+                    # =============================================================================
+                    "grains_nb_simulate" : 500,    ## Number of orientations to generate (takes advantage of crystal symmetry)
+                    "classes_with_frequency_to_remove": [100,100], ## classes_with_frequency_to_remove: HKL class with less appearance than 
+                                                                            ##  specified will be ignored in output
+                    "desired_classes_output": ["all","all"], ## desired_classes_output : can be all or an integer: to limit the number of output classes
+
+                    "include_small_misorientation": False, ## to include additional data with small angle misorientation
+                    "misorientation": 5, ##only used if "include_small_misorientation" is True
+                    "maximum_angle_to_search":20, ## Angle of radial distribution to reconstruct the histogram (in deg)
+                    "step_for_binning" : 0.1,      ## bin widht of angular radial distribution in degree
+                    }
+
+    # =============================================================================
+    # Step 1 
+    # =============================================================================
+    ## if LaueToolsNN is properly installed
+    from lauetoolsnn.utils_lauenn import get_material_detail, generate_classHKL
+    # ## Get material parameters 
+    # ### Generates a folder with material name and gets material unit cell parameters 
+    # ### and symmetry object from the get_material_detail function
+    material_= input_params["material_"][0]
+    material1_= input_params["material_"][1]
+    n = input_params["hkl_max_identify"][0]
+    n1 = input_params["hkl_max_identify"][1]
+    maximum_angle_to_search = input_params["maximum_angle_to_search"]
+    step_for_binning = input_params["step_for_binning"]
+    symm_ = input_params["symmetry"][0]
+    symm1_ = input_params["symmetry"][1]
+    SG = input_params["SG"][0]
+    SG1 = input_params["SG"][1]
+
+    ## read hkl information from a fit file in case too large HKLs
+    manual_hkl_list=False
+    if manual_hkl_list:
+        import numpy as np
+        temp = np.loadtxt(r"img_0000_LT_1.fit")
+        hkl_array = temp[:,2:5]
+        hkl_array1 = None
+    else:
+        hkl_array = None
+        hkl_array1 = None
+
+    if material_ != material1_:
+        save_directory = os.path.join(input_params["global_path"],material_+"_"+material1_+input_params["prefix"])
+    else:
+        save_directory = os.path.join(input_params["global_path"],material_+input_params["prefix"])
+    print("save directory is : "+save_directory)
+    if not os.path.exists(save_directory):
+        os.makedirs(save_directory)
+
+    ## get unit cell parameters and other details required for simulating Laue patterns
+    rules, symmetry, lattice_material,\
+        crystal, SG, rules1, symmetry1,\
+        lattice_material1, crystal1, SG1 = get_material_detail(material_, SG, symm_,
+                                                               material1_, SG1, symm1_)
+
+    # ## Generate Neural network output classes (Laue spot hkls) using the generate_classHKL function
+    ## procedure for generation of GROUND TRUTH classes
+    # general_diff_cond = True will eliminate the hkl index that does not 
+    # satisfy the general reflection conditions, otherwise they will be eliminated in the next stage
+    generate_classHKL(n, rules, lattice_material, symmetry, material_, 
+                      crystal=crystal, SG=SG, general_diff_cond=False,
+                      save_directory=save_directory, write_to_console=print, 
+                      ang_maxx = maximum_angle_to_search, 
+                      step = step_for_binning, mat_listHKl=hkl_array)

lauetoolsnn/utils_lauenn.py

@@ -32,7 +32,6 @@
 # matplotlib.use('Qt5Agg')
 # matplotlib.rcParams.update({'font.size': 14})
 from mpl_toolkits.axes_grid1 import make_axes_locatable
-
 import numpy as np
 from math import acos
 import time
@@ -44,10 +43,8 @@
 # from scipy.spatial.transform import Rotation as R
 import _pickle as cPickle
 import configparser
-
 from multiprocessing import Process, Queue, cpu_count
 from tqdm import tqdm
-
 # from skimage.transform import (hough_line, hough_line_peaks)
 # =============================================================================
 # Additonal networkx module
@@ -105,7 +102,6 @@
     tensorflow_keras = False
 
 # import h5py
-
 ## GPU Nvidia drivers needs to be installed! Ughh
 ## if wish to use only CPU set the value to -1 else set it to 0 for GPU
 ## CPU training is suggested (as the model requires more RAM)
@@ -480,7 +476,6 @@ def array_generator(path_, batch_size, n_classes, loc_new, write_to_console=None
             obj = np.load(array_path)
             trainX = obj["arr_0"]
             loc1 = obj["arr_1"]
-
             if len(trainX) == 0 or len(loc1) == 0:
                 if write_to_console != None:
                     write_to_console("Skipping File: "+ array_path+"; No data is found")
@@ -496,21 +491,17 @@ def array_generator(path_, batch_size, n_classes, loc_new, write_to_console=None
                     loc1_new.append(temp_loc)
                 else:
                     loc1_new_rmv.append(k)   
-
             loc1_new = np.array(loc1_new).ravel()
             loc1_new_rmv = np.array(loc1_new_rmv).ravel() 
-
             if len(trainX) != len(loc1_new):
                 if len(loc1_new_rmv) > 0:
                     trainX = np.delete(trainX, loc1_new_rmv, axis=0) 
-
             if bs == 0 or temp_var:
                 trainX1 = np.copy(trainX)
                 trainY1 = np.copy(loc1_new)
             else:
                 trainX1 = np.vstack((trainX1, trainX))
                 trainY1 = np.hstack((trainY1, loc1_new))
-
         ## To normalize the size of one hot encoding
         count = 0
         if np.min(trainY1) != 0:
@@ -519,14 +510,12 @@ def array_generator(path_, batch_size, n_classes, loc_new, write_to_console=None
         if np.max(trainY1) != (n_classes-1):
             trainY1 = np.append(trainY1, n_classes-1)
             count += 1
-
         if tocategorical:
             trainY1 = to_categorical(trainY1)
         if count == 1:
             trainY1 = np.delete(trainY1, [len(trainY1)-1] ,axis=0)
         elif count == 2:
             trainY1 = np.delete(trainY1, [len(trainY1)-1,len(trainY1)-2] ,axis=0)
-
         yield trainX1, trainY1
 
 def vali_array(path_, batch_size, n_classes, loc_new, write_to_console=None, tocategorical=True):
@@ -818,18 +807,20 @@ def generate_classHKL(n, rules, lattice_material, symmetry, material_, crystal=N
         list with hkl's removed from the dictionary
     n : int
         Max index hkl
-
     """
-
-    temp_ = GT.threeindices_up_to(int(n))
-
+    ##add user defined categories of hkl to be used (non sequential also)
+    ## add directly to the temp_
     if np.all(mat_listHKl != None):
-        mat_listHKl = np.array(mat_listHKl)
-        temp_ = np.vstack((temp_, mat_listHKl))
+        temp_ = mat_listHKl
+    else:
+        temp_ = GT.threeindices_up_to(int(n))
+
+    # if np.all(mat_listHKl != None):
+    #     mat_listHKl = np.array(mat_listHKl)
+    #     temp_ = np.vstack((temp_, mat_listHKl))
 
     classhkl_ = temp_
 
-
     if general_diff_cond:
         classhkl_ = crystal.hkl_allowed_array(classhkl_)
 
@@ -841,7 +832,6 @@ def generate_classHKL(n, rules, lattice_material, symmetry, material_, crystal=N
         progress.setMaximum(len(classhkl_))
 
     hkl_all = {}
-    # another_method = False
     for i in range(len(classhkl_)):
         new_hkl = classhkl_[i,:]
 
@@ -916,8 +906,6 @@ def generate_classHKL(n, rules, lattice_material, symmetry, material_, crystal=N
     list_remove = []
     for i, j in enumerate(codebars):
         for k, l in enumerate(codebars):
-            # if i in list_appended and k in list_appended:
-            #     continue
             if i != k and np.all(j == l):
                 if keys_rmv[i] in list_remove or keys_rmv[k] in list_remove:
                     if write_to_console !=None:
@@ -1084,17 +1072,13 @@ def write_training_testing_dataMTEX(save_directory,material_, material1_, lattic
 def get_material_data(material_="Cu", ang_maxx = 45, step = 0.5, hkl_ref=13, classhkl = None):
     a, b, c, alpha, beta, gamma = dictLT.dict_Materials[material_][1]
     Gstar = CP.Gstar_from_directlatticeparams(a, b, c, alpha, beta, gamma)
-    rules = dictLT.dict_Materials[material_][-1]
-
-    hkl2 = GT.threeindices_up_to(int(hkl_ref))
-    hkl2 = CP.ApplyExtinctionrules(hkl2,rules)
-    hkl2 = hkl2.astype(np.int16)
 
     query_angle = ang_maxx/2.
     angle_tol = ang_maxx/2.
     metrics = Gstar
 
-    hkl1 = classhkl
+    hkl1 = np.copy(classhkl)
+    hkl2 = np.copy(classhkl)
     H1 = hkl1
     n1 = hkl1.shape[0]
     H2 = hkl2