diff --git a/lauetoolsnn/end_to_end_scripts/LaueNN_pyScript_1_or_2phase.py b/lauetoolsnn/end_to_end_scripts/LaueNN_pyScript_1_or_2phase.py index 9acaf58..b688496 100644 --- a/lauetoolsnn/end_to_end_scripts/LaueNN_pyScript_1_or_2phase.py +++ b/lauetoolsnn/end_to_end_scripts/LaueNN_pyScript_1_or_2phase.py @@ -159,9 +159,9 @@ } generate_dataset = True - train_network = True - write_config_GUI = True - run_prediction = True + train_network = False + write_config_GUI = False + run_prediction = False prediction_GUI = False # ============================================================================= # END OF USER INPUT @@ -314,11 +314,20 @@ SG1 = input_params["SG"][1] ## read hkl information from a fit file in case too large HKLs - manual_hkl_list=False + manual_hkl_list=True if manual_hkl_list: + ##read from a text file (fit file) import numpy as np - temp = np.loadtxt(r"img_0000_LT_1.fit") - hkl_array = temp[:,2:5] + # temp = np.loadtxt(r"img_0000_LT_1.fit") + # hkl_array = temp[:,2:5] + # hkl_array1 = None + + ## or define your own list of hkls + import lauetoolsnn.lauetools.generaltools as GT + hkl_array = GT.threeindices_up_to(5) + hkl_max_index = np.max(np.abs(hkl_array), axis=1) + filter_cond = (hkl_max_index >= 3) * (hkl_max_index <= 4) + hkl_array = hkl_array[filter_cond] hkl_array1 = None else: hkl_array = None diff --git a/lauetoolsnn/lauetools/dict_LaueTools.py b/lauetoolsnn/lauetools/dict_LaueTools.py index c9f1b83..0895f14 100644 --- a/lauetoolsnn/lauetools/dict_LaueTools.py +++ b/lauetoolsnn/lauetools/dict_LaueTools.py @@ -77,8 +77,8 @@ def get_extinct_mat(): dict_CCD = { "MARCCD165": ((2048, 2048), 0.079142, 65535, "no", 4096, "uint16", "MAR Research 165 mm now rayonix", "mccd", ), "sCMOS": [(2018, 2016), 0.0734, 65535, "no", 3828, "uint16", "file as produced by sCMOS camera with checked fliplr transform. CCD parameters read from tif header by fabio", "tif"], - "cor1": [(2018, 2016), 0.0734, 65535, "no", 3828, "uint16", "file as produced by sCMOS camera with checked fliplr transform. CCD parameters read from tif header by fabio", "tif"], - "cor": [(2 * 2018, 2 * 2016), 0.0734 / 2.0, 65535, "no", 3828, "uint16", "binned 1x1, CCD parameters binned 1x1 read from tif header by fabio ", "tif"], + "cor": [(2018, 2016), 0.0734, 65535, "no", 3828, "uint16", "file as produced by sCMOS camera with checked fliplr transform. CCD parameters read from tif header by fabio", "cor"], + "cor_16M": [(2 * 2018, 2 * 2016), 0.0734 / 2.0, 65535, "no", 3828, "uint16", "binned 1x1, CCD parameters binned 1x1 read from tif header by fabio ", "cor"], "sCMOS_fliplr": [(2018, 2016), 0.0734, 65535, "sCMOS_fliplr", 3828, "uint16", "binned 2x2, CCD parameters read from tif header by fabio", "tif"], "sCMOS_fliplr_16M": [(2 * 2018, 2 * 2016), 0.0734 / 2.0, 65535, "sCMOS_fliplr", 3828, "uint16", "binned 1x1, CCD parameters binned 1x1 read from tif header by fabio ", "tif"], "sCMOS_16M": [(2 * 2018, 2 * 2016), 0.0734 / 2.0, 65535, "no", 3828, "uint16", "binned 1x1, CCD parameters binned 1x1 read from tif header by fabio ", "tif"], diff --git a/lauetoolsnn/lauetoolsneuralnetwork.py b/lauetoolsnn/lauetoolsneuralnetwork.py index 1ff2c9e..d138561 100644 --- a/lauetoolsnn/lauetoolsneuralnetwork.py +++ b/lauetoolsnn/lauetoolsneuralnetwork.py @@ -149,7 +149,6 @@ def resource_path(relative_path, verbose=0): print("Base path of the library: ",base_path) return os.path.join(base_path, relative_path) - try: import pkg_resources # part of setuptools version_package = pkg_resources.require("lauetoolsnn")[0].version @@ -159,7 +158,6 @@ def resource_path(relative_path, verbose=0): latest_version = versions("lauetoolsnn")[0] frame_title = "Laue Neural-Network model- v3 @Ravi @Jean-Sebastien \n@author: Ravi raj purohit PURUSHOTTAM RAJ PUROHIT (purushot@esrf.fr) \n@guide: Jean-Sebastien MICHA (micha@esrf.fr)" - Logo = resource_path("lauetoolsnn_logo_bXM_2.png", verbose=0) Logo_splash = resource_path("lauetoolsnn_splash_bXM_2.png", verbose=0) gif_path = resource_path("frames_medres.gif", verbose=0) @@ -171,7 +169,7 @@ def resource_path(relative_path, verbose=0): material1_global = "Si" ## same key as used in LaueTools symmetry1_global = "cubic" prefix_global = "" - detectorparameters_global = [79.50900, 977.9000, 931.8900, 0.3570000, 0.4370000] + detectorparameters_global = [79.51, 977.9, 931.9, 0.36, 0.44] pixelsize_global = 0.0734 # 0.079142 # ccd_label_global = "sCMOS" #"MARCCD165" #"Cor"# dim1_global = 2018 #2048 # diff --git a/lauetoolsnn/misc_scripts/check_classHKL_function.py b/lauetoolsnn/misc_scripts/check_classHKL_function.py new file mode 100644 index 0000000..a4721fa --- /dev/null +++ b/lauetoolsnn/misc_scripts/check_classHKL_function.py @@ -0,0 +1,99 @@ +# coding: utf-8 +""" +Created on June 18 06:54:04 2021 + +Check generate_classHKL function +@author: Ravi raj purohit PURUSHOTTAM RAJ PUROHIT (purushot@esrf.fr) + +""" +__author__ = "Ravi raj purohit PURUSHOTTAM RAJ PUROHIT, CRG-IF BM32 @ ESRF" + +if __name__ == '__main__': #enclosing required because of multiprocessing + + ## Import modules used for this Notebook + import os + + # ============================================================================= + # Step 0: Define the dictionary with all parameters + # ============================================================================= + ## User Input dictionary with parameters + ## In case of only one phase/material, keep same value for material_ and material1_ key + input_params = { + "global_path" : r"C:\Users\purushot\Desktop\LaueNN_script", + "prefix" : "_2phase", ## prefix for the folder to be created for training dataset + + "material_": ["GaN", "Si"], ## same key as used in dict_LaueTools + "symmetry": ["hexagonal", "cubic"], ## crystal symmetry of material_ + "SG": [184, 227], #186 ## Space group of material_ (None if not known) + "hkl_max_identify" : [6,5], ## Maximum hkl index to classify in a Laue pattern + "nb_grains_per_lp_mat" : [2,1], ## max grains to be generated in a Laue Image + + ## hkl_max_identify : can be "auto" or integer: Maximum index of HKL to build output classes + + # ============================================================================= + # ## Data generation settings + # ============================================================================= + "grains_nb_simulate" : 500, ## Number of orientations to generate (takes advantage of crystal symmetry) + "classes_with_frequency_to_remove": [100,100], ## classes_with_frequency_to_remove: HKL class with less appearance than + ## specified will be ignored in output + "desired_classes_output": ["all","all"], ## desired_classes_output : can be all or an integer: to limit the number of output classes + + "include_small_misorientation": False, ## to include additional data with small angle misorientation + "misorientation": 5, ##only used if "include_small_misorientation" is True + "maximum_angle_to_search":20, ## Angle of radial distribution to reconstruct the histogram (in deg) + "step_for_binning" : 0.1, ## bin widht of angular radial distribution in degree + } + + # ============================================================================= + # Step 1 + # ============================================================================= + ## if LaueToolsNN is properly installed + from lauetoolsnn.utils_lauenn import get_material_detail, generate_classHKL + # ## Get material parameters + # ### Generates a folder with material name and gets material unit cell parameters + # ### and symmetry object from the get_material_detail function + material_= input_params["material_"][0] + material1_= input_params["material_"][1] + n = input_params["hkl_max_identify"][0] + n1 = input_params["hkl_max_identify"][1] + maximum_angle_to_search = input_params["maximum_angle_to_search"] + step_for_binning = input_params["step_for_binning"] + symm_ = input_params["symmetry"][0] + symm1_ = input_params["symmetry"][1] + SG = input_params["SG"][0] + SG1 = input_params["SG"][1] + + ## read hkl information from a fit file in case too large HKLs + manual_hkl_list=False + if manual_hkl_list: + import numpy as np + temp = np.loadtxt(r"img_0000_LT_1.fit") + hkl_array = temp[:,2:5] + hkl_array1 = None + else: + hkl_array = None + hkl_array1 = None + + if material_ != material1_: + save_directory = os.path.join(input_params["global_path"],material_+"_"+material1_+input_params["prefix"]) + else: + save_directory = os.path.join(input_params["global_path"],material_+input_params["prefix"]) + print("save directory is : "+save_directory) + if not os.path.exists(save_directory): + os.makedirs(save_directory) + + ## get unit cell parameters and other details required for simulating Laue patterns + rules, symmetry, lattice_material,\ + crystal, SG, rules1, symmetry1,\ + lattice_material1, crystal1, SG1 = get_material_detail(material_, SG, symm_, + material1_, SG1, symm1_) + + # ## Generate Neural network output classes (Laue spot hkls) using the generate_classHKL function + ## procedure for generation of GROUND TRUTH classes + # general_diff_cond = True will eliminate the hkl index that does not + # satisfy the general reflection conditions, otherwise they will be eliminated in the next stage + generate_classHKL(n, rules, lattice_material, symmetry, material_, + crystal=crystal, SG=SG, general_diff_cond=False, + save_directory=save_directory, write_to_console=print, + ang_maxx = maximum_angle_to_search, + step = step_for_binning, mat_listHKl=hkl_array) \ No newline at end of file diff --git a/lauetoolsnn/utils_lauenn.py b/lauetoolsnn/utils_lauenn.py index a67e0fc..dc81a37 100644 --- a/lauetoolsnn/utils_lauenn.py +++ b/lauetoolsnn/utils_lauenn.py @@ -32,7 +32,6 @@ # matplotlib.use('Qt5Agg') # matplotlib.rcParams.update({'font.size': 14}) from mpl_toolkits.axes_grid1 import make_axes_locatable - import numpy as np from math import acos import time @@ -44,10 +43,8 @@ # from scipy.spatial.transform import Rotation as R import _pickle as cPickle import configparser - from multiprocessing import Process, Queue, cpu_count from tqdm import tqdm - # from skimage.transform import (hough_line, hough_line_peaks) # ============================================================================= # Additonal networkx module @@ -105,7 +102,6 @@ tensorflow_keras = False # import h5py - ## GPU Nvidia drivers needs to be installed! Ughh ## if wish to use only CPU set the value to -1 else set it to 0 for GPU ## CPU training is suggested (as the model requires more RAM) @@ -480,7 +476,6 @@ def array_generator(path_, batch_size, n_classes, loc_new, write_to_console=None obj = np.load(array_path) trainX = obj["arr_0"] loc1 = obj["arr_1"] - if len(trainX) == 0 or len(loc1) == 0: if write_to_console != None: write_to_console("Skipping File: "+ array_path+"; No data is found") @@ -496,21 +491,17 @@ def array_generator(path_, batch_size, n_classes, loc_new, write_to_console=None loc1_new.append(temp_loc) else: loc1_new_rmv.append(k) - loc1_new = np.array(loc1_new).ravel() loc1_new_rmv = np.array(loc1_new_rmv).ravel() - if len(trainX) != len(loc1_new): if len(loc1_new_rmv) > 0: trainX = np.delete(trainX, loc1_new_rmv, axis=0) - if bs == 0 or temp_var: trainX1 = np.copy(trainX) trainY1 = np.copy(loc1_new) else: trainX1 = np.vstack((trainX1, trainX)) trainY1 = np.hstack((trainY1, loc1_new)) - ## To normalize the size of one hot encoding count = 0 if np.min(trainY1) != 0: @@ -519,14 +510,12 @@ def array_generator(path_, batch_size, n_classes, loc_new, write_to_console=None if np.max(trainY1) != (n_classes-1): trainY1 = np.append(trainY1, n_classes-1) count += 1 - if tocategorical: trainY1 = to_categorical(trainY1) if count == 1: trainY1 = np.delete(trainY1, [len(trainY1)-1] ,axis=0) elif count == 2: trainY1 = np.delete(trainY1, [len(trainY1)-1,len(trainY1)-2] ,axis=0) - yield trainX1, trainY1 def vali_array(path_, batch_size, n_classes, loc_new, write_to_console=None, tocategorical=True): @@ -818,18 +807,20 @@ def generate_classHKL(n, rules, lattice_material, symmetry, material_, crystal=N list with hkl's removed from the dictionary n : int Max index hkl - """ - - temp_ = GT.threeindices_up_to(int(n)) - + ##add user defined categories of hkl to be used (non sequential also) + ## add directly to the temp_ if np.all(mat_listHKl != None): - mat_listHKl = np.array(mat_listHKl) - temp_ = np.vstack((temp_, mat_listHKl)) + temp_ = mat_listHKl + else: + temp_ = GT.threeindices_up_to(int(n)) + + # if np.all(mat_listHKl != None): + # mat_listHKl = np.array(mat_listHKl) + # temp_ = np.vstack((temp_, mat_listHKl)) classhkl_ = temp_ - if general_diff_cond: classhkl_ = crystal.hkl_allowed_array(classhkl_) @@ -841,7 +832,6 @@ def generate_classHKL(n, rules, lattice_material, symmetry, material_, crystal=N progress.setMaximum(len(classhkl_)) hkl_all = {} - # another_method = False for i in range(len(classhkl_)): new_hkl = classhkl_[i,:] @@ -916,8 +906,6 @@ def generate_classHKL(n, rules, lattice_material, symmetry, material_, crystal=N list_remove = [] for i, j in enumerate(codebars): for k, l in enumerate(codebars): - # if i in list_appended and k in list_appended: - # continue if i != k and np.all(j == l): if keys_rmv[i] in list_remove or keys_rmv[k] in list_remove: if write_to_console !=None: @@ -1084,17 +1072,13 @@ def write_training_testing_dataMTEX(save_directory,material_, material1_, lattic def get_material_data(material_="Cu", ang_maxx = 45, step = 0.5, hkl_ref=13, classhkl = None): a, b, c, alpha, beta, gamma = dictLT.dict_Materials[material_][1] Gstar = CP.Gstar_from_directlatticeparams(a, b, c, alpha, beta, gamma) - rules = dictLT.dict_Materials[material_][-1] - - hkl2 = GT.threeindices_up_to(int(hkl_ref)) - hkl2 = CP.ApplyExtinctionrules(hkl2,rules) - hkl2 = hkl2.astype(np.int16) query_angle = ang_maxx/2. angle_tol = ang_maxx/2. metrics = Gstar - hkl1 = classhkl + hkl1 = np.copy(classhkl) + hkl2 = np.copy(classhkl) H1 = hkl1 n1 = hkl1.shape[0] H2 = hkl2