From c2df57b6922d673e71c04f39d881aeb8e045190c Mon Sep 17 00:00:00 2001 From: "Manuel D. Morales" Date: Tue, 20 Apr 2021 06:37:58 -0500 Subject: [PATCH] Compute_CheckLabels.m --- Compute_CNNcfg.m | 70 ++++ Compute_CNNconstructXY.m | 21 ++ Compute_CNNeliminateSNR.m | 61 ++++ Compute_ClassificationApply.m | 13 + Compute_ClassificationApplyNB.m | 26 ++ Compute_ClassificationApplySVM.m | 26 ++ Compute_ClassificationCrossValidation.m | 242 ++++++++++++++ Compute_ClassificationMetrics.m | 124 ++++++++ Compute_ClassificationTrain.m | 147 +++++++++ Compute_ClassificationTrainNB.m | 54 ++++ Compute_ClassificationTrainSVM.m | 54 ++++ Compute_WaveletMorlet.m | 186 +++++++++++ MetaProg_DetectionCNN_1PrepareData.m | 30 ++ gw_computegetdata.m | 87 +++++ gw_computewhitendata.m | 235 ++++++++++++++ gw_getinjectioninfoS6.m | 227 +++++++++++++ gw_getsegmentsinfoOVERLAP.m | 43 +++ gw_readligo.m | 310 ++++++++++++++++++ prog021_DetectionCNN_1PrepareData.m | 407 ++++++++++++++++++++++++ prog021_DetectionCNN_2PrepareData.m | 148 +++++++++ prog021_DetectionCNN_3ClasifyBasic.m | 227 +++++++++++++ prog021_DetectionCNN_4ClasifyNfold.m | 141 ++++++++ 22 files changed, 2879 insertions(+) create mode 100755 Compute_CNNcfg.m create mode 100755 Compute_CNNconstructXY.m create mode 100755 Compute_CNNeliminateSNR.m create mode 100755 Compute_ClassificationApply.m create mode 100755 Compute_ClassificationApplyNB.m create mode 100755 Compute_ClassificationApplySVM.m create mode 100755 Compute_ClassificationCrossValidation.m create mode 100755 Compute_ClassificationMetrics.m create mode 100755 Compute_ClassificationTrain.m create mode 100755 Compute_ClassificationTrainNB.m create mode 100755 Compute_ClassificationTrainSVM.m create mode 100755 Compute_WaveletMorlet.m create mode 100755 MetaProg_DetectionCNN_1PrepareData.m create mode 100755 gw_computegetdata.m create mode 100755 gw_computewhitendata.m create mode 100755 gw_getinjectioninfoS6.m create mode 100755 gw_getsegmentsinfoOVERLAP.m create mode 100755 gw_readligo.m create mode 100755 prog021_DetectionCNN_1PrepareData.m create mode 100755 prog021_DetectionCNN_2PrepareData.m create mode 100755 prog021_DetectionCNN_3ClasifyBasic.m create mode 100755 prog021_DetectionCNN_4ClasifyNfold.m diff --git a/Compute_CNNcfg.m b/Compute_CNNcfg.m new file mode 100755 index 0000000..91aa2d4 --- /dev/null +++ b/Compute_CNNcfg.m @@ -0,0 +1,70 @@ +function cfg = Compute_CNNcfg(Nsamples) + + + +% ---------------------------------------- +% 1) Initialize structure +% +cfg = []; + + + +% ---------------------------------------- +% 2) Iteration and epochs: +% +% Recall that an iteration is one step taken in the gradient descent +% algorithm towards minimizing the loss function using a mini-batch. In +% other words, each iteration is an estimation of the gradient and an +% update of the network parameters +% +cfg.MiniBatchSize = 128; % Subset of the training set that is used to evaluate the gradient of the loss function and update the weights +cfg.MaxEpochs = 100; % An epoch is the full pass of the training algorithm over the entire training set +cfg.Shuffle = 'once'; % (never|once|every-epoch) Shuffle the training data before each training epoch, and shuffle the validation data before each network validation + + + +% ---------------------------------------- +% 3) Validation process +% +% A validation set is used to test learning and generalization during the +% training process. +% By default, if the validation loss is larger than or equal to the +% previously smallest loss five times in a row, then network training +% stops. To change the number of times that the validation loss is allowed +% to not decrease before training stops, use the 'ValidationPatience'. You +% can add additional stopping criteria using output functions +% +cfg.ValidationData = []; % ({Xval,Yval}) Used to validate the network at regular intervals during training. +% cfg.ValidationFrequency = 20; % Number of iterations between evaluations of validation metrics. A suggestion is to choose this value so that the network is validated once/twice/.. per epoch. +cfg.NumValPerEpoch = 1; % Number of validations per epoch. This is not a "trainingOptions" parameter but it is used to gently compute the "ValidationFrequency" +cfg.ValidationFrequency = floor(Nsamples/cfg.MiniBatchSize/cfg.NumValPerEpoch); +cfg.ValidationPatience = 5; % (scalar|Inf) Turn off the built-in validation stopping criterion (which uses the loss) by setting the 'ValidationPatience' value to Inf. +cfg.OutputFcn = '@(info)stopIfAccuracyNotImproving(info,3));'; % The traininf calls the specified functions once before the start of training, after each iteration, and once after training has finished. The training passes a structure containing information in the following fields: +%cfg.OutputFcn = '@(info)savetrainingplot(info);'; + +% ---------------------------------------- +% 4) Learning rate +% +cfg.InitialLearnRate = 1.01; % (default:0.01) If the learning rate is too low, then training takes a long time. If the learning rate is too high, then training might reach a suboptimal result +cfg.LearnRateSchedule = 'none'; % (none|piecewise) Option for dropping the learning rate during training. The software updates the learning rate every certain number of epochs by multiplying with a certain factor. +cfg.LearnRateDropFactor = 0.1; % (default:0.1) Factor for dropping the learning rate. Multiplicative factor to apply to the learning rate every time a certain number of epochs passes. Valid only when the value of LearnRateSchedule is 'piecewise'. +cfg.LearnRateDropPeriod = 4; % (default:10) Number of epochs for dropping the learning rate. Valid only when the value of LearnRateSchedule is 'piecewise'. + + + +% ---------------------------------------- +% Otros paramteres +% +% cfg.CheckpointPath = % Path for saving checkpoint networks +% cfg.ExecutionEnvironment = % (default:??????) (auto|cpu|gpu|multi-gpu|parallel) Hardware resource for training network +% cfg.L2Regularization = % (default:0.0001) Factor for L2 regularizer (weight decay). You can specify a multiplier for the L2 regularizer for network layers with learnable parameters. +% cfg.Momentum = % (default:0.9) Contribution of the gradient step from the previous iteration to the current iteration of the training. A value of 0 means no contribution from the previous step, whereas a value of 1 means maximal contribution from the previous step. + + + +% ---------------------------------------- +% Progress visualization +% +cfg.Plots = 'none'; % (none|training-progress) +cfg.Verbose = 1; % Indicator to display training progress information in the command window +cfg.VerboseFrequency = cfg.ValidationFrequency; % Frequency of verbose printing, which is the number of iterations between printing to the command window diff --git a/Compute_CNNconstructXY.m b/Compute_CNNconstructXY.m new file mode 100755 index 0000000..0dc9950 --- /dev/null +++ b/Compute_CNNconstructXY.m @@ -0,0 +1,21 @@ +function Data = Compute_CNNconstructXY(Data) + +% Convertir Xtfr a X: the data format of X for the CNN should be (16 32 1 Nsamples) +for i=1:size(Data.Xtfr,1) + Data.X(:,:,1,i) = squeeze(Data.Xtfr(i,:,:)); +end +Data = rmfield(Data,'Xtfr'); + +% Save currect Y and create the class label Y vector +YALL = Data.Y; +Data = rmfield(Data,'Y'); + +Data.Y = YALL(:,1); +Data.YInfo = YALL(:,2:end); +% Data.M1 = YALL(:,2); +% Data.M1 = YALL(:,3); +% Data.D = YALL(:,4); +% Data.SNR = YALL(:,5); + +% Clear garbage +clear ans doplot IFO Nsamples IndRan IndTrai IndTest Nlabels IndRan i YALL \ No newline at end of file diff --git a/Compute_CNNeliminateSNR.m b/Compute_CNNeliminateSNR.m new file mode 100755 index 0000000..4320a86 --- /dev/null +++ b/Compute_CNNeliminateSNR.m @@ -0,0 +1,61 @@ +function Data = Compute_CNNeliminateSNR(Data,SNR2ELIM) + +% SNR2ELIM: SNR limit value to eliminate + +% ---------------------------------------- +% Get data for class noise: 1 +Ind_c0 = Data.Y(:,1)==1; +Y_c0 = Data.Y(Ind_c0,:); +TFR_c0 = Data.Xtfr(Ind_c0,:,:); + +% ---------------------------------------- +% Get data for class gw: 2 +Ind_c1 = Data.Y(:,1)==2; +Y_c1 = Data.Y(Ind_c1,:); +TFR_c1 = Data.Xtfr(Ind_c1,:,:); + +% ---------------------------------------- +% Eliminate data for which SNR<=10 +Ind2Eli = Y_c1(:,5)<=SNR2ELIM; +Y_c0(Ind2Eli,:) = []; +Y_c1(Ind2Eli,:) = []; +TFR_c0(Ind2Eli,:,:) = []; +TFR_c1(Ind2Eli,:,:) = []; + +% ---------------------------------------- +% Construct data to keep +Data.Y = [Y_c0 ; Y_c1 ]; +Data.Xtfr = [TFR_c0 ; TFR_c1]; + +% ---------------------------------------- +% Clear garbage +clear ans Ind2Eli Ind_c1 Ind_c0 TFR_c0 TFR_c1 Y_c0 Y_c1 + + +%% PLOT FOR DEBUGGING + +% if (0) +% +% % ---------------------------------------- +% for i=1:1:size(Data.Y,1)/2 +% figure(1) +% +% subplot(2,1,1) +% imagesc(Data.t,Data.f,squeeze(Data.Xtfr(i,:,:))) +% xlabel('Time (s)'), ylabel('Frequency (Hz)'), title('n(t)') +% colormap jet, view(0,90), box on, grid on, set(gca,'YDir','normal') +% +% subplot(2,1,2) +% imagesc(Data.t,Data.f,squeeze(Data.Xtfr(i+size(Data.Y,1)/2,:,:))) +% xlabel('Time (s)'), ylabel('Frequency (Hz)'), title(['n(t)+h(t) | SNR=' num2str(Data.Y(i+size(Data.Y,1)/2,5))]) +% colormap jet, view(0,90), box on, grid on, set(gca,'YDir','normal') +% +% pause(0.2) +% +% end % for i=1:1:size(Data.Y,1)/2 +% clear ans i +% +% % ---------------------------------------- +% % Return +% return +% end % if (1) \ No newline at end of file diff --git a/Compute_ClassificationApply.m b/Compute_ClassificationApply.m new file mode 100755 index 0000000..59a66f9 --- /dev/null +++ b/Compute_ClassificationApply.m @@ -0,0 +1,13 @@ +function [YEsti, YProb] = Compute_ClassificationApply(XTest,Model) + + + +%% APPLY CLASSIFIER: APPLY MODEL TO NEW DATA + +% Apply classifier. Get predicted class and the probability for each class +[YEsti,YProb] = classify(Model.net,XTest); + +% Convert predictec class from categorical to double +if iscategorical(YEsti) + YEsti = grp2idx(YEsti); +end \ No newline at end of file diff --git a/Compute_ClassificationApplyNB.m b/Compute_ClassificationApplyNB.m new file mode 100755 index 0000000..f9ff71d --- /dev/null +++ b/Compute_ClassificationApplyNB.m @@ -0,0 +1,26 @@ +function [YEstiNB, YProbNB, CostNB] = Compute_ClassificationApplyNB(XTest,Model) + + + +%% APPLY CLASSIFIER: APPLY MODEL TO NEW DATA + +% Size of input data +Sx = size(XTest,1); +Sy = size(XTest,2); +Sz = size(XTest,3); +Sn = size(XTest,4); + +% i) flat images in XTrain, ii) remove dimension of length 1, iii) transpose +XTest_flat = transpose(squeeze(reshape(XTest,[Sx*Sy,Sz,Sn]))); + +% Apply classifier. Get predicted class and the probability for each class +[YEstiNB,YProbNB,CostNB] = predict(Model.NBModel,XTest_flat); + + +% Convert predictec class from categorical to double +if iscategorical(YEstiNB) + YEstiNB = grp2idx(YEstiNB); +end + +% Convert cell array to double array +YEstiNB = str2num(cell2mat(YEstiNB)); \ No newline at end of file diff --git a/Compute_ClassificationApplySVM.m b/Compute_ClassificationApplySVM.m new file mode 100755 index 0000000..1bc671c --- /dev/null +++ b/Compute_ClassificationApplySVM.m @@ -0,0 +1,26 @@ +function [YEstiSVM, YScoreSVM] = Compute_ClassificationApplySVM(XTest,Model) + + + +%% APPLY CLASSIFIER: APPLY MODEL TO NEW DATA + +% Size of input data +Sx = size(XTest,1); +Sy = size(XTest,2); +Sz = size(XTest,3); +Sn = size(XTest,4); + +% i) flat images in XTrain, ii) remove dimension of length 1, iii) transpose +XTest_flat = transpose(squeeze(reshape(XTest,[Sx*Sy,Sz,Sn]))); + +% Apply classifier. Get predicted class and the probability for each class +[YEstiSVM,YScoreSVM] = predict(Model.SVMModel,XTest_flat); + + +% Convert predictec class from categorical to double +if iscategorical(YEstiSVM) + YEstiSVM = grp2idx(YEstiSVM); +end + +% Convert cell array to double array +YEstiSVM = str2num(cell2mat(YEstiSVM)); \ No newline at end of file diff --git a/Compute_ClassificationCrossValidation.m b/Compute_ClassificationCrossValidation.m new file mode 100755 index 0000000..144896c --- /dev/null +++ b/Compute_ClassificationCrossValidation.m @@ -0,0 +1,242 @@ +function [folds,foldsSVM,foldsNB] = Compute_ClassificationCrossValidation(X,Y,cfg) + + +%% INITIALIZE VARIABLES + +% For CNN algorithm + +folds.Nfolds = cfg.Nfolds; + +folds.Model = cell(cfg.Nfolds,1); +folds.YTest = cell(cfg.Nfolds,1); +folds.YEsti = cell(cfg.Nfolds,1); +folds.YProb = cell(cfg.Nfolds,1); +folds.YInfo = cell(cfg.Nfolds,1); + +folds.Metrics.CM = zeros(2,2,cfg.Nfolds); +folds.Metrics.CA = zeros(cfg.Nfolds,1); +folds.Metrics.F1 = zeros(cfg.Nfolds,1); +folds.Metrics.KP = zeros(cfg.Nfolds,1); +folds.Metrics.Tclass = zeros(cfg.Nfolds,1); + +% new included metrics +folds.Metrics.LL = zeros(cfg.Nfolds,1); +folds.Metrics.PR = zeros(cfg.Nfolds,1); +folds.Metrics.RE = zeros(cfg.Nfolds,1); +folds.Metrics.FO = zeros(cfg.Nfolds,1); + +folds.All.YTest = []; +folds.All.YEsti = []; +folds.All.YProb = []; +folds.All.Metrics = cell(cfg.Nfolds,1); + +% Additional arrays for SVM algorithm + +foldsSVM.Nfolds = cfg.Nfolds; + +foldsSVM.Model = cell(cfg.Nfolds,1); +foldsSVM.YEsti = cell(cfg.Nfolds,1); +foldsSVM.YScore = cell(cfg.Nfolds,1); + +foldsSVM.Metrics.CM = zeros(2,2,cfg.Nfolds); +foldsSVM.Metrics.CA = zeros(cfg.Nfolds,1); +foldsSVM.Metrics.F1 = zeros(cfg.Nfolds,1); +foldsSVM.Metrics.KP = zeros(cfg.Nfolds,1); +foldsSVM.Metrics.Tclass = zeros(cfg.Nfolds,1); + +foldsSVM.Metrics.PR = zeros(cfg.Nfolds,1); +foldsSVM.Metrics.RE = zeros(cfg.Nfolds,1); +foldsSVM.Metrics.FO = zeros(cfg.Nfolds,1); + +foldsSVM.All.YEsti = []; +foldsSVM.All.YScore = []; +foldsSVM.All.Metrics = cell(cfg.Nfolds,1); + +% Additional arrays for NB algorithm +foldsNB.Nfolds = cfg.Nfolds; + +foldsNB.Model = cell(cfg.Nfolds,1); +foldsNB.YEsti = cell(cfg.Nfolds,1); +foldsNB.YProb = cell(cfg.Nfolds,1); +foldsNB.Cost = cell(cfg.Nfolds,1); + +foldsNB.Metrics.CM = zeros(2,2,cfg.Nfolds); +foldsNB.Metrics.CA = zeros(cfg.Nfolds,1); +foldsNB.Metrics.F1 = zeros(cfg.Nfolds,1); +foldsNB.Metrics.KP = zeros(cfg.Nfolds,1); +foldsNB.Metrics.Tclass = zeros(cfg.Nfolds,1); + +foldsNB.Metrics.PR = zeros(cfg.Nfolds,1); +foldsNB.Metrics.RE = zeros(cfg.Nfolds,1); +foldsNB.Metrics.FO = zeros(cfg.Nfolds,1); + +foldsNB.All.YEsti = []; +foldsNB.All.YProb = []; +foldsNB.All.Cost = []; +foldsNB.All.Metrics = cell(cfg.Nfolds,1); + +%% TRAINNING AND VALIDATION FOR EACH FOLD + + +for ifold = 1:cfg.Nfolds + %ifold = 1; % solo para debugear mientras testeo algoritmos SVM y NB, comentar despues + + fprintf('Fold %i of %i \r',ifold,cfg.Nfolds) + + + % -------------------------------------------- + % Indices of the train and test sets for the current fold + Ind_test = (cfg.IndCroossVal == ifold); + Ind_train = ~Ind_test; + + + % -------------------------------------------- + % Get train trials for the current fold + XTrain = X(:,:,:,Ind_train); + YTrain = Y(Ind_train); + + + % -------------------------------------------- + % Get test trials for the current fold + XTest = X(:,:,:,Ind_test); + YTest = Y(Ind_test); + + + % -------------------------------------------- + % Train the CNN + % disp(YTrain) + % pause + folds.Model{ifold} = Compute_ClassificationTrain(XTrain,YTrain,cfg); + % Salidas (nombres definidos dentro de la función): Model.net,Model.traininfo + + % Train the SVM + foldsSVM.Model{ifold} = Compute_ClassificationTrainSVM(XTrain,YTrain,cfg); + % Salida (nombre definido dentro de la función): SVMModel + + % Train the NB algorithm + foldsNB.Model{ifold} = Compute_ClassificationTrainNB(XTrain,YTrain,cfg); + % Salida (nombre definido dentro de la función): NBModel + + % -------------------------------------------- + % Classify the XTest data + % With CNN algorithm + tic + [YEsti,YProb] = Compute_ClassificationApply(XTest,folds.Model{ifold}); + folds.Metrics.Tclass(ifold) = toc; + + % With SVM algorithm + tic + [YEstiSVM,YScoreSVM] = Compute_ClassificationApplySVM(XTest,foldsSVM.Model{ifold}); + foldsSVM.Metrics.Tclass(ifold) = toc; + + % With NB algorithm + tic + [YEstiNB,YProbNB,CostNB] = Compute_ClassificationApplyNB(XTest,foldsNB.Model{ifold}); + foldsNB.Metrics.Tclass(ifold) = toc; + + % -------------------------------------------- + % Save YTest, YEsti, YProb, YEstiSVM, and YScoreSVM for the current fold + + % CNN + folds.YTest{ifold} = YTest; + folds.YEsti{ifold} = YEsti; + folds.YProb{ifold} = YProb; + + % Additional data for SVM + foldsSVM.YEsti{ifold} = YEstiSVM; + foldsSVM.YScore{ifold} = YScoreSVM; + + % Additional data for NB + foldsNB.YEsti{ifold} = YEstiNB; + foldsNB.YProb{ifold} = YProbNB; + foldsNB.Cost{ifold} = CostNB; + + % -------------------------------------------- + % Compute metrics for the current fold +% disp([YTest,YEsti]) +% disp(unique(YTest)) +% disp(unique(YEsti)) + + % Metrics for CNN algorithm + Metrics = Compute_ClassificationMetrics(YTest,YEsti,YProb); + folds.Metrics.CM(:,:,ifold) = Metrics.CM; + folds.Metrics.CA(ifold) = Metrics.CA; + folds.Metrics.F1(ifold) = Metrics.F1; + folds.Metrics.KP(ifold) = Metrics.KP; + folds.Metrics.LL(ifold) = Metrics.LL; + % new included metrics + folds.Metrics.PR(ifold) = Metrics.PR; + folds.Metrics.RE(ifold) = Metrics.RE; + folds.Metrics.FO(ifold) = Metrics.FO; + folds.Metrics.GM(ifold) = Metrics.GM; + + % Metrics for SVM algorithm + Metrics = Compute_ClassificationMetrics(YTest,YEstiSVM,YScoreSVM); + foldsSVM.Metrics.CM(:,:,ifold) = Metrics.CM; + foldsSVM.Metrics.CA(ifold) = Metrics.CA; + foldsSVM.Metrics.F1(ifold) = Metrics.F1; + foldsSVM.Metrics.KP(ifold) = Metrics.KP; + foldsSVM.Metrics.PR(ifold) = Metrics.PR; + foldsSVM.Metrics.RE(ifold) = Metrics.RE; + foldsSVM.Metrics.FO(ifold) = Metrics.FO; + foldsSVM.Metrics.GM(ifold) = Metrics.GM; + + % Metrics for NB algorithm + Metrics = Compute_ClassificationMetrics(YTest,YEstiNB,YProbNB); + foldsNB.Metrics.CM(:,:,ifold) = Metrics.CM; + foldsNB.Metrics.CA(ifold) = Metrics.CA; + foldsNB.Metrics.F1(ifold) = Metrics.F1; + foldsNB.Metrics.KP(ifold) = Metrics.KP; + foldsNB.Metrics.PR(ifold) = Metrics.PR; + foldsNB.Metrics.RE(ifold) = Metrics.RE; + foldsNB.Metrics.FO(ifold) = Metrics.FO; + foldsNB.Metrics.GM(ifold) = Metrics.GM; + + % -------------------------------------------- + % Save YInfo for the current fold + folds.YInfo{ifold} = cfg.YInfo(Ind_test,:); + + + % -------------------------------------------- + % Append YTest, YEsti, YProb across folds + folds.All.YTest = [ folds.All.YTest ; YTest ]; + folds.All.YEsti = [ folds.All.YEsti ; YEsti ]; + folds.All.YProb = [ folds.All.YProb ; YProb ]; + % for SVM algorithm + foldsSVM.All.YEsti = [ foldsSVM.All.YEsti ; YEstiSVM ]; + foldsSVM.All.YScore = [ foldsSVM.All.YScore ; YScoreSVM ]; + % for NB algorithm + foldsNB.All.YEsti = [ foldsNB.All.YEsti ; YEstiNB ]; + foldsNB.All.YProb = [ foldsNB.All.YProb ; YProbNB ]; + foldsNB.All.Cost = [ foldsNB.All.Cost ; CostNB ]; + + + % create initial arrays for All SVM and NB results + foldsSVM.All.Metrics = cell(cfg.Nfolds,1); + foldsNB.All.Metrics = cell(cfg.Nfolds,1); + + +end % for ifold = 1:Nfolds + + +% -------------------------------------------- +% Compute metrics across-all-folds + +% CNN algorithm +folds.All.Metrics = Compute_ClassificationMetrics(folds.All.YTest,... + folds.All.YEsti,folds.All.YProb); + +% SVM algorithm +foldsSVM.All.Metrics = Compute_ClassificationMetrics(folds.All.YTest,... + foldsSVM.All.YEsti,foldsSVM.All.YScore); + +% NB algorithm +foldsNB.All.Metrics = Compute_ClassificationMetrics(folds.All.YTest,... + foldsNB.All.YEsti,foldsNB.All.YProb); + +% -------------------------------------------- +fprintf('PILAS: CNN accuracy of %3.2f%% \n',folds.All.Metrics.CA) + +fprintf('PILAS: SVM accuracy of %3.2f%% \n',foldsSVM.All.Metrics.CA) + +fprintf('PILAS: NB accuracy of %3.2f%% \n',foldsNB.All.Metrics.CA) \ No newline at end of file diff --git a/Compute_ClassificationMetrics.m b/Compute_ClassificationMetrics.m new file mode 100755 index 0000000..262e035 --- /dev/null +++ b/Compute_ClassificationMetrics.m @@ -0,0 +1,124 @@ +function Metrics = Compute_ClassificationMetrics(YTest,YEsti,YProb) + + fprintf('num. de clases en YTest: %d\n', length(unique(YTest))) + fprintf('num. de clases en YEsti: %d\n', length(unique(YEsti))) + +% -------------------------------------------- +% verificar que ambos vectores tengan el mismo numero de clases +%if length(unique(YTest)) ~= length(unique(YEsti)) +% error('PILAS: no hay el mismo numero de clases') +%end + +% -------------------------------------------- +% Convertir de categorical a double +if iscategorical(YTest) + YTest = grp2idx(YTest); +end +if iscategorical(YEsti) + YEsti = grp2idx(YEsti); +end + + +% -------------------------------------------- +% Verificar que ambos vectores tengan el mismo nombre para las clases i.e. 1,2,3 +% error('PILAS: los nombres de las clases son diferentes entre YTest y YEsti') + + +% -------------------------------------------- +currentlabels = unique(YTest); +Nclasses = length(currentlabels); +if Nclasses==2 + classes = [1 2]; + confu = zeros(2,2); +elseif Nclasses==3 + classes = [1 2 3]; + confu = zeros(3,3); +else + error('PILAS: trabaje perro') +end +clear ans currentlabels + + +% -------------------------------------------- +% Compute classification accuracy +% acc = 100*sum((YTest-YEsti)==0)/length(YTest); PILAS BORRAR +acc = 100*sum(YEsti == YTest) / numel(YTest); + +% -------------------------------------------- +% Compute confusion matrix [YTest YEsti] +if Nclasses==2 + Ind_Class1 = (YTest==classes(1)); + Ind_Class2 = (YTest==classes(2)); + + confu(1,1) = sum(YEsti(Ind_Class1)==1); + confu(1,2) = sum(YEsti(Ind_Class1)==2); + confu(1,:) = 100*confu(1,:)/sum(Ind_Class1); + + confu(2,1) = sum(YEsti(Ind_Class2)==1); + confu(2,2) = sum(YEsti(Ind_Class2)==2); + confu(2,:) = 100*confu(2,:)/sum(Ind_Class2); + + % esto posiblemente esta mal + tp = 100*confu(1,1); + fp = 100*confu(1,2); + fn = 100*confu(2,1); + tn = 100*confu(2,2); + f1score = 2 * tp ./ (2 * tp + fp + fn); + + kappa = (acc-50)/(100-50); + + % new metrics included + + precision = tp / (tp + fp) ; + recall = tp / (tp + fn) ; + fallout = fp / (fp + tn) ; + + % last metric included: gmean1=sqrt(recall*fallout) + gmean1 = sqrt(recall*fallout) ; + +elseif Nclasses==3 + Ind_Class1 = (YTest==classes(1)); + Ind_Class2 = (YTest==classes(2)); + Ind_Class3 = (YTest==classes(3)); + + confu(1,1) = sum(YEsti(Ind_Class1)==1); + confu(1,2) = sum(YEsti(Ind_Class1)==2); + confu(1,3) = sum(YEsti(Ind_Class1)==3); + confu(1,:) = 100*confu(1,:)/sum(Ind_Class1); + + confu(2,1) = sum(YEsti(Ind_Class2)==1); + confu(2,2) = sum(YEsti(Ind_Class2)==2); + confu(2,3) = sum(YEsti(Ind_Class2)==3); + confu(2,:) = 100*confu(2,:)/sum(Ind_Class2); + + confu(3,1) = sum(YEsti(Ind_Class3)==1); + confu(3,2) = sum(YEsti(Ind_Class3)==2); + confu(3,3) = sum(YEsti(Ind_Class3)==3); + confu(3,:) = 100*confu(3,:)/sum(Ind_Class3); + + f1score = NaN; + + kappa = (acc-(100/3))/(100-(100/3)); + +end + +% -------------------------------------------- +% Compute the log-loss +if any(YProb(:)<0) + log_loss = 0; +else + log_loss = -1*sum(log(YProb(1))) + sum(log(YProb(2))); +end + +% Save results +Metrics.CA = acc; +Metrics.CM = confu; +Metrics.F1 = f1score; +Metrics.KP = kappa; +Metrics.LL = log_loss; + +Metrics.PR = precision; +Metrics.RE = recall; +Metrics.FO = fallout; + +Metrics.GM = gmean1; \ No newline at end of file diff --git a/Compute_ClassificationTrain.m b/Compute_ClassificationTrain.m new file mode 100755 index 0000000..4907895 --- /dev/null +++ b/Compute_ClassificationTrain.m @@ -0,0 +1,147 @@ +function Model = Compute_ClassificationTrain(XTrain,YTrain,cfg) + + + +%% SHUFFLING + +if ~isfield(cfg,'Shuffling') + % do nothing +else + if strcmp(cfg.Shuffling,'YES') + IndRandom = randperm(length(YTrain)); + YTrain = YTrain(IndRandom); + else + % do nothing + end % if strcmp(cfg.Shuffling,'YES') +end % if isfield((cfg,'Shuffling') + + + +%% TRAIN CLASSIFIER: COMPUTE MODEL + +% % Fix random number generator seed +% rng(2) + +% Separate train set into two sets: (1) train set and (2) validation set +Nsamples = size(XTrain,4); +IndRan = randperm(Nsamples); +IndVal = IndRan(1:round(Nsamples*.1)); +IndTra = IndRan(round(Nsamples*.1)+1:end); + +XValid = XTrain(:,:,:,IndVal); +YValid = YTrain(IndVal,1); + +XTrain = XTrain(:,:,:,IndTra); +YTrain = YTrain(IndTra,1); + +% Define model options: 'Plots','training-progress' +cfgOptions = Compute_CNNcfg(Nsamples); + +Model.options = trainingOptions('sgdm',... + 'MaxEpochs',cfgOptions.MaxEpochs,... + 'MiniBatchSize',cfgOptions.MiniBatchSize,... + 'Shuffle',cfgOptions.Shuffle,... + 'ValidationData',{XValid,categorical(YValid)},... + 'ValidationFrequency',cfgOptions.ValidationFrequency,... + 'ValidationPatience',cfgOptions.ValidationPatience,... + 'Plots',cfgOptions.Plots,... + 'Verbose',cfgOptions.Verbose,... + 'VerboseFrequency',cfgOptions.ValidationFrequency... + ); +%'OutputFcn',cfgOptions.OutputFcn... + +% Size of the input map +Sx = size(XTrain,1); +Sy = size(XTrain,2); +Sz = size(XTrain,3); + +%disp(Sx) +%disp(Sy) +%disp(Sz) + +% Define model arquitecture +if cfg.Nstacks==1 + + % CNN arquitecture (PILAS: manually fix the dimension of the input layer) + Model.layers = [ ... + imageInputLayer([Sx Sy Sz],'Name','INPUT') + + %convolution2dLayer([5 5],cfg.Nfilters,'Name','CONV1') + convolution2dLayer([4 5],cfg.Nfilters,'Name','CONV1') + + reluLayer('Name','RELU1') + maxPooling2dLayer(2,'Stride',2,'Name','MAXPOOL1') + + fullyConnectedLayer(2,'Name','FULL') + softmaxLayer('Name','SOFTMAX') + classificationLayer('Name','OUTPUT')]; + +elseif cfg.Nstacks==2 + + % CNN arquitecture (PILAS: manually fix the dimension of the input layer) + Model.layers = [ ... + imageInputLayer([Sx Sy Sz],'Name','INPUT') + + %convolution2dLayer([5 5],cfg.Nfilters(1),'Name','CONV1') + convolution2dLayer([4 5],cfg.Nfilters(1),'Name','CONV1') + reluLayer('Name','RELU1') + maxPooling2dLayer(2,'Stride',2,'Name','MAXPOOL1') + + %convolution2dLayer([5 5],cfg.Nfilters(1),'Name','CONV2') + convolution2dLayer([4 5],cfg.Nfilters(1),'Name','CONV2') + reluLayer('Name','RELU2') + maxPooling2dLayer(2,'Stride',2,'Name','MAXPOOL2') + + fullyConnectedLayer(2,'Name','FULL') + softmaxLayer('Name','SOFTMAX') + classificationLayer('Name','OUTPUT')]; + +elseif cfg.Nstacks==3 + + % CNN arquitecture (PILAS: manually fix the dimension of the input layer) + Model.layers = [ ... + imageInputLayer([Sx Sy Sz],'Name','INPUT') + + %convolution2dLayer([5 5],cfg.Nfilters(1),'Name','CONV1') + convolution2dLayer([4 5],cfg.Nfilters(1),'Name','CONV1') + reluLayer('Name','RELU1') + maxPooling2dLayer(2,'Stride',2,'Name','MAXPOOL1') + + %convolution2dLayer([5 5],cfg.Nfilters(1),'Name','CONV2') + convolution2dLayer([4 5],cfg.Nfilters(1),'Name','CONV2') + reluLayer('Name','RELU2') + maxPooling2dLayer(2,'Stride',2,'Name','MAXPOOL2') + + %convolution2dLayer([1 2],cfg.Nfilters(1),'Name','CONV3') + convolution2dLayer([1 4],cfg.Nfilters(1),'Name','CONV3') + reluLayer('Name','RELU3') + %maxPooling2dLayer(1,'Stride',1,'Name','MAXPOOL3') + maxPooling2dLayer([1 2],'Stride',1,'Name','MAXPOOL3') + + fullyConnectedLayer(2,'Name','FULL') + softmaxLayer('Name','SOFTMAX') + classificationLayer('Name','OUTPUT')]; + +else + error('PILAS: trabaje perro') + +end + +% Train classifier: compute the model +tic +[Model.net,Model.traininfo] = trainNetwork(XTrain,categorical(YTrain),Model.layers,Model.options); +Model.Ttrain = toc; + +% function plotTrainingLoss(info) +% persistent plotObj +% info.State == "start" +% plotObj = animatedline('Color','r'); +% xlabel("Iteration") +% ylabel("Loss") +% title("Training loss evolution") +% elseif info.State == "iteration" +% %addpoints(plotObj,info.Iteration,info.TrainingLoss) +% addpoints(plotObj, info.Iteration, gather(double(info.TrainingLoss))) +% drawnow limitrate nocallbacks +% fprintf('%d \n',info.TrainingLoss) +% end \ No newline at end of file diff --git a/Compute_ClassificationTrainNB.m b/Compute_ClassificationTrainNB.m new file mode 100755 index 0000000..3b33b04 --- /dev/null +++ b/Compute_ClassificationTrainNB.m @@ -0,0 +1,54 @@ +function Model = Compute_ClassificationTrainNB(XTrain,YTrain,cfg) + + + +%% SHUFFLING + +if ~isfield(cfg,'Shuffling') + % do nothing +else + if strcmp(cfg.Shuffling,'YES') + IndRandom = randperm(length(YTrain)); + YTrain = YTrain(IndRandom); + else + % do nothing + end % if strcmp(cfg.Shuffling,'YES') +end % if isfield((cfg,'Shuffling') + + + +%% TRAIN CLASSIFIER: COMPUTE MODEL + +% % Fix random number generator seed +% rng(2) + +% Separate train set into two sets: (1) train set and (2) validation set +Nsamples = size(XTrain,4); +IndRan = randperm(Nsamples); +IndVal = IndRan(1:round(Nsamples*.1)); +IndTra = IndRan(round(Nsamples*.1)+1:end); + +XValid = XTrain(:,:,:,IndVal); +YValid = YTrain(IndVal,1); + +XTrain = XTrain(:,:,:,IndTra); +YTrain = YTrain(IndTra,1); + +% Indices de XTrain: +% 1 y 2 -> Dimensiones de la imagen en pixeles +% 3 -> Canal (para todos es igual a 1) +% 4 -> Etiqueta para contar imágenes + +% Size of input data +Sx = size(XTrain,1); +Sy = size(XTrain,2); +Sz = size(XTrain,3); +Sn = size(XTrain,4); + +% i) flat images in XTrain, ii) remove dimension of length 1, iii) transpose +XTrain_flat = transpose(squeeze(reshape(XTrain,[Sx*Sy,Sz,Sn]))); + +tic +Model.NBModel = fitcnb(XTrain_flat,categorical(YTrain),... + 'ClassNames',{'1','2'}); +Model.Ttrain = toc; \ No newline at end of file diff --git a/Compute_ClassificationTrainSVM.m b/Compute_ClassificationTrainSVM.m new file mode 100755 index 0000000..08af942 --- /dev/null +++ b/Compute_ClassificationTrainSVM.m @@ -0,0 +1,54 @@ +function Model = Compute_ClassificationTrainSVM(XTrain,YTrain,cfg) + + + +%% SHUFFLING + +if ~isfield(cfg,'Shuffling') + % do nothing +else + if strcmp(cfg.Shuffling,'YES') + IndRandom = randperm(length(YTrain)); + YTrain = YTrain(IndRandom); + else + % do nothing + end % if strcmp(cfg.Shuffling,'YES') +end % if isfield((cfg,'Shuffling') + + + +%% TRAIN CLASSIFIER: COMPUTE MODEL + +% % Fix random number generator seed +% rng(2) + +% Separate train set into two sets: (1) train set and (2) validation set +Nsamples = size(XTrain,4); +IndRan = randperm(Nsamples); +IndVal = IndRan(1:round(Nsamples*.1)); +IndTra = IndRan(round(Nsamples*.1)+1:end); + +XValid = XTrain(:,:,:,IndVal); +YValid = YTrain(IndVal,1); + +XTrain = XTrain(:,:,:,IndTra); +YTrain = YTrain(IndTra,1); + +% Indices de XTrain: +% 1 y 2 -> Dimensiones de la imagen en pixeles +% 3 -> Canal (para todos es igual a 1) +% 4 -> Etiqueta para contar imágenes + +% Size of input data +Sx = size(XTrain,1); +Sy = size(XTrain,2); +Sz = size(XTrain,3); +Sn = size(XTrain,4); + +% i) flat images in XTrain, ii) remove dimension of length 1, iii) transpose +XTrain_flat = transpose(squeeze(reshape(XTrain,[Sx*Sy,Sz,Sn]))); + +tic +Model.SVMModel = fitcsvm(XTrain_flat,categorical(YTrain),'KernelFunction','linear',... + 'Standardize',true,'ClassNames',{'1','2'}); +Model.Ttrain = toc; \ No newline at end of file diff --git a/Compute_WaveletMorlet.m b/Compute_WaveletMorlet.m new file mode 100755 index 0000000..dcc7ad7 --- /dev/null +++ b/Compute_WaveletMorlet.m @@ -0,0 +1,186 @@ +function [ WL,timeVec,freqVec] = Compute_WaveletMorlet( data, fs, Fstart, Fstop, BinWidth, MorletWidth, doplot ) +% +% +% +% +% +% INPUT: +% data -> Spatio-temporal data. Nsamples x Nchannels +% fs -> +% freqVec -> +% MorletWidth -> 'width' of the morlet wavelet expressed in cycles +% doplot -> +% +% OUTPUT: +% WL -> time-frequency representation. Nchannels x Nfreq x Nsamples +% +% +% +% ------------------------------------------------------------------------- +% NOTE: This function is equivalent to the Time-Frequency-Representation +% analysis in FieldTrip for averaged trials with the following parameters: +% cfg = []; +% cfg.output = 'pow'; +% cfg.channel = 'all'; +% cfg.method = 'tfr'; % (mtmfft|mtmconvol|mtmwelch|wltconvol|tfr) +% cfg.foi = 1:1:70; +% TFR = ft_freqanalysis(cfg, avgData); + + + +%% INITIALIZE SOME VARIABLES + +% Width of the morlet wavelet expressed in cycles +% MorletWidth = 7; + +% Compute number of channels and number of samples of the input data +[ Nsamples Nchannels ] = size(data); + +% Compute the time vector and the time sampling +timeVec = (0:1:Nsamples-1)/fs; +Ts = 1/fs; + +% Compute the frequency vector +% Fstart = 5; +% Fstop = fs/2; if Fstop>50, Fstop=50; end +Nfreq = round((Fstop-Fstart)/BinWidth)+1; +freqVec = linspace(Fstart,Fstop,Nfreq)'; +% freqVec = (Fstart:BinWidth:Fstop)'; +% Nfreq = length(freqVec); + +% Initialize the WL matriz for all the channels +WL = zeros(Nchannels,Nfreq,Nsamples); +%WLphase = zeros(Nchannels,Nfreq,Nsamples); + + + +%% COMPUTE THE TIME-FREQUENCY REPRESENTATION + +for ichan = 1:Nchannels + + % For the current temporal signal... + Signal = data(:,ichan); + Signal = detrend(Signal,'linear'); + + % For each frequency... + for ifre = 1:Nfreq + + %doplot_test = 0; + %if ifre == ceil(Nfreq/4) + % doplot_test = 1; + %end + + % Compute the morlet wavelet + + Morlet = Compute_Morlet(freqVec(ifre),Ts,MorletWidth); + + % Convolution of the current morlet wavelet with the signal + WLcomplex = conv(Signal,Morlet); + + % Get indexes + li = ceil(length(Morlet)/2); + ls = length(WLcomplex)-floor(length(Morlet)/2); + + % Complex coeffiecients + WLcomplex = WLcomplex(li:ls); + + % Plot wavelet decomposition + if (0) + figure, hold on + subplot(3,1,1), hold on + plot(real(WLcomplex),'r'), plot(imag(WLcomplex),'b'), legend('real','imag'), box on + title(['Frequency: ' num2str(freqVec(ifre)) ' Hz']) + subplot(3,1,2), hold on + plot(abs(WLcomplex)), legend('magnitude'), box on + subplot(3,1,3), hold on + plot(angle(WLcomplex)), legend('phase'), box on + pause + print('wavelet_decomposition','-depsc') + end + + % Compute the magnitud + WLmag = 2*(abs(WLcomplex).^2)/fs; + + % % % Compute the phase + % % WLphase = angle(WLcomplex); + + % Save wavelet decomposition magnitude + WL(ichan,ifre,:) = WLmag; + + end + +end + +if ichan==1 + WL = squeeze(WL); + %WLphase = squeeze(WLphase); +end + + +if (doplot) + % plot the across-channels average wavelet + figure, clf, hold on + [X,Y] = meshgrid(timeVec,freqVec); + Z = zeros(size(X)); + + if ichan==1 + WL2plot = WL; + titulo = 'Time-Frequency representation'; + else + %WL2plot = squeeze(mean(WL,1)); + %titulo = 'Across-channels mean Time-Frequency representation'; + + % Plot scalogram for the first signal + WL2plot = squeeze(WL(1,:,:)); + titulo = 'Time-Frequency representation'; + end + + surface(X,Y,Z,WL2plot,'EdgeColor','none','FaceColor','interp'); + contour(X,Y,WL2plot,6,'w'); + + axis([min(timeVec) max(timeVec) min(freqVec) max(freqVec)]) + xlabel('Time (s)') + ylabel('Frequency (Hz)') + title(titulo) + box on + colorbar +end + + +function Morlet = Compute_Morlet(fi,Ts,MorletWidth) +% COMPUTE THE MORLET WAVELET FOR FREQUENCY "fi" AND TIME "t" +% The wavelet will be normalized so the total energy is 1. +% 'MorletWidth' defines the width of the wavelet. (width>= 5 is suggested) +% +% PILAS: THIS FUNCTION HAS BEEN COPY FROM THE OPEN SOURCE FIELDTRIP TOOLBOX +% Reference: Tallon-Baudry et al., J. Neurosci. 15, 722-734 (1997) + +sf = fi/MorletWidth; +st = 1/(2*pi*sf); + +t = -3.5*st:Ts:3.5*st; + +A = 1/sqrt(st*sqrt(pi)); + +Morlet = A*exp(-t.^2/(2*st^2)).*exp(1i*2*pi*fi.*t); + +% Plot the Morlet wavelet +if (0) + + figure + subplot(3,1,1), hold on + plot(t,real(Morlet),'r','LineWidth',2), plot(t,imag(Morlet),'b','LineWidth',2) + ylabel('Morlet'), legend('Real','Imag'), box on + title(['Frequency = ' num2str(fi) 'Hz']) + subplot(3,1,2), plot(t,abs(Morlet),'.-r'), %axis([-4 4 0 6]) + xlabel('Time (s)'), ylabel('Magnitude') + subplot(3,1,3), plot(t,angle(Morlet),'.-b'), %axis([-4 4 -4 4]) + xlabel('Time (s)'), ylabel('Angle') + + pause +end + + + + + diff --git a/MetaProg_DetectionCNN_1PrepareData.m b/MetaProg_DetectionCNN_1PrepareData.m new file mode 100755 index 0000000..0ef4f1c --- /dev/null +++ b/MetaProg_DetectionCNN_1PrepareData.m @@ -0,0 +1,30 @@ +% ======================================================================= +% Meta Program: For repeatedly running program prog021_DetectionCNN_1PrepareData.m +% ======================================================================= + +% Written by Dr. Manuel David Morales and Dr. Javier M. Antelis +% Any questions? manueld.morales@academicos.udg.mx + +%% RUN prog021_DetectionCNN_1PrepareData.m for each ifile + +% Vary ifile and get data segments with injection and noise + +% 722 files for Data2016_LIGOS6 H1 was used +% 652 files for Data2016_LIGOS6 L1 was used + +for ifile = 1:652 + fprintf('\nProcessing hdf5 file num. %d \n',ifile) + + Twin = 0.25; + try + run prog021_DetectionCNN_1PrepareData; + catch + end + + for Twin = 0.50:0.25:2.00 + try + run prog021_DetectionCNN_1PrepareData; + catch + end + end +end \ No newline at end of file diff --git a/gw_computegetdata.m b/gw_computegetdata.m new file mode 100755 index 0000000..71156ff --- /dev/null +++ b/gw_computegetdata.m @@ -0,0 +1,87 @@ +function data = gw_computegetdata(ligo,doplot) +% Compute h(t) using 2PN aproximation +% Compute h(f) using the stationary phase aproximation + +% ventana de datos, eliminar bordes por cuestiones de discontinuidad + +%% GET DATA SEGMENT + +% Initialize data segment structure +data.st = []; +data.t = []; +% data.timegps = []; +data.sf = []; +data.f = []; +data.psd = []; +data.fpsd = []; + +% Compute the window the data segment +data.st_window = blackman(ligo.NFFT); +% Construct inverse window +data.inv_win = (1./data.st_window); + +% Reject 20 seconds on edges due to window edge effects +tr = ceil(0.15*ligo.segments.Twin); +ind2eli = 1:tr*ligo.fs; +data.inv_win(ind2eli) = 0; +ind2eli = length(data.inv_win)-tr*ligo.fs:length(data.inv_win); +data.inv_win(ind2eli) = 0; + +% Get the data segment +iseg = ligo.segments.seginj; +data.st = ligo.strain(ligo.segments.Sint(iseg,1):ligo.segments.Sint(iseg,2)); +data.t = ligo.timegps(ligo.segments.Sint(iseg,1):ligo.segments.Sint(iseg,2)); +% data.timegps = ligo.timegps(ligo.segments.Sint(iseg,1):ligo.segments.Sint(iseg,2)); +% data.t = data.timegps-data.timegps(1); + +% Compute single-sided FFT of the data segment +data.sf = fft(data.st .* data.st_window,ligo.NFFT); +data.sf = data.sf(1:(ligo.NFFT/2)+1); +data.f = ( linspace(0,ligo.fs/2,length(data.sf)) )'; + +% Save other info +data.fs = ligo.fs; +data.NFFT = ligo.NFFT; + + +% Save injection info +data.injection = ligo.injection; + +% Clear garbage +clear ans tr ind2eli iseg + + +% Plot dtrain data sigle-sided FFT +if (doplot) + + figure, hold on + + subplot(2,1,1), hold on + plot(data.t-data.t(1),data.st,'k') + line([ligo.injection.GPS ligo.injection.GPS]-data.t(1),[-1e-10 1e-10],'Color',[1 0 0]) + %grid on, box on + xlabel('Time (gps)'), ylabel('strain'), title(['Start=' num2str(data.t(1)-data.t(1)) ' | ' 'End=' num2str(data.t(end)-data.t(1)) ' | ' 'Coal=' num2str(ligo.injection.GPS-data.t(1))]) + set(gca,'Xlim',[data.t(1) data.t(end)]-data.t(1),'Ylim',[-1e-15 1e-15]), box on + + subplot(2,1,2), hold on + plot(data.f,abs(data.sf),'k','LineWidth',2) + grid on, box on + xlabel('Frequency [Hz]'), ylabel('Spectral Power [Hz^{-1/2}]'), title('s(f): Strain data sigle-sided FFT') + set(gca,'XLim',[1 data.injection.fs/2],'XScale','Log','YScale','Log') + + +% % GRAFICA PARA PAPER/PRESENTACION: PILAS: poner injection.NFFT a [] +% figure, hold on +% plot(data.f,abs(data.sf),'k','LineWidth',2) +% grid on, box on +% xlabel('Frequency [Hz]'), ylabel('Spectral Power [Hz^{-1/2}]'), title('s(f): Strain data sigle-sided FFT') +% set(gca,'XLim',[1 data.injection.fs/2],'XScale','Log','YScale','Log') +% %set(gca,'XTickLabel',[ ],'YTickLabel',[]) +end + + + +%% FILTER STRAIN DATA + +% [b, a] = butter(4,[template.prm.flow/(ligo.fs/2),ceil(template.prm.fisco)/(ligo.fs/2)],'bandpass'); +% data.stfilt = filter(b,a,data.st); \ No newline at end of file diff --git a/gw_computewhitendata.m b/gw_computewhitendata.m new file mode 100755 index 0000000..23abc1c --- /dev/null +++ b/gw_computewhitendata.m @@ -0,0 +1,235 @@ +function data_out = gw_computewhitendata(data,Twindow,BPFbanda,TW2reject,doplot) +% Compute the whitening of the strain data +% Whitening: transform to freq domain, divide by ASD, then transform back, +% +% Twindow --> +% TW2reject --> + +% just for debug +%Twindow = 8; +%BPFbanda = [20 1000]; +%TW2reject = 16; +%doplot = 1; + + +%% INITIALIZE STRUCTURES + +% if ~isfield(data,'timegps') +% data.timegps = data.t; +% end + +% Raw data +data_raw = []; +data_raw.st = data.st; +data_raw.t = data.t; +data_raw.timegps = data.timegps; +data_raw.sf = []; +data_raw.f = []; +data_raw.psd = []; +data_raw.fpsd = []; +data_raw.fs = data.fs; + +% Whitened data +data_whi = []; +data_whi.st = []; +data_whi.t = data.t; +data_whi.timegps = data.timegps; +data_whi.sf = []; +data_whi.f = []; +data_whi.psd = []; +data_whi.fpsd = []; +data_whi.fs = data.fs; + +% Whitened plus band-pass filtered data +data_out = []; +data_out.st = []; +data_out.t = data.t; +data_out.timegps = data.timegps; +data_out.sf = []; +data_out.f = []; +data_out.psd = []; +data_out.fpsd = []; +data_out.fs = data.fs; + + + +%% COMPUTE DATA WITHENING + + +% ------------------------------ +% Compute the PSD of the raw strain data + +Nsamples = length(data_raw.st); +if mod(Nsamples,2)==1 + error('PILAS PERRO: Nsamples is odd') +end +% Twindow = 4; % Twindow == 4 for GW150914 +NFFT = Twindow*data_raw.fs; + +% Compute the single-sided PSD of the raw strain data +[data_raw.psd,data_raw.fpsd] = pwelch(data_raw.st,hann(NFFT),[],NFFT,data_raw.fs); + +% Compute the Nsamples/2+1 points single-sided PSD of the raw strain data +data_raw.fpsdss = data_raw.fs *(0:1:Nsamples/2)'/Nsamples; +data_raw.psdss = interp1(data_raw.fpsd,data_raw.psd,abs(data_raw.fpsdss)); + +% Compute the Nsamples-points double-sided PSD of the raw strain data +data_raw.fpsdds = data_raw.fs * linspace(0,1,Nsamples)'; +data_raw.psdds = [data_raw.psdss ; fliplr(data_raw.psdss(2:end-1)')']; + + +% ------------------------------ +% Compute Nsamples-points two-sided fft of the raw strain data +%data_raw.sf = fft(data_raw.st .* hamming(length(data_raw.st))); +data_raw.sf = fft(data_raw.st); +data_raw.f = data_raw.fs * linspace(0,1,Nsamples)'; + + +% ------------------------------ +% Compute the frequency-domain whitened strain signal +data_whi.sf = data_raw.sf ./ sqrt(data_raw.psdds); +% data_whi.sf = data_raw.sf ./ ( 1e21 * sqrt(data_raw.psdds /(1/data_raw.fs)/2) ); +% data_whi.sf = data_raw.sf ./ ( 1e23 *sqrt(data_raw.psdds) ); +data_whi.f = data_raw.f; + +% Compute and apply scaling factor to the frequency-domain whitened strain +ind = and(data_raw.fpsd>=90,data_raw.fpsd<=200); +sf = mean(sqrt(data_raw.psd(ind))); +data_whi.sf = sf * data_whi.sf; + +% Compute the time-domain whitened strain signal +data_whi.st = ifft(data_whi.sf); + +% Compute the single-sided PSD of the whitened strain data +[data_whi.psd,data_whi.fpsd] = pwelch(data_whi.st,hann(NFFT),[],NFFT,data_whi.fs); + + + +%% BAND PASS FILTERING OF THE WITHENED STRAIN DATA + +% ------------------------------ +% Desing a band-pass filter of the whitened strain data +[b,a] = butter(4,BPFbanda/(data_whi.fs/2),'bandpass'); +if (0) + [fresponse,ffreq] = freqz(b,a,data_whi.fs/2); + frecuencia = ffreq/pi*data_whi.fs/2; + respuesta = abs(fresponse); + + figure + plot(frecuencia,respuesta,'w','LineWidth',2) + xlabel('Frequency (Hz)'), ylabel('Amplitud'), title('Filter Response') + set(gca,'XScale','Log','YScale','Linear') + axis([0 data_whi.fs/2 0 1.2]); grid on + set(gca,'color',[.2 .5 .3]); +end % if (0) + + +% ------------------------------ +% Band pass filtering of the whitened strain data +data_out.st = filtfilt(b,a,data_whi.st); + +% Reject TR seconds at the beginning and at the end of the pre-proc data +TR = TW2reject; % TW2reject == 2 for GW150914 +data_out.st = data_out.st(TR*data_out.fs+1:end-TR*data_out.fs,1); +data_out.t = data_out.t(TR*data_out.fs+1:end-TR*data_out.fs,1); +data_out.t = data_out.t - data_out.t(1); % OJO: esta line si GW150 +data_out.timegps = data_out.timegps(TR*data_out.fs+1:end-TR*data_out.fs,1); + +data_out.TW2reject = TW2reject; + +% % Compute Nsamples-points two-sided fft filtered and whitened strain data +% data_out.sf = fft(data_out.st); +% data_out.f = data_out.fs*linspace(0,1,Nsamples)'; + +% Compute the single-sided PSD of the filtered and whitened strain data +[data_out.psd,data_out.fpsd] = pwelch(data_out.st,hann(NFFT),[],NFFT,data_out.fs); + + + +%% PLOT FOR DEBUGGING +if (doplot) + + % Time domain signals + figure + + subplot(3,1,1) + plot(data_raw.t,data_raw.st,'-','Color',[1.0 0.0 0.0],'LineWidth',1), hold on + set(gca,'XLim',[data_raw.t(1) data_raw.t(end)]), %set(gca,'XLim',[-0.1 0.05]+16.4414), + %set(gca,'YLim',[-1e-18 1e-18]), + xlabel('Time (s)'), ylabel('Strain (unitless)') + title('s_{raw}(t)') + + subplot(3,1,2) + plot(data_whi.t,data_whi.st,'-','Color',[0.0 0.5 0.0],'LineWidth',1), hold on + set(gca,'XLim',[data_whi.t(1) data_whi.t(end)]), %set(gca,'XLim',[-0.1 0.05]+16.4414), + set(gca,'YLim',[-14e-21 14e-21]), + xlabel('Time (s)'), ylabel('Strain (unitless)') + title('s_{white}(t)') + + subplot(3,1,3) + plot(data_out.t+16,data_out.st,'-','Color',[0.0 0.0 1.0],'LineWidth',1), hold on + set(gca,'XLim',[data_whi.t(1) data_whi.t(end)]), %set(gca,'XLim',[-0.1 0.05]+16.4414), + set(gca,'YLim',[-9e-21 9e-21]), + xlabel('Time (s)'), ylabel('Strain (unitless)') + title('s_{white+bpf}(t)') + + print('t_signals','-depsc') + + % % Magnitude spectrum + % figure + % + % plot(data_raw.f,abs(data_raw.sf),'-','Color',[1.0 0.0 0.0],'LineWidth',1), hold on + % set(gca,'XScale','Linear','YScale','Log') + % xlabel('Frequency (Hz)'), ylabel('|S_{raw}(f)| (?)') + % title('Magnitude Spectrum') + % grid on, box on + % + % plot(data_whi.f,abs(data_whi.sf),'-','Color',[0.0 0.5 0.0],'LineWidth',1), hold on + % set(gca,'XScale','Linear','YScale','Log') + % xlabel('Frequency (Hz)'), ylabel('|S_{white}(f)| (?)') + % title('Magnitude Spectrum') + % grid on, box on + % + % plot(data_out.f,abs(data_out.sf),'-','Color',[0.0 0.0 1.0],'LineWidth',1), hold on + % set(gca,'XScale','Linear','YScale','Log') + % xlabel('Frequency (Hz)'), ylabel('|S_{white+bpf}(f)| (?)') + % title('Magnitude Spectrum') + % grid on, box on + + + % Amplitude Spectral Density + figure + plot(data_raw.fpsd,sqrt(data_raw.psd),'-','Color',[1.0 0.0 0.0],'LineWidth',1), hold on + set(gca,'XScale','Log','YScale','Log') + %xlabel('Frequency (Hz)'), ylabel('ASD (Hz^{-1/2})') + %title('ASD_{raw}(f)') + grid on, box on + + plot(data_whi.fpsd,sqrt(data_whi.psd),'-','Color',[0.0 0.5 0.0],'LineWidth',1), hold on + set(gca,'XScale','Log','YScale','Log') + %xlabel('Frequency (Hz)'), ylabel('ASD (Hz^{-1/2})') + %title('ASD_{white}(f)') + grid on, box on + + plot(data_out.fpsd,sqrt(data_out.psd),'-','Color',[0.0 0.0 1.0],'LineWidth',1), hold on + set(gca,'XScale','Log','YScale','Log') + %xlabel('Frequency (Hz)'), ylabel('ASD (Hz^{-1/2})') + %title('ASD_{white+bpf}(f)') + grid on, box on + + xlim([10^(-1) 3*10^3]) + ylim([10^(-34) 10^(-15)]) + + xlabel('Frequency (Hz)'), ylabel('ASD (Hz^{-1/2})') + title('Amplitude Spectral Density') + + legend({'Raw','White','White+BPF'},'Location','best','FontSize',12) + + print('ASD','-depsc') + +end % if (doplot) + + + + + diff --git a/gw_getinjectioninfoS6.m b/gw_getinjectioninfoS6.m new file mode 100755 index 0000000..d032c73 --- /dev/null +++ b/gw_getinjectioninfoS6.m @@ -0,0 +1,227 @@ +function injection = gw_getinjectioninfoS6(INTERFEROMETER,Ni) +% close all; clear all; clc; Ni=2; INTERFEROMETER='H1'; +% +% PILAS: se asume que el Ni injection information coincide con el Ni +% datafile contenido en el folder Data_LIGOS6 + + +%%% Read injection information + +% GPS time of the end of the injection, or the "merger" time, in GPS seconds +% Mass, in solar units, of each compact object +% Distance to the simulated source in Mpc +% Expected SNR of the injection +% Recovered SNR of the injection +% SNRs from 1-5 are typical for times with no signal. + +m = csvread([INTERFEROMETER '_s6cbc_simple.txt'],1,0); % Data obtained from https://www.gw-openscience.org/s6hwcbc/ +GPS = m(Ni,1); +M1 = m(Ni,2); +M2 = m(Ni,3); +D = m(Ni,4); +Exp_SNR = m(Ni,5); +Rec_SNR = m(Ni,6); + +%%% New section for burst injections + +% GPS_burst, time of the injection +% hrss, amplitude of the burst (root-sum-squared) +% waveform-freq-q-tau-bandwidth: +% -- waveform, name of the waveform +% -- freq, frequency +% -- q, quality factor (tau*f0) +% -- tau, + +%n = csvread([INTERFEROMETER '_s6burst_simple'],1,0); % Data obtained from https://www.gw-openscience.org/s6hwburst/ +%GPS_burst = n(Ni,1); +%hrss = n(Ni,2); +%waveform = textscan ...n(Ni,3); +%D = n(Ni,4); +%Exp_SNR = n(Ni,5); +%Rec_SNR = n(Ni,6); + + + +% Get filenames +if strcmp(INTERFEROMETER,'H1') + hdf5Files = dir(['/media/manuel/ADATA HD710 PRO/Files CUVALLES/Codes Manuel CUVALLES/Datasets/Data2016_LIGOS6/' INTERFEROMETER '/H-' INTERFEROMETER '*.hdf5']); + %['C:\_DataSets\Data2017_LIGOS6\' INTERFEROMETER '\H-' INTERFEROMETER '*.hdf5'] +elseif strcmp(INTERFEROMETER,'L1') + hdf5Files = dir(['/media/manuel/ADATA HD710 PRO/Files CUVALLES/Codes Manuel CUVALLES/Datasets/Data2016_LIGOS6/' INTERFEROMETER '/L-' INTERFEROMETER '*.hdf5']); +end + +% hdf5Files + +% Save injection info +injection.GPS = GPS; +injection.IFO = INTERFEROMETER; +injection.M1 = M1; +injection.M2 = M2; +injection.D = D; +injection.Log = []; % que es esto?? +injection.Exp_SNR = Exp_SNR; +injection.Rec_SNR = Rec_SNR; +injection.filename = hdf5Files(Ni).name; + +injection.PNorder = 3.5; % que es esto? + + +%% PLOT INJECTION INFO + +if (0) + + % Limpiar workspace + clear all; close all; clc + + RUN = 'S6'; % (S5|S6|O1CBC) + IFO = 'L1'; % (L1|H1) + + % Load data + if strcmp(RUN,'S5') + % Read data + [~,~,M1,M2,D,LOG,~,~] = textread([IFO '_cleanlog_cbc.txt'],'%d %s %f %f %f %s %f %f'); % que es este archivo? + + % Compute total mass + M = M1 + M2; + + % % Eliminar si D==0 + % M1(D==0) = []; + % M2(D==0) = []; + % M(D==0) = []; + % D(D==0) = []; + + % Eliminar si D==0 or if injection is marked as not 'Successful' + Ind2Eli = or((~strcmp(LOG,'Successful')),(D==0)); % Number of discharged files: sum(Ind2Eli) + M1(Ind2Eli) = []; + M2(Ind2Eli) = []; + M(Ind2Eli) = []; + D(Ind2Eli) = []; + + fprintf([RUN ' - ' IFO '\r']) + fprintf('Total number of data blocks with injections: %d \r',length(Ind2Eli)) + fprintf('Discharged data blocks (unsuccessful or missinf info): %d \r',sum(Ind2Eli)) + fprintf('Numer of used data blocks: %d \r',length(Ind2Eli)-sum(Ind2Eli)) + + % Contruct data for m1=m2=1.4 + ind = and(M1==1.4,M2==1.4); + M1_14 = M1(ind); M1_14 = M1_14 + linspace(-0.75,0.75,length((M1_14)))'; M1_14 = M1_14(randperm(length(M1_14))); + M2_14 = M2(ind); M2_14 = M2_14 + linspace(-0.75,0.75,length((M2_14)))'; + D_14 = D(ind); + M_14 = M(ind); + + % Contruct data for m1=m2=3.0 + ind = and(M1==3.0,M2==3.0); + M1_03 = M1(ind); M1_03 = M1_03 + linspace(-0.75,0.75,length((M1_03)))'; M1_03 = M1_03(randperm(length(M1_03))); + M2_03 = M2(ind); M2_03 = M2_03 + linspace(-0.75,0.75,length((M2_03)))'; + D_03 = D(ind); + M_03 = M(ind); + + % Contruct data for m1=m2=10 + ind = and(M1==10,M2==10); + M1_10 = M1(ind); M1_10 = M1_10 + linspace(-0.75,0.75,length((M1_10)))'; M1_10 = M1_10(randperm(length(M1_10))); + M2_10 = M2(ind); M2_10 = M2_10 + linspace(-0.75,0.75,length((M2_10)))'; + D_10 = D(ind); + M_10 = M(ind); + + % % Contruct data for m1=1.12||m2=5.08 + % ind = or(and(M1==1.12,M2==5.08),and(M1==5.08,M2==1.12)); + % M1_12508 = M1(ind); %M1_12508 = M1_12508 + linspace(-0.75,0.75,length((M1_12508)))'; M1_12508 = M1_12508(randperm(length(M1_12508))); + % M2_12508 = M2(ind); %M2_12508 = M2_12508 + linspace(-0.75,0.75,length((M2_12508)))'; + % D_12508 = D(ind); + + % Contruct data for m1=1.4||m2=10 + ind = and(M1==1.4,M2==10); + M1_1410 = M1(ind); M1_1410 = M1_1410 + linspace(-0.75,0.75,length((M1_1410)))'; M1_1410 = M1_1410(randperm(length(M1_1410))); + M2_1410 = M2(ind); M2_1410 = M2_1410 + linspace(-0.75,0.75,length((M2_1410)))'; + D_1410 = D(ind); + M_1410 = M(ind); + + % Apeend data + M1 = [ M1_14-1.4+2 ; M1_03-3+4 ; M1_10-10+6; M1_1410-1.4+2]; + M2 = [ M2_14-1.4+2 ; M2_03-3+4 ; M2_10-10+6; M2_1410-10+6]; + D = [ D_14 ; D_03 ; D_10; D_1410]; + M = [ M_14 ; M_03 ; M_10; M_1410]; + + elseif strcmp(RUN,'S6') + % Read data + m = csvread([IFO '_s6cbc_simple.txt'],1,0); + + m = m(1:end-6,:); + % % Eliminar si D==0 + % M1(D==0) = []; + % M2(D==0) = []; + % M(D==0) = []; + % D(D==0) = []; + + % Get info + M1 = m(:,2); + M2 = m(:,3); + D = m(:,4); + + % Compute total mass + M = M1 + M2; + + elseif strcmp(RUN,'O1CBC') + + else + error('PILAS: unknown run') + end % if strmcp(RUN,'S5') + clear ans m + + + % Plot distribution of masses. Option 1 + figure + + scatter(M1,M2,12*M,1*D,'o','filled','MarkerEdgeColor','k') % + xlabel('$m_1$','Interpreter','Latex','FontSize',12) + ylabel('$m_2$','Interpreter','Latex','FontSize',12) + title([RUN ' - ' IFO]) + box on + if strcmp(RUN,'S5') + axis([1 7 1 7]) + set(gca,'Xtick',[2 4 6],'Ytick',[2 4 6]) + set(gca,'XtickLabel',[1.4 3 10],'YtickLabel',[1.4 3 10]) + + h=colorbar; + set(h,'Location','East','YLim',[0 150],'YTick',[0.1 50 100 150]) + set(h,'Position',[0.8476 0.4148 0.0476 0.2110]) + text(6,4,'$D$','Interpreter','Latex') + elseif strcmp(RUN,'S6') + axis([0 31 0 31]) + + h=colorbar; + set(h,'Location','East','YLim',[10 90],'YTick',20:20:80) + set(h,'Position',[0.8476 0.3548 0.0476 0.3619]) + text(26,16,'$D$','Interpreter','Latex') + end +% return + + + + + + % Grafica interna + if strcmp(RUN,'S5') + handaxes2 = axes('Position', [0.6 0.18 0.2 0.2]); + x = [2.8 4.2 6 8 11.4 15 20]; + h = hist(M,x); + bar(x,h) + elseif strcmp(RUN,'S6') + handaxes2 = axes('Position', [0.55 0.70 0.2 0.2]); + hist(M); + end + h = findobj(gca,'Type','patch'); + set(h,'FaceColor',[0.8 0.8 0.8],'EdgeColor','k') + xlabel('$M$','Interpreter','Latex','FontSize',8) + ylabel('N','Interpreter','Latex','FontSize',8) + box on, set(handaxes2,'FontSize',8) + if strcmp(RUN,'S5') + set(gca,'XLim',[0 22]) + set(gca,'Xtick',[2.8 6 11.4 20],'Ytick',[0 100 200 300 400]) + elseif strcmp(RUN,'S6') + set(gca,'Xtick',[0 10 20 30 40],'Ytick',[0 100 200 300]) + end + + +end % if (0) +% clear ans RUN IFO M \ No newline at end of file diff --git a/gw_getsegmentsinfoOVERLAP.m b/gw_getsegmentsinfoOVERLAP.m new file mode 100755 index 0000000..eee2b53 --- /dev/null +++ b/gw_getsegmentsinfoOVERLAP.m @@ -0,0 +1,43 @@ +function Segments = gw_getsegmentsinfoOVERLAP(Tblock,Twin,Tove,fs) +% +% % Funciona cualquier overlap +% % Example: +% Tblock = 4096; +% Twin = 32; +% Tove = 4; +% fs = 4096; + +% Initialize +Segments = []; + +% Duration of the segment +Segments.Tblock = Tblock; + +% Duration of the data window and overlap (seconds) +Segments.Twin = Twin; +Segments.Tove = Tove; + +% Time ini and Time end of each window (seconds) +Tini = (0:Twin-Tove:Tblock-Twin)'; +Tend = Tini+Segments.Twin; +% [Tini Tend] + +% Sample ini and Sample end of each window (sample) +% remark: sample = index for each data point in time vector +Sini = Tini*fs+1; +Send = Tend*fs+0; + +% Time and samples of each window (interval) +Segments.Tint = [Tini Tend]; +Segments.Sint = [Sini Send]; +% [Sini-Send]/fs + +% Number of segments +Segments.Nseg = size(Segments.Sint,1); + +% Clear garbage +clear ans Tini Tend Sini Send + + + + diff --git a/gw_readligo.m b/gw_readligo.m new file mode 100755 index 0000000..2d00ed4 --- /dev/null +++ b/gw_readligo.m @@ -0,0 +1,310 @@ +function ligo = gw_readligo(RUN,IFO,Ni,doplot) +% close all; clear all; clc; doplot = 1; Ni=1; IFO='H1'; RUN='S5'; + + + + +%% GET INJECTION INFORMATION + +% Load injection data and filename con the Ni-th file +if strcmp(RUN,'S5') + ligo.injection = gw_getinjectioninfoS5(IFO,Ni); + ligo.path = '/media/manuel/ADATA HD710 PRO/Files CUVALLES/Codes Manuel CUVALLES/Datasets/Data2016_LIGOS6/'; + ligo.filename = ligo.injection.filename; + +elseif strcmp(RUN,'S6') + ligo.injection = gw_getinjectioninfoS6(IFO,Ni); + ligo.path = ['/media/manuel/ADATA HD710 PRO/Files CUVALLES/Codes Manuel CUVALLES/Datasets/Data2016_LIGOS6/' IFO '/']; + ligo.filename = ligo.injection.filename; + + + + % **************************** + % GW150914 +elseif strcmp(RUN,'GW150914_32') + ligo.path = 'C:\_DataSets\GW150914\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_4_V2-1126259446-32.hdf5']; + +elseif strcmp(RUN,'GW150914_4096') + ligo.path = 'C:\_DataSets\GW150914\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_4_V1-1126257414-4096.hdf5']; + + + + % **************************** + % LVT151012 +elseif strcmp(RUN,'LVT151012_32') + ligo.path = 'C:\_DataSets\LVT151012\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_4_V2-1128678884-32.hdf5']; +elseif strcmp(RUN,'LVT151012_4096') + ligo.path = 'C:\_DataSets\LVT151012\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_4_V2-1128676852-4096.hdf5']; + + + + % **************************** + % GW151226 +elseif strcmp(RUN,'GW151226_32') + ligo.path = 'C:\_DataSets\GW151226\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_4_V2-1135136334-32.hdf5']; + +elseif strcmp(RUN,'GW151226_4096') + ligo.path = 'C:\_DataSets\GW151226\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_4_V2-1135136228-4096.hdf5']; + + + + % **************************** + % GW170104 +elseif strcmp(RUN,'GW170104_32') + ligo.path = 'C:\_DataSets\GW170104\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_4_V1-1167559920-32.hdf5']; + +elseif strcmp(RUN,'GW170104_4096') + ligo.path = 'C:\_DataSets\GW170104\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_4_V1-1167557888-4096.hdf5']; + + + + % **************************** + % GW170608 +elseif strcmp(RUN,'GW170608_32') + ligo.path = 'C:\_DataSets\GW170608\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_C01_4_V1-1180922478-32.hdf5']; + +elseif strcmp(RUN,'GW170608_4096') + ligo.path = 'C:\_DataSets\GW170608\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_C01_4_V1-1180920446-4096.hdf5']; + + + + % **************************** + % GW170814 +elseif strcmp(RUN,'GW170814_32') + ligo.path = 'C:\_DataSets\GW170814\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_CLN_4_V1-1186741845-32.hdf5']; +elseif strcmp(RUN,'GW170814_4096') + ligo.path = 'C:\_DataSets\GW170814\'; + %ligo.filename = [IFO(1) '-' IFO '_LOSC_C00_4_V1-1186739813-4096.hdf5']; + ligo.filename = [IFO(1) '-' IFO '_LOSC_CLN_4_V1-1186740069-3584.hdf5']; + + + + % **************************** + % GW170817 +elseif strcmp(RUN,'GW170817_32') + error('PILAS: This LIGO data is not available') +elseif strcmp(RUN,'GW170817_4096') + ligo.path = 'C:\_DataSets\GW170817\'; + ligo.filename = [IFO(1) '-' IFO '_LOSC_C00_4_V1-1187006834-4096.hdf5']; + + + + % **************************** + % LIGO O1 +elseif strcmp(RUN,'O1') + % Get the path of the O1 LIGO data + ligo.path = ['D:\_DataSets\' 'LIGOO1' '\']; + + % Get the filename of all existing HDF5 files + AllFilenames = dir([ligo.path IFO(1:1) '-' IFO '_LOSC_4_V1-*-4096.hdf5']); + + % Save the current filename + ligo.filename = AllFilenames(Ni).name; + + + + % **************************** + % LIGO O1 WITH INJECTIONS +elseif strcmp(RUN(1:6),'LIGOO1') + + ligo.injection = gw_getinjectioninfoO1CBC(RUN,IFO); + ligo.path = ['D:\_DataSets\' RUN(1:6) '\']; % PC TEC + %ligo.path = ['/Users/gravwaves/Documents/_DataSets/' RUN(1:6) '/']; % MAC PERSONAL + ligo.filename = ligo.injection.filename; + + + + % **************************** +else + error('PILAS: Unknown LIGO data') + +end + + + +%% LOAD DATA + +% Visualize info from the file +%h5disp([path filename]) + +% Sampling frequency and period +% [ligo.path ligo.filename] +ligo.ts = hdf5read([ligo.path ligo.filename],'/strain/Strain','Xspacing'); % delta t +ligo.fs = 1/ligo.ts; % frecuencia +ligo.Npoints = double(hdf5read([ligo.path ligo.filename],'/strain/Strain','Npoints')); % j (posicion en el tiempo) +ligo.Tblock = ligo.Npoints/ligo.fs; % periodo + +% Get the start and end time of the data +ligo.gpsini = double(hdf5read([ligo.path ligo.filename],'/strain/Strain','Xstart')); +ligo.gpsend = ligo.gpsini + ligo.Tblock-1*ligo.ts; + +% Read the strain values +ligo.strain = hdf5read([ligo.path ligo.filename],'/strain/Strain'); +ligo.timegps = linspace(ligo.gpsini,ligo.gpsend,length(ligo.strain))'; +ligo.timesec = ligo.timegps - ligo.timegps(1); + +% Plot strain data +if (doplot) + if isfield(ligo,'injection') + % % GRAFICA PARA PAPER/PRESENTACION + % figure, clf, set(gcf,'Position',[19 336 1227 289]) + % plot(ligo.timegps,ligo.strain,'k','LineWidth',1), hold on + % line([ligo.injection.GPS ligo.injection.GPS],[-1e-10 1e-10],'Color',[1 0 0]) + % %xlabel('Time (gps)'), ylabel('strain'), title(['Start=' num2str(ligo.timegps(1)) ' | ' 'End=' num2str(ligo.timegps(end)) ' | ' 'Coal=' num2str(ligo.injection.GPS)]) + % set(gca,'Xlim',[ligo.timegps(1) ligo.timegps(end)],'Ylim',[-1.5e-16 1.5e-16]), + % set(gca,'XTickLabel',[ ],'YTickLabel',[]), + + % OTRA GRAFICA + figure(1), clf + subplot(2,1,1), hold on + plot(ligo.timegps,ligo.strain), line([ligo.injection.GPS ligo.injection.GPS],[-1e-10 1e-10],'Color',[1 0 0]) + xlabel('Time (gps)'), ylabel('strain'), title(['Start=' num2str(ligo.timegps(1)) ' | ' 'End=' num2str(ligo.timegps(end)) ' | ' 'Coal=' num2str(ligo.injection.GPS)]) + set(gca,'Xlim',[ligo.timegps(1) ligo.timegps(end)],'Ylim',[-1e-15 1e-15]), box on + subplot(2,1,2), hold on + plot(ligo.timesec,ligo.strain), line([ligo.injection.GPS ligo.injection.GPS]-ligo.gpsini,[-1e-10 1e-10],'Color',[1 0 0]) + xlabel('Time (s)'), ylabel('strain'), title(['Start=' num2str(ligo.timesec(1)) ' | ' 'End=' num2str(ligo.timesec(end)) ' | ' 'Coal=' num2str(ligo.injection.GPS-ligo.gpsini)]) + set(gca,'Xlim',[ligo.timesec(1) ligo.timesec(end)],'Ylim',[-1e-15 1e-15]), box on + else + % figure(1), clf + % + % subplot(2,1,1), hold on + % plot(ligo.timegps,ligo.strain,'b'), + % xlabel('Time (gps)'), ylabel('strain'), title(['Start=' num2str(ligo.timegps(1)) ' | ' 'End=' num2str(ligo.timegps(end)) ]) + % set(gca,'Xlim',[ligo.timegps(1) ligo.timegps(end)]) + % %set(gca,'Ylim',[-1e-18 1e-18]), + % box on + % + % subplot(2,1,2), hold on + % plot(ligo.timesec,ligo.strain,'b'), + % xlabel('Time (s)'), ylabel('strain'), title(['Start=' num2str(ligo.timesec(1)) ' | ' 'End=' num2str(ligo.timesec(end)) ]) + % set(gca,'Xlim',[ligo.timesec(1) ligo.timesec(end)]) + % %set(gca,'Ylim',[-1e-18 1e-18]), + % box on + + figure + + % Strain data + subplot(2,1,1), hold on + plot(ligo.timesec,ligo.strain,'b'), + xlabel('Time (s)'), ylabel('Strain (unitless)'), title('s(t): strain signal') + set(gca,'Xlim',[ligo.timesec(1) ligo.timesec(end)]) + %set(gca,'Ylim',[-1e-18 1e-18]), + box on + + % Amplitude Spectral Density + PSD = gw_computePSD(ligo,1,0); + + + subplot(2,1,2), hold on + plot(PSD.fpsd,PSD.asd,'b','LineWidth',1) + plot(PSD.fpsd,PSD.asdref,'k','LineWidth',1) + xlabel('Frequency (Hz)'), ylabel('ASD (Hz^{-1/2})') + title('ASD(f): amplitude spectral density') + set(gca,'XScale','Log','YScale','Log') + set(gca,'XLim',[10 ligo.fs/2]), set(gca,'YLim',[1e-24 1e-19]) + grid on, box on + + end % if ~isempty(ligo.injection) +end % if (1) + + + +%% COMPUTE SEGMENTS INFORMATION (VALID FOR S5 AND S6) + +if isfield(ligo,'injection') + % Duration of the data segment and overlap (seconds) + ligo.segments.Twin = 128; % even and power of two + ligo.segments.Tove = ligo.segments.Twin/2; + + % Number of segments + ligo.segments.Nseg = 2*(ligo.Tblock/ligo.segments.Twin)-1; + + % % Debugging: Compute Tblock + % Tblock = ((ligo.segments.Nseg-1)*(ligo.segments.Tove*ligo.fs)+ligo.segments.Twin*ligo.fs)*ligo.ts + + % Time ini and Time end of each segment (seconds) + Tini = (1:ligo.segments.Tove:ligo.Tblock-ligo.segments.Tove)'; + Tend = Tini+ligo.segments.Twin-1; + + % Sample ini and Sample end of each segment (sample) + Sini = (Tini-1)*ligo.fs+1; + Send = (Tend-0)*ligo.fs+0; + + % Time and samples of each interval + ligo.segments.Tint = [Tini Tend]; + ligo.segments.Sint = [Sini Send]; + + % Get segment where the GW was injected + tcoal = ligo.injection.GPS - ligo.gpsini; + d = ligo.segments.Tint - tcoal; + md = abs(d(:,1)+d(:,2)); + [~, ligo.segments.seginj] = min(md); + +else + % do nothing + +end % if isfield(ligo,'injection') + + + +%% INITIALIZE OTHER THINGS (VALID FOR S5 AND S6) + +if isfield(ligo,'injection') + ligo.NFFT = ligo.segments.Twin * ligo.fs; + ligo.injection.fs = ligo.fs; + ligo.injection.NFFT = ligo.NFFT; + +else + % do nothing + +end % if isfield(ligo,'injection') + + + +%% VERIFICAR QUE EL GPS INJETION ESTE DENTRO DEL GPS DE LOS DATOS LEIDOS + +if isfield(ligo,'injection') + if ligo.injection.GPS-ligo.gpsini>0 && ligo.gpsend-ligo.injection.GPS>0 + % Todo bien + fprintf('FINO: The strain data contains an injection\n') + else + ligo %#ok + ligo.injection + error('PILAS: GPSinjection is outside GPSdata') + end % if ligo.injection.GPS-ligo.gpsini>0 && ligo.gpsend-ligo.injection.GPS>0 + +else + % do nothing + +end % if isfield(ligo,'injection') + + + +%% CHECK WHETHER THE INJECTION WAS SUCESSFULL (VALID FOR S5 AND S6) + +if isfield(ligo,'injection') + if strcmp(ligo.injection.Log,'Injection-Compromised') || strcmp(ligo.injection.Log,'Did-not-execute') + error('PILAS: Injection compromised. Not posible to evaluate CBC hardware injection') + else + % Injection was sucesfull + + end % if strcmp(ligo.injection.Log,'Injection-Compromised') || strcmp(ligo.injection.Log,'Did-not-execute') + +else + % do nothing + +end % if isfield(ligo,'injection') + + + diff --git a/prog021_DetectionCNN_1PrepareData.m b/prog021_DetectionCNN_1PrepareData.m new file mode 100755 index 0000000..88adb06 --- /dev/null +++ b/prog021_DetectionCNN_1PrepareData.m @@ -0,0 +1,407 @@ +% ================================== +% Get data segments with injection and noise +% ================================== + +% Written by Dr. Manuel David Morales and Dr. Javier M. Antelis +% Any questions? manueld.morales@academicos.udg.mx + +%% INITIALIZE + +% ---------------------------------------- +% Comment the next three lines when the "Meta Program" is executed +clearvars +close all +clc + +% ---------------------------------------- +% Select RUN, IFO, file and Twin +RUN = 'S6'; % (S5|S6) % RUN +IFO = 'L1'; % (H1|L1) % Interferometer + +% Comment the next two lines just when the "Meta Program" is executed +ifile = 160; % hdf5 file (1:722) for H1, (1:652) for L1 +Twin = 0.25; % (0.25|0.50|1.00|1.50|2.00|...) % Window's width + + + +%% LOAD DATA AND GET THE SEGMENT OF 128S CONTANING AN INJECTION + +% Load ligo data +% last index 0: no plot, 1: plot +ligo = gw_readligo(RUN,IFO,ifile,0); + +% Get a 128s-long data segment +data = gw_computegetdata(ligo,0); + +% Save path and filename +data.path = ligo.path; +data.filename = ligo.filename; + +% Clear garbage +clear ans ligo + + + +%% VERIFY THAT ALL THE DATA IS OK + +% Check if the strain data in the segment contain NaN values +if isnan(sum(data.st)) + % The strain data in the segment contain NaN values + save([data.path 'FileStrainWithNaN_' num2str(ifile)],'ifile') + fprintf('data contains NaN') +else + % Do nothing, all strain data in the segment in OK + +end % if isnan(sum(data.st)) + + + +%% ARREGLAR "TIME", "TIMEGPS" AND "TIME INJECTION" + +% Create "timegps" vector en unidades de gps +data.timegps = data.t; + +% Create "t" vector en unidades de seconds +data.t = data.t - data.timegps(1); + +% Calculat el tcoal inyectado en unidades de seconds +data.injection.t = data.injection.GPS - data.timegps(1); + + + +%% WHITENING AND BAND PASS FILTERING OF THE STRAIN DATA + +% Apply whitening and band pass filterig +datawhitened = gw_computewhitendata(data,8,[20 1000],16,0); +% Cantidad entre [ ] define el filtro + +% Save injection informarion in the "datawhitened" variable +datawhitened.injection = data.injection; + +% Adjust tcoal (as the whitening rejected a section of data) +if (1) + % Adjust the tcoal inyectado en unidades de seconds para la TW actual + datawhitened.injection.t = datawhitened.injection.t - datawhitened.TW2reject; +else + % Recover the eliminated TW2reject in the time vector + % datawhitened.t = datawhitened.t + datawhitened.TW2reject; +end + +% Save path and filename +datawhitened.path = data.path; +datawhitened.filename = data.filename; + +% Plot for debugging +if (0) + figure, clf + + subplot(3,1,1), hold on + plot(data.t,data.st,'b'), hold on + %if ligo.injection.PNorder==2 + % plot(template.t+data.injection.t-template.t(end),1*template.st,'r') + %end + line([data.injection.t data.injection.t],[-1.2*max(abs(data.st)) 1.2*max(abs(data.st))],'Color',[1 0 0]) + xlabel('Time (s)'), ylabel('s(t)'), title(['Data segment ' '']), box on, + set(gca,'Xlim',[data.t(1) data.t(end)]), set(gca,'Ylim',[-1.2*max(abs(data.st)) 1.2*max(abs(data.st))]), %set(gca,'Ylim',[-1e-15 1e-15]) + title(['Injection at t=' num2str(data.injection.t)]) + + subplot(3,1,2), hold on + plot(datawhitened.t+datawhitened.TW2reject,datawhitened.st,'b'), hold on + %if ligo.injection.PNorder==2 + % plot(template.t+datawhitened.injection.t-template.t(end)+0.06,2*template.st,'r') + %end + line([datawhitened.injection.t datawhitened.injection.t]+datawhitened.TW2reject,[-1.2*max(abs(data.st)) 1.2*max(abs(data.st))],'Color',[1 0 0]) + xlabel('Time (s)'), ylabel('s(t)'), title(['Data segment ' '']), box on, + title(['Injection at t=' num2str(data.injection.t)]) + set(gca,'Xlim',[data.t(1) data.t(end)]), + set(gca,'Ylim',[-1.2*max(abs(datawhitened.st)) 1.2*max(abs(datawhitened.st))]), %set(gca,'Ylim',[-1e-15 1e-15]) + + subplot(3,1,3), hold on + plot(datawhitened.t+0*datawhitened.TW2reject,datawhitened.st,'b'), hold on + %if ligo.injection.PNorder==2 + % plot(template.t+datawhitened.injection.t-template.t(end)+0.06,2*template.st,'r') + %end + line([datawhitened.injection.t datawhitened.injection.t]+0*datawhitened.TW2reject,[-1.2*max(abs(data.st)) 1.2*max(abs(data.st))],'Color',[1 0 0]) + xlabel('Time (s)'), ylabel('s(t)'), title(['Data segment ' '']), box on, + title(['Injection at t=' num2str(datawhitened.injection.t)]) + set(gca,'Xlim',[data.t(1) data.t(end)]), + set(gca,'Ylim',[-1.2*max(abs(datawhitened.st)) 1.2*max(abs(datawhitened.st))]), %set(gca,'Ylim',[-1e-15 1e-15]) + + return + +end % if (0) + +% Clear garbage +clear ans data + + + +%% COMPUTE SEGMENT INFORMATION + +% Get segments info +Segments = gw_getsegmentsinfoOVERLAP(round(datawhitened.t(end)-datawhitened.t(1)),Twin,3*Twin/4,4096); + +% Identify the two/four/X (if Tove=50/75/X%) consecutive segments with the injection +Segments.InIn = and(Segments.Tint(:,1)=datawhitened.injection.t); + +% Stop if there is no segments with injection +if sum(Segments.InIn)==0 + error('PILAS: No hay injeccion en ninguno de los segmentos') +elseif sum(Segments.InIn)==4 % (2s:Tove=50% | 4s:Tove=75%) + % do nothing +else + error('PILAS: El numero de segmentos con injeccion no es correcto') +end +clear ans Sini Send Tend Tini + + + +%% DEBUGING: Plot strain and wavelet for each segment (time resolved) + +if (0) + + inicio = find(Segments.InIn); + inicio = inicio(1)-5; + for i=inicio:Segments.Nseg + + % Get data segments + datasegment = datawhitened.st(Segments.Sint(i,1):Segments.Sint(i,2)); + t = datawhitened.t(Segments.Sint(i,1):Segments.Sint(i,2)); + + % Normalize strain + datasegment = 1e20 * datasegment; + + % Compute time-frequency representation based on Morlet wavelet + [tfr,aja,ftfr] = Compute_WaveletMorlet(datasegment,datawhitened.fs,1,500,10,7,1); + + + % Plot for debugging + if (0) + figure(12), clf + + subplot(3,1,1) + area(Segments.Tint(i,:)+0*datawhitened.t(1),[2 2],'BaseValue',-2,'FaceColor',[0.93 0.93 0.93],'LineStyle','none'), hold on + plot(datawhitened.t,1e20 *datawhitened.st,'b','LineWidth',1), hold on + line([datawhitened.injection.t datawhitened.injection.t],[-2 2],'Color',[1 0 0]) + set(gca,'XLim',[datawhitened.t(1) datawhitened.t(end)]), + set(gca,'YLim',[-1 1]), + xlabel('Time (s)'), ylabel('Strain') + title('Whitened and band-pass filtered strain data') + grid on, box on + + subplot(3,1,2) + plot(t,datasegment,'b','LineWidth',1), hold on + set(gca,'XLim',[t(1) t(end)]), + set(gca,'YLim',[-1 1]), + xlabel('Time (s)'), ylabel('Strain') + title('Current time window') + grid on, box on + + subplot(3,1,3) + mesh(t,ftfr,tfr) + xlabel('Time (s)'), ylabel('Frequency (Hz)') + title('h(t,f)') + colormap jet, view(0,90) + box on, grid on + set(gca,'XLim',[t(1) t(end)]), + set(gca,'Ylim',[min(ftfr) max(ftfr)]) + + drawnow + + if Segments.InIn(i)==1 + title('GW SIGNAL IS PRESENT') + pause + end % if Segments.InIn(i)==1 + + end % if (1) + + end % for i=1:Segments.Nseg + clear ans i inicio + + return + +end % if (doplot) + + + +%% COMPUTE AND CONSTRUCT THE STRAIN-BASED FEATURES + +% Get ID de los segmentos a usar +if (0) + % Option 1: uno con injection y uno con noise + SEGinjec = find(Segments.InIn); % ID de todos los segmentos con injection + SEGinjec = SEGinjec(2); % ID del segmento con injection con el que nos quedamos + SEGnoise = SEGinjec-3; % ID del segmento con ruido +else + % Option 2: Todos los que tengan injection y el mismo numero con noise + if sum(Segments.InIn)==2 + SEGinjec = find(Segments.InIn); % ID de los dos segmentos con injection + SEGnoise = [SEGinjec(1)-3; SEGinjec(2)+3]; % ID de los dos segmentos con ruido + elseif sum(Segments.InIn)==4 + SEGinjec = find(Segments.InIn); % ID de los cuatro segmentos con injection + SEGnoise = [SEGinjec(1)-4; SEGinjec(1)-3; SEGinjec(end)+3; SEGinjec(end)+4]; % ID de los cuatro segmentos con ruido + else + error('PILAS PERRO: trabaje') + end % +end % if (0) + + +% Be sure that the number of segments with injection and with noise is the same +if length(SEGnoise)~=length(SEGinjec) + error('PILAS: the number of segments with injection/noise is not the same') +end % if length(SEGnoise)~=length(SEGinjec) + + +% Get data, i.e., segments with injection and segments with noise +DataInjec = zeros(Twin*4096,length(SEGinjec)); +DataNoise = zeros(Twin*4096,length(SEGnoise)); +for i=1:length(SEGnoise) + % Segments with injection + IDseg = SEGinjec(i); + DataInjec(:,i) = datawhitened.st(Segments.Sint(IDseg,1):Segments.Sint(IDseg,2)); + + % Segments with noise + IDseg = SEGnoise(i); + DataNoise(:,i) = datawhitened.st(Segments.Sint(IDseg,1):Segments.Sint(IDseg,2)); +end % for i=1:length(SEGnoise) +clear ans i IDseg + + +% Clear garbage +clear ans SEGinjec SEGnoise Segments + + +% Debugging: compute and plot TFR +if (0) + + % Number of segments with injection and noise + Nsegments = size(DataInjec,2); + + % Time vector + t = (0:1:size(DataNoise,1)-1)/4096; + + % For each segment with injection and noise + for i=1:Nsegments + + % Compute TFR + [DataInjec1_tfr, ~, ~] = Compute_WaveletMorlet(1e20*DataInjec(:,i),datawhitened.fs,20,400,10,7,0); + [DataNoise1_tfr,aja,ftfr] = Compute_WaveletMorlet(1e20*DataNoise(:,i),datawhitened.fs,20,400,10,7,0); + + % Plot for debugging + figure(1) + subplot(2,Nsegments,i) + plot(t,1e20*DataInjec(:,i)) + subplot(2,Nsegments,i+Nsegments) + mesh(t,ftfr,DataInjec1_tfr) + xlabel('Time (s)'), %ylabel('Frequency (Hz)'), % title('h(t,f)'), colormap jet, + view(0,90), box on, grid on + set(gca,'XLim',[t(1) t(end)]), set(gca,'Ylim',[min(ftfr) max(ftfr)]) + + figure(2) + subplot(2,Nsegments,i) + plot(t,1e20*DataNoise(:,i)) + subplot(2,Nsegments,i+Nsegments) + mesh(t,ftfr,DataNoise1_tfr) + xlabel('Time (s)'), %ylabel('Frequency (Hz)'), % title('h(t,f)'), colormap jet, + view(0,90), box on, grid on + set(gca,'XLim',[t(1) t(end)]), set(gca,'Ylim',[min(ftfr) max(ftfr)]) + + end % for i=1:Nsegments + clear ans i Nsegments t aja ftfr DataInjec1_tfr DataNoise1_tfr + +end % if (0) + + +% Number of segments with injection and noise +Nsegments = size(DataInjec,2); + + +% Contruct X and y +X = [DataNoise DataInjec]; + +y = [1*ones(Nsegments,1) ; 2*ones(Nsegments,1)]; +M1 = [NaN(Nsegments,1) ; datawhitened.injection.M1*ones(Nsegments,1)]; +M2 = [NaN(Nsegments,1) ; datawhitened.injection.M2*ones(Nsegments,1)]; +D = [NaN(Nsegments,1) ; datawhitened.injection.D*ones(Nsegments,1)]; +SNR = [NaN(Nsegments,1) ; datawhitened.injection.Exp_SNR*ones(Nsegments,1)]; + +Y = [y M1 M2 D SNR]; + + +% Contruct Data structure with only the important info +Data.Twin = Twin; +Data.fs = datawhitened.fs; +Data.Xstrain = X; +Data.Y = Y; + + +% Part of the name to save data +if Twin==0.25, TwinStr='025'; +elseif Twin==0.50, TwinStr='050'; +elseif Twin==0.75, TwinStr='075'; +elseif Twin==1.00, TwinStr='100'; +elseif Twin==1.25, TwinStr='125'; +elseif Twin==1.50, TwinStr='150'; +elseif Twin==1.75, TwinStr='175'; +elseif Twin==2.00, TwinStr='200'; +else, error('PILAS: unknown Tslice') +end +% Save only if the strain data do not contain NaN values +if isnan(sum([sum(DataInjec(:)) sum(DataNoise(:))])) % isnan(sum(datawhitened.st)) + % The strain data in the segment contain NaN values + save([datawhitened.path 'FileTFRWithNaN_' num2str(ifile)],'ifile') +else + % OK: save data + save([datawhitened.path datawhitened.filename(1:end-5) '_Twin' TwinStr],'Data') +end % if isnan(sum(data.st)) + + +% Clear garbage +%clear ans y M1 M2 D SNR X Y Log DataNoise1 DataNoise2 DataInjec1 DataInjec2 +%clear ans DataNoise DataInjec TwinStr Nsegments datawhitened Data + + + +%% PERFORM ANALYSIS FOR ALL EXISTING FILES + +% Keep this "if" always in 0 and comment lines 3 to 11 +if (0) + % Initialize + clearvars + close all + clc + + RUN = 'S6'; + IFOS = {'L1'}; % {'H1','L1'}; + + Twin = 1.5; % (0.25|0.50|1.00|1.50|2.00|...) + + for ii=1:length(IFOS) + + IFO = IFOS{ii}; + + % List of existing datafiles + if strcmp(IFO,'H1') + % THE FILE IS NOT AVAILABLE IN LOSC FOR DOWNLOAD: 59,60,61,206:212,657:676 (para evitar errores en el codigo, cree un fake archivo) + % TOO EARLY OR TOO LATE INJECTION: 73,106,145,224,244,313,688 + datafiles = 1:724; + datafiles([59,60,61,206:212,657:676,73,106,145,224,244,313,688]) = [ ]; + elseif strcmp(IFO,'L1') + % THE FILE IS NOT AVAILABLE IN LOSC FOR DOWNLOAD: 59,60,147:150,566 (para evitar errores en el codigo, cree un fake archivo) + % TOO EARLY OR TOO LATE INJECTION: 118,134,235,264,375,415 + datafiles = 1:656; + datafiles([59,60,147:150,566,118,134,235,264,375,415]) = [ ]; + end + + % Perform detection for each datafile + for ifile=datafiles + fprintf('\nProcessing datafile %d of %d\n',ifile,max(datafiles)) + prog021_DetectionCNN_1PrepareData + clear ans data datawhitened Data Segments + end + clear ans ifile datafiles + + end % for ii=1:length(IFOS) + clear ans ii Twin + +end % if(0) diff --git a/prog021_DetectionCNN_2PrepareData.m b/prog021_DetectionCNN_2PrepareData.m new file mode 100755 index 0000000..4954ee5 --- /dev/null +++ b/prog021_DetectionCNN_2PrepareData.m @@ -0,0 +1,148 @@ +% ============================================ +% Compute Time-Frequency maps for each data segment +% ============================================ + +% Written by Dr. Manuel David Morales and Dr. Javier M. Antelis +% Any questions? manueld.morales@academicos.udg.mx + +%% INITIALIZE + +% ---------------------------------------- +clearvars +close all +clc + +% ---------------------------------------- +% Select IFO and get existing filenames +IFO = 'L1'; % (H1|L1) +Twin = '100'; % (025|050|100|150|200) besides 075, 125, 175 +RutaData = ['/home/claudia/Codes Manuel/Datasets/Data2016_LIGOS6/' IFO '/']; +filenames = dir([RutaData '*_Twin' Twin '.mat']); + + + +%% COMPUTE AND CONSTRUCT THE TFR-BASED FEATURES FOR ALL EXISTING FILES + + +% ---------------------------------------- +% Initialize variables +DataAll.Twin = []; +DataAll.Xtfr = []; +DataAll.Y = []; + + +% ---------------------------------------- +% For each existing file: +for i=1:length(filenames) + %i=1; %just for debug, remember restablish the loop for when debug ends + fprintf('Processing file number %d of %d\n',i,length(filenames)) + + % ---------------------------------------- + % Load data + load([RutaData filenames(i).name]) + + % ---------------------------------------- + % Compute TFR + for idata = 1:size(Data.Xstrain,2) + + %fprintf('Processing %d of %d\n',idata,size(Data.Xstrain,2)) + + % Compute TFR + [TFR,timeVec,freqVec] = Compute_WaveletMorlet(1e20*Data.Xstrain(:,idata),Data.fs,40,500,10,7,0); + + % Compute reduced TFR + TFRreduced = imresize(TFR,[16,32]); + + % Save reduced TFR + Data.Xtfr(idata,:,:) = TFRreduced; + Data.time = linspace(min(timeVec),max(timeVec),size(TFRreduced,2)); + Data.freq = linspace(min(freqVec),max(freqVec),size(TFRreduced,1)); + + % Plot for debugging + % PLOT OF SAMPLES (time length = Twin): NOISE ALONE, and NOISE+GW + if (1) + + time1 = timeVec; + freq1 = freqVec; + time2 = linspace(min(time1),max(time1),size(TFRreduced,2)); + freq2 = linspace(min(freq1),max(freq1),size(TFRreduced,1)); + + figure(1), clf + + subplot(3,1,1) + plot(time1,Data.Xstrain(:,idata)) + + condit = double(isnan(Data.Y(idata,2))); + + if (condit) + title(['Noise Only'], 'FontSize',11) + else + title(['M1=' num2str(Data.Y(idata,2)) ' | M2=' num2str(Data.Y(idata,3)) ' | D=' num2str(Data.Y(idata,4)) ' | SNR=' num2str(Data.Y(idata,5))], 'FontSize',11) + end + + subplot(3,1,2) + imagesc(time1,freq1,TFR) + xlabel('Time (s)'), ylabel('Frequency (Hz)'), + colormap jet, view(0,90), box on, grid on, set(gca,'YDir','normal') + set(gca,'XLim',[time1(1) time1(end)]), set(gca,'Ylim',[min(freq1) max(freq1)]) + + subplot(3,1,3) + imagesc(time2,freq2,TFRreduced) + xlabel('Time (s)'), ylabel('Frequency (Hz)'), + colormap jet, view(0,90), box on, grid on, set(gca,'YDir','normal') + set(gca,'XLim',[time2(1) time2(end)]), set(gca,'Ylim',[min(freq2) max(freq2)]) + pause + clear ans time1 time2 freq1 freq2 + end % if (0) + clear ans TFR TFRreduced + + %pause % just for debug....comment for normal run + + end % for idata = 1:size(DataAll.X,1) + clear ans idata + + % ---------------------------------------- + % Append data + DataAll.Xtfr = [ DataAll.Xtfr ; Data.Xtfr]; + DataAll.Y = [ DataAll.Y ; Data.Y ]; + +end % for i=1:length(filenames) +clear ans i filenames + + +% ---------------------------------------- +% Save Twin and sampling frequency +%Data.Twin = Twin +DataAll.Twin = Data.Twin; +DataAll.t = Data.time; +DataAll.f = Data.freq; + + +% ---------------------------------------- +% Clear garbage +clear ans TFR TFRreduced Data + + +% ---------------------------------------- +% Data to save +Data = DataAll; + + +% ---------------------------------------- +% Save data +if Data.Twin==0.25, TwinStr='025'; +elseif Data.Twin==0.50, TwinStr='050'; +elseif Data.Twin==0.75, TwinStr='075'; +elseif Data.Twin==1.00, TwinStr='100'; +elseif Data.Twin==1.25, TwinStr='125'; +elseif Data.Twin==1.50, TwinStr='150'; +elseif Data.Twin==1.75, TwinStr='175'; +elseif Data.Twin==2.00, TwinStr='200'; +else, error('PILAS: unknown Tslice') +end +save([RutaData 'TFR-TW' TwinStr],'Data') + + +% ---------------------------------------- +% Clear garbage +clear ans IFO DataAll diff --git a/prog021_DetectionCNN_3ClasifyBasic.m b/prog021_DetectionCNN_3ClasifyBasic.m new file mode 100755 index 0000000..afbd95b --- /dev/null +++ b/prog021_DetectionCNN_3ClasifyBasic.m @@ -0,0 +1,227 @@ +% ============================================== +% Basic classification: Train/Test a single CNN as sanity check +% ============================================== + +% Written by Dr. Manuel David Morales and Dr. Javier M. Antelis +% Any questions? manueld.morales@academicos.udg.mx + +%% A) INITIALIZE +% ##### Parameters to manually set: IFO, TWIN + +% ---------------------------------------- +clearvars +close all +clc + +% ---------------------------------------- +% Select IFO (interferometer) anb Twin (old detector) +% H1: Hanford detector | L1: Livingstone detector +IFO = 'H1'; % (H1|L1|BOTH) +Twin = '050'; % (025|050|100|150|200) + + +%% B) LOAD DATA + +% ---------------------------------------- +% Load data +% Remark: Dots (.) define structure arrays +% Fields of Data.Y: class (0 OR 1); M1, M2 (mass in solar masses), +% distance (Mpc), and SNR (signal-to-noise ratio). + +if strcmp(IFO,'H1')||strcmp(IFO,'L1') + % Load data for the H1 or L1 + RutaData = ['/home/manuel/Projects Science/Data analysis/Datasets/Data2017_LIGOS6/' IFO '/']; + %RutaData = ['C:\_DataSets\Data2017_LIGOS6\' IFO '\']; + load([RutaData 'TFR-TW' Twin]) +elseif strcmp(IFO,'BOTH') + % Load data for H1 + RutaData = ['/home/manuel/Projects Science/Data analysis/Datasets/Data2017_LIGOS6/' 'H1' '/']; + H1 = load([RutaData 'TFR-TW' Twin]); + % Load data for L1 + RutaData = ['/home/manuel/Projects Science/Data analysis/Datasets/Data2017_LIGOS6/' 'L1' '/']; + L1 = load([RutaData 'TFR-TW' Twin]); + % Save path + RutaData = ['/home/manuel/Projects Science/Data analysis/Datasets/Data2017_LIGOS6/' 'BOTH' '/']; + % Append data + Data = H1.Data; + Data.Y = [H1.Data.Y ; L1.Data.Y ]; + Data.Xtfr = [H1.Data.Xtfr ; L1.Data.Xtfr]; + +else + error('PILAS: unknown IFO') +end + +% ---------------------------------------- +% Remove unused fields +Data.IFO = IFO; +Data.RutaData = RutaData; + +% ---------------------------------------- +% Remove unused fields +Data = rmfield(Data,'t'); +Data = rmfield(Data,'f'); + +% ---------------------------------------- +% Clear garbage +clear ans TwinStr Twin IFO RutaData + + + +%% C) CONSTRUCT X AND Y +% ##### Parameters to manually set: pointer "S" in the module +% Compute_CNNeliminateSNR(Data,S); + +% Here external functions are called +% Class 1: NOISE, Class 2: Gravitational Waves +% Make sure that the class labels are 1 and 2 +Data = Compute_CheckLabels(Data); + +% Eliminate data for which SNR<10 (eliminate the same samples for both class) +Data = Compute_CNNeliminateSNR(Data,10); + +% Construct X and Y +% 1) Convert from Xtfr to X in the data format for the CNN [images] +% 2) Save the current Y matrix as YInfo and create the class label vector Y +Data = Compute_CNNconstructXY(Data); + + + +%% D) CLASSIFICATION 0: TRAIN AND TEST A SINGLE CNN TO CHECK IF ALL IS WORKING + +% ---------------------------------------- +% 0) Initialize variable +CNN = []; +CNN.layers = []; +CNN.options = []; +CNN.net = []; +CNN.traininfo = []; + +CNN.XTrain = []; +CNN.YTrain = []; + +CNN.XValid = []; +CNN.YValid = []; + +CNN.XTest = []; +CNN.YTest = []; + +CNN.YEsti = []; +CNN.YProb = []; +CNN.ACC = []; +CNN.CM = []; + + +% ---------------------------------------- +% 1) Separate dataset into two mutually exclusive sets: (1) train and (2) test +% ##### Parameters to manually set: float f (0