test.cpp.iris

#include <list>
#include <vector>
#include <iostream>
#include <iomanip>
#include <algorithm>
#include <chrono>

#include "png.h"


#include "graph.h"
#include "variable.h"
#include "model.h"
#include "dataset.h"
#include "batchdata.h"
#include "iris.h"
#include "mnist.h"
#include "optimizer_adam.h"
#include "optimizer_sgd_moment.h"
#include "word_embed.h"

using namespace std;

MallocCounter mallocCounter;


void asMatrix(PVariable x1, float *X){
    x1->data.memSetHost(X);
}


float getAccurecy(Graph *g_softmax, PVariable h, PVariable d, int batchSize){
    PVariable y = ((Softmax *)g_softmax)->forward(h);

    int maxIdx[batchSize];
    //y->data.maxRowIndex(maxIdx);
    h->data.maxRowIndex(maxIdx);

    int maxIdx_d[batchSize];
    d->data.maxRowIndex(maxIdx_d);

    int hit = 0;
    for(int i=0; i<batchSize; i++){
        if (maxIdx_d[i] == maxIdx[i]) hit++;
    }
    float accurecy = ((float)hit) / ((float) batchSize);
    return accurecy;
}

PVariable forward_one_step(Model &model, PVariable x1, bool is_train) {

    ((Dropout *)model.G("g_drop1"))->isTrain(is_train);
    ((Dropout *)model.G("g_drop2"))->isTrain(is_train);

    PVariable h1 = model.G("g_drop1")->forward(model.G("g_relu1")->forward(model.G("g1")->forward(x1)));
    PVariable h2 = model.G("g_drop2")->forward(model.G("g_relu2")->forward(model.G("g2")->forward(h1)));
    PVariable h3 = model.G("g3")->forward(h2);

    return h3;
}

float test_accurecy(Model &model, vector<BatchData *> &bds_test, int i_size, int o_size, int totalTestSize, int batchSize){
    float accurecy = 0.0;
    int predict_epoch = totalTestSize/batchSize;
    for(int i=0; i<predict_epoch; i++){

        std::random_shuffle(bds_test.begin(), bds_test.end());

        PVariable x1(new Variable(i_size, batchSize));
        PVariable d(new Variable(o_size, batchSize));

        // create mini-batch =========================
        float *X = bds_test.at(i)->getX();
        float *D = bds_test.at(i)->getD();
        asMatrix(x1, X);
        asMatrix(d, D);

        // forward ------------------------------------------
        PVariable h3 = forward_one_step(model, x1, false);

        accurecy += getAccurecy(model.G("g_softmax"), h3, d, batchSize);

        model.zero_grads();
        model.unchain();

    }

    return accurecy/((float)predict_epoch);
}


int main(){

    int epochNums = 2000;
    int epochAENums = 2000;

    int batchSize = 10;
    int i_size = 4;
    int n_size = 10;
    int o_size = 3;
    float learning_rate = 0.001;
    float ae_learning_rate = 0.001;
    float dropout_p = 0.5;
    float ae_dropout_p = 0.5;

    cout << "init dataset..." << endl;
    vector<vector<float>> train_data, test_data;
    vector<float> label_data, label_test_data;

    Iris iris(0.3);
    train_data = iris.getTrainData();
    label_data = iris.getLabelData();
    test_data = iris.getTestTrainData();
    label_test_data = iris.getTestLabelData();

    int totalSampleSize = train_data.size();
    int totalTestSize = test_data.size();

    Dataset *dataset = new Dataset();
    dataset->standrize(&train_data);
    vector<BatchData *> bds;
    for(int i=0; i<totalSampleSize/batchSize; i++){
        BatchData *bdata = new BatchData(i_size, o_size, batchSize);
        dataset->createMiniBatch(train_data, label_data, bdata->getX(), bdata->getD(), batchSize, o_size, i);
        bds.push_back(bdata);
    }
    dataset->standrize(&test_data);
    vector<BatchData *> bds_test;
    for(int i=0; i<totalTestSize/batchSize; i++){
        BatchData *bdata = new BatchData(i_size, o_size, batchSize);
        dataset->createMiniBatch(test_data, label_test_data, bdata->getX(), bdata->getD(), batchSize, o_size, i);
        bds_test.push_back(bdata);
    }


    std::chrono::system_clock::time_point  start, end;

    cout << "create model..." << endl;
    Model model;
    model.putG("g1", new Linear(n_size, i_size));
    model.putG("g_relu1", new ReLU());
    model.putG("g_drop1", new Dropout(dropout_p));
    model.putG("g2", new Linear(n_size, n_size));
    model.putG("g_relu2", new ReLU());
    model.putG("g_drop2", new Dropout(dropout_p));
    model.putG("g3", new Linear(o_size, n_size));
    model.putG("g_softmax_cross_entoropy", new SoftmaxCrossEntropy());
    model.putG("g_softmax", new Softmax());

    OptimizerAdam optimizer(&model, learning_rate);
    optimizer.init();

    float loss_mean = 0.0;
    float accurecy_mean = 0.0;
    float test_acc = 0.0;

    cout << "start training ..." << endl;
    for(int k=0; k<epochNums; k++){

        std::random_shuffle(bds.begin(), bds.end());

        float sum_loss = 0.0;
        float accurecy = 0.0;

        PVariable loss_graph(new Variable(1, 1));

        for(int i=0; i<totalSampleSize/batchSize; i++){

            PVariable x1(new Variable(i_size, batchSize));
            PVariable d(new Variable(o_size, batchSize));

            // create mini-batch =========================
            float *X = bds.at(i)->getX();
            float *D = bds.at(i)->getD();
            asMatrix(x1, X);
            asMatrix(d, D);

            // forward ------------------------------------------
            PVariable h3 = forward_one_step(model, x1, true);

            PVariable loss = model.G("g_softmax_cross_entoropy")->forward(h3, d);

            // loss ---------------------------------------------
            sum_loss += loss->val();

            // backward -----------------------------------------
            loss->backward();

            // update -------------------------------------------
            optimizer.update();

            accurecy += getAccurecy(model.G("g_softmax"), h3, d, batchSize);

            model.unchain();
        }


        loss_mean += sum_loss/((float)totalSampleSize/batchSize);
        accurecy_mean += accurecy/((float)totalSampleSize/batchSize);
        test_acc += test_accurecy(model, bds_test, i_size, o_size, totalTestSize, batchSize);

        if (k!=0 && k % 50 == 0) {
            cout << accurecy_mean/50.0 << "," << test_acc/50.0 << endl;
            loss_mean = 0.0;
            accurecy_mean = 0.0;
            test_acc = 0.0;
        }
    }


    //cout << "saving model..." << endl;
    //model.save("mlp_test.model");



    //cout << "loading model..." << endl;
    //Model model_train;
    //model_train.load("mlp_test.model");
    //cout << "loaded" << endl;

}