Merge pull request #69 from yirongjie/main

Feat: Optimize the operation process
UbiquitousLearning · Mar 19, 2024 · df557bd · df557bd
2 parents 527ba72 + c5324bb
commit df557bd
Show file tree

Hide file tree

Showing 12 changed files with 672 additions and 1,145 deletions.
diff --git a/.gitignore b/.gitignore
@@ -25,3 +25,4 @@ Makefile
 models/*
 /.devcontainer/
 /.vscode/
+models
diff --git a/README.md b/README.md
@@ -96,7 +96,7 @@ Building mllm requires following tools:
 
 #### Run Fuyu-8B
 
-Download the model from [here](https://huggingface.co/mllmTeam), or using the following instructions
+Download the model from [here](https://huggingface.co/mllmTeam/fuyu-8b-mllm/tree/main/), or using the following instructions
 
 ```bash
 mkdir ../models && cd ../models
@@ -125,7 +125,7 @@ Result are as followed:
 
 #### Run LLaMA-2-7B
 
-Download model
+Download model from [here](https://huggingface.co/mllmTeam/llama-2-7b-mllm/tree/main/), or using the following instructions
 
 ```bash
 mkdir ../models && cd ../models
@@ -165,7 +165,7 @@ BUPT offers a wide range of undergraduate and graduate programs in fields such a
 
 #### Run ImageBind
 
-Download model
+Download model from [here](https://huggingface.co/mllmTeam/imagebind_huge-mllm/tree/main), or using the following instructions
 
 ```bash
 mkdir ../models && cd ../models

diff --git a/examples/demo_vit.cpp b/examples/demo_vit.cpp
@@ -1,5 +1,4 @@
 #include <iostream>
-#include <utility>
 #include "cmdline.h"
 #include "models/vit/modeling_vit.hpp"
 #include "models/vit/labels_vit.hpp"

diff --git a/include/Types.hpp b/include/Types.hpp
@@ -27,8 +27,7 @@ typedef enum {
 enum TensorStatus {
     TENSOR_DYNAMIC,
     TENSOR_STATIC_INIT ,
-    TENSOR_STATIC_SHAPED ,
-    TENSOR_STATIC_ALLOCED ,
+    TENSOR_STATIC_READY ,
 };
 
 enum ErrorCode {

diff --git a/src/Layer.hpp b/src/Layer.hpp
diff --git a/src/Module.cpp b/src/Module.cpp
@@ -11,5 +11,5 @@ ParamLoader *Module::loader;
 int Module::listIdx;
 int Module::runlistIdx;
 TensorStatus Module::tensor_status;
-
+bool Module::doLoad = false;
 } // namespace mllm
diff --git a/src/Module.hpp b/src/Module.hpp
@@ -21,6 +21,7 @@ class Module {
     static map<BackendType, Backend *> backends;
     static ParamLoader *loader;
     static TensorStatus tensor_status;
+    static bool doLoad;
 
     Module() = default;
     virtual ~Module() = default;
@@ -48,6 +49,17 @@ class Module {
 
     void load(string path) {
         initLoader(path);
+        Module::doLoad = true;
+        vector<Tensor> tmps;
+        int max_in_size = 5;
+        for (int i = 0; i < max_in_size; ++i) {
+            Tensor::gph_[std::to_string(i)] = Tensor();
+            tmps.push_back(Tensor::gph_[std::to_string(i)]);
+        }
+        vector<int> tmpt = {0, 0};
+        operator()(tmps, tmpt);
+        Module::doLoad = false;
+        Tensor::gph_.clear();
     }
 
     virtual vector<Tensor> Forward(vector<Tensor> inputs, vector<std::any> args) = 0;
@@ -59,6 +71,9 @@ class Module {
     template <typename... Args>
     vector<Tensor> operator()(vector<Tensor> inputs, Args... args) {
         vector<std::any> anyArgs = convertArgsToAnyVector(args...);
+        if(doLoad) {
+            return Forward(inputs, anyArgs);
+        }
         if (inputs[0].ttype() == TensorType::INPUT_TENSOR) {
             for (auto &input : inputs) {
                 input.setTtype(TensorType::NORMAL_TENSOR);
@@ -68,36 +83,16 @@ class Module {
 
             Forward(inputs, anyArgs);
             for (auto &input : inputs) {
-                input.status() = TENSOR_STATIC_SHAPED;
-            }
-            tensor_status = TENSOR_STATIC_SHAPED;
-
-            Forward(inputs, anyArgs);
-            for (auto &input : inputs) {
-                input.status() = TENSOR_STATIC_ALLOCED;
+                input.status() = TENSOR_STATIC_READY;
             }
-            tensor_status = TENSOR_STATIC_ALLOCED;
+            tensor_status = TENSOR_STATIC_READY;
 
             return Forward(inputs, anyArgs);
         } else {
             return Forward(inputs, anyArgs);
         }
     }
 
-    // vector<Tensor> call(vector<Tensor> inputs, vector<std::any> args) {
-    //     return operator()(inputs, args);
-    // }
-
-    // template <typename T>
-    // static vector<T *> List(int n) {
-    //     static_assert(std::is_base_of<Module, T>::value, "T must be a subclass of Module");
-    //
-    //     vector<T *> modules;
-    //     for (int i = 0; i < n; i++) {
-    //         modules.push_back(new T());
-    //     }
-    //     return modules;
-    // }
     static int listIdx;
     static int runlistIdx;