mirror of https://github.com/alibaba/MNN.git
380 lines
12 KiB
C++
380 lines
12 KiB
C++
//
|
|
// Interpreter.hpp
|
|
// MNN
|
|
//
|
|
// Created by MNN on 2018/07/23.
|
|
// Copyright © 2018, Alibaba Group Holding Limited
|
|
//
|
|
|
|
#ifndef Interpreter_hpp
|
|
#define Interpreter_hpp
|
|
|
|
#include <functional>
|
|
#include <map>
|
|
#include <memory>
|
|
#include <string>
|
|
#include <MNN/ErrorCode.hpp>
|
|
#include <MNN/MNNForwardType.h>
|
|
#include <MNN/Tensor.hpp>
|
|
|
|
namespace MNN {
|
|
|
|
/** session schedule config */
|
|
struct ScheduleConfig {
|
|
/** which tensor should be kept */
|
|
std::vector<std::string> saveTensors;
|
|
/** forward type */
|
|
MNNForwardType type = MNN_FORWARD_CPU;
|
|
/** CPU:number of threads in parallel , Or GPU: mode setting*/
|
|
union {
|
|
int numThread = 4;
|
|
int mode;
|
|
};
|
|
|
|
/** subpath to run */
|
|
struct Path {
|
|
std::vector<std::string> inputs;
|
|
std::vector<std::string> outputs;
|
|
|
|
enum Mode {
|
|
/**
|
|
* Op Mode
|
|
* - inputs means the source op, can NOT be empty.
|
|
* - outputs means the sink op, can be empty.
|
|
* The path will start from source op, then flow when encounter the sink op.
|
|
* The sink op will not be compute in this path.
|
|
*/
|
|
Op = 0,
|
|
|
|
/**
|
|
* Tensor Mode
|
|
* - inputs means the inputs tensors, can NOT be empty.
|
|
* - outputs means the outputs tensors, can NOT be empty.
|
|
* It will find the pipeline that compute outputs from inputs.
|
|
*/
|
|
Tensor = 1
|
|
};
|
|
|
|
/** running mode */
|
|
Mode mode = Op;
|
|
};
|
|
Path path;
|
|
|
|
/** backup backend used to create execution when desinated backend do NOT support any op */
|
|
MNNForwardType backupType = MNN_FORWARD_CPU;
|
|
|
|
/** extra backend config */
|
|
BackendConfig* backendConfig = nullptr;
|
|
};
|
|
|
|
class Session;
|
|
struct Content;
|
|
class Tensor;
|
|
class Backend;
|
|
class Runtime;
|
|
|
|
class MNN_PUBLIC OperatorInfo {
|
|
struct Info;
|
|
|
|
public:
|
|
/** Operator's name*/
|
|
const std::string& name() const;
|
|
|
|
/** Operator's type*/
|
|
const std::string& type() const;
|
|
|
|
/** Operator's flops, in M*/
|
|
float flops() const;
|
|
|
|
protected:
|
|
OperatorInfo();
|
|
~OperatorInfo();
|
|
Info* mContent;
|
|
};
|
|
|
|
typedef std::function<bool(const std::vector<Tensor*>&, const std::string& /*opName*/)> TensorCallBack;
|
|
typedef std::function<bool(const std::vector<Tensor*>&, const OperatorInfo*)> TensorCallBackWithInfo;
|
|
typedef std::pair<std::map<MNNForwardType, std::shared_ptr<Runtime>>, std::shared_ptr<Runtime>> RuntimeInfo;
|
|
|
|
/** net data holder. multiple sessions could share same net. */
|
|
class MNN_PUBLIC Interpreter {
|
|
public:
|
|
/**
|
|
* @brief create net from file.
|
|
* @param file given file.
|
|
* @return created net if success, NULL otherwise.
|
|
*/
|
|
static Interpreter* createFromFile(const char* file);
|
|
/**
|
|
* @brief create net from buffer.
|
|
* @param buffer given data buffer.
|
|
* @param size size of data buffer.
|
|
* @return created net if success, NULL otherwise.
|
|
*/
|
|
static Interpreter* createFromBuffer(const void* buffer, size_t size);
|
|
~Interpreter();
|
|
|
|
enum SessionMode {
|
|
/** About CallBack, Default Session_Debug*/
|
|
/** runSessionWithCallBack is allowed and can get internal op info*/
|
|
Session_Debug = 0,
|
|
/** runSessionWithCallBack is not valid and can't get any info of op in session*/
|
|
Session_Release = 1,
|
|
|
|
/** About input tenosr, Default Session_Input_Inside*/
|
|
/** The input tensor is alloced by session, input data after session resized*/
|
|
Session_Input_Inside = 2,
|
|
/** The input tensor is alloced by user, set input data before session resize*/
|
|
Session_Input_User = 3,
|
|
|
|
/** The output tensor depends on session, and can't be seperate used*/
|
|
Session_Output_Inside = 4,
|
|
/** The output tensor can be seperated from session*/
|
|
Session_Output_User = 5,
|
|
|
|
/** Try Resize Session when create Session or not, default direct: */
|
|
Session_Resize_Direct = 6,
|
|
Session_Resize_Defer = 7,
|
|
|
|
/** Determine the Execution's forward type is determine by user or auto determine */
|
|
Session_Backend_Fix = 8, // Use the backend user set, when not support use default backend
|
|
Session_Backend_Auto = 9, // Auto Determine the Op type by MNN
|
|
};
|
|
/**
|
|
* @brief The API shoud be called before create session.
|
|
* @param mode session mode
|
|
*/
|
|
void setSessionMode(SessionMode mode);
|
|
|
|
/**
|
|
* @brief The API shoud be called before create session.
|
|
* If the cache exist, try to load cache from file.
|
|
* After createSession, try to save cache to file.
|
|
* @param cacheFile cache file name
|
|
* @param keySize depercerate, for future use.
|
|
*/
|
|
void setCacheFile(const char* cacheFile, size_t keySize = 128);
|
|
|
|
/**
|
|
* @brief The API shoud be called after last resize session.
|
|
* If resize session generate new cache info, try to rewrite cache file.
|
|
* If resize session do not generate any new cache info, just do nothing.
|
|
* @param session giveb session
|
|
* @param flag Protected param, not used now
|
|
*/
|
|
ErrorCode updateCacheFile(Session *session, int flag = 0);
|
|
|
|
enum HintMode {
|
|
// Max Op number for async tuning
|
|
MAX_TUNING_NUMBER = 0,
|
|
};
|
|
/**
|
|
* @brief The API shoud be called before create session.
|
|
* @param mode Hint type
|
|
* @param value Hint value
|
|
*/
|
|
void setSessionHint(HintMode mode, int value);
|
|
public:
|
|
/**
|
|
* @brief create runtimeInfo seperately with schedule config.
|
|
* @param configs session schedule configs.
|
|
*/
|
|
static RuntimeInfo createRuntime(const std::vector<ScheduleConfig>& configs);
|
|
|
|
/**
|
|
* @brief create session with schedule config. created session will be managed in net.
|
|
* @param config session schedule config.
|
|
* @return created session if success, NULL otherwise.
|
|
*/
|
|
Session* createSession(const ScheduleConfig& config);
|
|
|
|
/**
|
|
* @brief create session with schedule config and user-specified runtime.
|
|
* @param config session schedule config, runtime runtimeInfo used by the created session.
|
|
* @return created session if success, NULL otherwise.
|
|
*/
|
|
Session* createSession(const ScheduleConfig& config, const RuntimeInfo& runtime);
|
|
|
|
/**
|
|
* @brief create multi-path session with schedule configs. created session will be managed in net.
|
|
* @param configs session schedule configs.
|
|
* @return created session if success, NULL otherwise.
|
|
*/
|
|
Session* createMultiPathSession(const std::vector<ScheduleConfig>& configs);
|
|
|
|
/**
|
|
* @brief create multi-path session with schedule configs and user-specified runtime.
|
|
created session will be managed in net.
|
|
* @param configs session schedule configs.
|
|
* @return created session if success, NULL otherwise.
|
|
*/
|
|
Session* createMultiPathSession(const std::vector<ScheduleConfig>& configs, const RuntimeInfo& runtime);
|
|
|
|
/**
|
|
* @brief release session.
|
|
* @param session given session.
|
|
* @return true if given session is held by net and is freed.
|
|
*/
|
|
bool releaseSession(Session* session);
|
|
|
|
/**
|
|
* @brief call this function to get tensors ready. output tensor buffer (host or deviceId) should be retrieved
|
|
* after resize of any input tensor.
|
|
* @param session given session.
|
|
*/
|
|
void resizeSession(Session* session);
|
|
|
|
/**
|
|
* @brief call this function if don't need resize or create session any more, it will save a few memory that equal
|
|
* to the size of model buffer
|
|
*/
|
|
void releaseModel();
|
|
|
|
/**
|
|
* @brief Get the model buffer for user to save
|
|
* @return std::make_pair(modleBuffer, modelSize).
|
|
* @example:
|
|
* std::ofstream output("trainResult.alinn")
|
|
* auto buffer = net->getModelBuffer();
|
|
* output.write((const char*)buffer.first, buffer.second);
|
|
*/
|
|
std::pair<const void*, size_t> getModelBuffer() const;
|
|
|
|
/**
|
|
* @brief update Session's Tensor to model's Const Op
|
|
* @param session given session.
|
|
* @return result of running.
|
|
*/
|
|
ErrorCode updateSessionToModel(Session* session);
|
|
|
|
/**
|
|
* @brief run session.
|
|
* @param session given session.
|
|
* @return result of running.
|
|
*/
|
|
ErrorCode runSession(Session* session) const;
|
|
|
|
/*
|
|
* @brief run session.
|
|
* @param session given session.
|
|
* @param before callback before each op. return true to run the op; return false to skip the op.
|
|
* @param after callback after each op. return true to continue running; return false to interrupt the session.
|
|
* @param sync synchronously wait for finish of execution or not.
|
|
* @return result of running.
|
|
*/
|
|
ErrorCode runSessionWithCallBack(const Session* session, const TensorCallBack& before, const TensorCallBack& end,
|
|
bool sync = false) const;
|
|
|
|
/*
|
|
* @brief run session.
|
|
* @param session given session.
|
|
* @param before callback before each op. return true to run the op; return false to skip the op.
|
|
* @param after callback after each op. return true to continue running; return false to interrupt the session.
|
|
* @param sync synchronously wait for finish of execution or not.
|
|
* @return result of running.
|
|
*/
|
|
ErrorCode runSessionWithCallBackInfo(const Session* session, const TensorCallBackWithInfo& before,
|
|
const TensorCallBackWithInfo& end, bool sync = false) const;
|
|
|
|
/**
|
|
* @brief get input tensor for given name.
|
|
* @param session given session.
|
|
* @param name given name. if NULL, return first input.
|
|
* @return tensor if found, NULL otherwise.
|
|
*/
|
|
Tensor* getSessionInput(const Session* session, const char* name);
|
|
/**
|
|
* @brief get output tensor for given name.
|
|
* @param session given session.
|
|
* @param name given name. if NULL, return first output.
|
|
* @return tensor if found, NULL otherwise.
|
|
*/
|
|
Tensor* getSessionOutput(const Session* session, const char* name);
|
|
|
|
enum SessionInfoCode {
|
|
/** memory session used in MB, float* */
|
|
MEMORY = 0,
|
|
|
|
/** float operation needed in session in M, float* */
|
|
FLOPS = 1,
|
|
|
|
/** Backends in session in M, int*, length >= 1 + number of configs when create session */
|
|
BACKENDS = 2,
|
|
|
|
ALL
|
|
};
|
|
|
|
/**
|
|
* @brief get session info
|
|
* @param session given session.
|
|
* @param code given info code.
|
|
* @param ptr given info ptr, see SessionInfoCode for detail
|
|
* @return true if support the code, false otherwise.
|
|
*/
|
|
bool getSessionInfo(const Session* session, SessionInfoCode code, void* ptr);
|
|
|
|
/**
|
|
* @brief get all output tensors.
|
|
* @param session given session.
|
|
* @return all output tensors mapped with name.
|
|
*/
|
|
const std::map<std::string, Tensor*>& getSessionOutputAll(const Session* session) const;
|
|
/**
|
|
* @brief get all input tensors.
|
|
* @param session given session.
|
|
* @return all input tensors mapped with name.
|
|
*/
|
|
const std::map<std::string, Tensor*>& getSessionInputAll(const Session* session) const;
|
|
|
|
public:
|
|
/**
|
|
* @brief resize given tensor.
|
|
* @param tensor given tensor.
|
|
* @param dims new dims. at most 6 dims.
|
|
*/
|
|
void resizeTensor(Tensor* tensor, const std::vector<int>& dims);
|
|
|
|
/**
|
|
* @brief resize given tensor by nchw.
|
|
* @param batch / N.
|
|
* @param channel / C.
|
|
* @param height / H.
|
|
* @param width / W
|
|
*/
|
|
void resizeTensor(Tensor* tensor, int batch, int channel, int height, int width);
|
|
|
|
/**
|
|
* @brief get backend used to create given tensor.
|
|
* @param session given session.
|
|
* @param tensor given tensor.
|
|
* @return backend used to create given tensor, may be NULL.
|
|
*/
|
|
const Backend* getBackend(const Session* session, const Tensor* tensor) const;
|
|
|
|
/**
|
|
* @brief get business code (model identifier).
|
|
* @return business code.
|
|
*/
|
|
const char* bizCode() const;
|
|
|
|
/**
|
|
* @brief get model UUID
|
|
* @return Model UUID.
|
|
*/
|
|
const char* uuid() const;
|
|
|
|
private:
|
|
static Interpreter* createFromBufferInternal(Content* net);
|
|
|
|
Content* mNet = nullptr;
|
|
Interpreter(Content* net);
|
|
|
|
Interpreter(const Interpreter&) = delete;
|
|
Interpreter(const Interpreter&&) = delete;
|
|
Interpreter& operator=(const Interpreter&) = delete;
|
|
Interpreter& operator=(const Interpreter&&) = delete;
|
|
};
|
|
} // namespace MNN
|
|
|
|
#endif /* Interpreter_hpp */
|