issue/461 InfiniCore 推理运行时

Co-authored-by: Jiacheng Huang <huangjiacheng0709@outlook.com>
Co-authored-by: wooway777 <wooway777@gmail.com>

Author: 3 people

.github/workflows/build.yml

@@ -19,6 +19,9 @@ jobs:
 
     - name: checkout code
       uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        fetch-depth: 0
 
     - name: Check Format
       run: |

.gitignore

@@ -10,6 +10,7 @@ build/
 
 # Python
 __pycache__/
+*.egg-info/
 
 # Log
 *.log
@@ -22,3 +23,8 @@ cache/
 
 #GGUF
 *.gguf
+
+# Compressed
+*.gz
+*.zip
+*.tar

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/spdlog"]
+	path = third_party/spdlog
+	url = https://github.com/gabime/spdlog.git

README.md

@@ -28,6 +28,20 @@ API 定义以及使用方式详见 [`InfiniCore文档`](https://github.com/Infin
 
 ## 配置和使用
 
+### 子模块
+
+由于仓库中含有子模块，所以在克隆时请添加 `--recursive` 或 `--recurse-submodules`，如：
+
+```shell
+git clone --recursive https://github.com/InfiniTensor/InfiniCore.git
+```
+
+或者在普通克隆后进行更新：
+
+```shell
+git submodule update --init --recursive
+```
+
 ### 一键安装
 
 在 `script/` 目录中提供了 `install.py` 安装脚本。使用方式如下：

include/infinicore.hpp

@@ -1,6 +1,4 @@
-#ifndef __INFINICORE_API_HPP__
-#define __INFINICORE_API_HPP__
+#pragma once
 
+#include "infinicore/ops.hpp"
 #include "infinicore/tensor.hpp"
-
-#endif

include/infinicore/common/LRUCache.hpp

@@ -0,0 +1,137 @@
+#pragma once
+
+#include <cstddef>
+#include <functional>
+#include <iostream>
+#include <list>
+#include <optional>
+#include <stdexcept>
+#include <unordered_map>
+
+namespace infinicore::common {
+template <typename Key, typename Value>
+class LRUCache {
+public:
+    using KeyValuePair = std::pair<Key, Value>;
+    using ListIt = typename std::list<KeyValuePair>::iterator;
+    using Destructor = std::function<void(Value &)>;
+
+    explicit LRUCache(size_t capacity = 100, Destructor destructor = nullptr)
+        : capacity_(capacity), destructor_(destructor) {
+        if (capacity == 0) {
+            capacity_ = UINT64_MAX; // effectively unbounded
+        }
+    }
+
+    ~LRUCache() {
+        cleanup();
+    }
+
+    bool contains(const Key &key) const {
+        return map_.find(key) != map_.end();
+    }
+
+    void put(const Key &key, const Value &value) {
+        auto it = map_.find(key);
+        if (it != map_.end()) {
+            if (destructor_) {
+                destructor_(it->second->second);
+            }
+            it->second->second = value;
+            touch(it);
+        } else {
+            // insert new
+            if (list_.size() >= capacity_) {
+                evictLRU();
+            }
+            list_.emplace_front(key, value);
+            map_[key] = list_.begin();
+        }
+    }
+
+    std::optional<Value> get(const Key &key) {
+        auto it = map_.find(key);
+        if (it == map_.end()) {
+            return std::nullopt;
+        }
+        touch(it);
+        return it->second->second;
+    }
+
+    std::optional<Value> get(const Key &key) const {
+        auto it = map_.find(key);
+        if (it == map_.end()) {
+            return std::nullopt;
+        }
+        // Note: can't touch in const context
+        return it->second->second;
+    }
+
+    void setDestructor(Destructor destructor) {
+        destructor_ = destructor;
+    }
+
+    void setCapacity(size_t capacity) {
+        capacity_ = capacity;
+        while (list_.size() > capacity_) {
+            evictLRU();
+        }
+    }
+
+    void clear() {
+        if (destructor_) {
+            for (auto &item : list_) {
+                safeDestruct(item.second);
+            }
+        }
+        list_.clear();
+        map_.clear();
+    }
+
+    const std::list<KeyValuePair> &getAllItems() const {
+        return list_;
+    }
+
+protected:
+    std::list<KeyValuePair> list_; // front = most recent, back = least
+
+private:
+    void touch(typename std::unordered_map<Key, ListIt>::iterator it) {
+        // move this key to front (most recent)
+        list_.splice(list_.begin(), list_, it->second);
+        it->second = list_.begin();
+    }
+
+    void safeDestruct(Value &value) {
+        if (!destructor_) {
+            return;
+        }
+
+        try {
+            destructor_(value);
+        } catch (const std::exception &e) {
+            // Built-in default error handling
+            std::cerr << "Cache destructor error (type: " << typeid(Value).name()
+                      << "): " << e.what() << std::endl;
+        }
+    }
+
+    void evictLRU() {
+        if (!list_.empty()) {
+            auto &kv = list_.back();
+            safeDestruct(kv.second);
+            map_.erase(kv.first);
+            list_.pop_back();
+        }
+    }
+
+    void cleanup() {
+        clear();
+    }
+
+    size_t capacity_;
+    std::unordered_map<Key, ListIt> map_;
+    Destructor destructor_;
+};
+
+} // namespace infinicore::common

include/infinicore/common/hash.hpp

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "../tensor.hpp"
+
+#include <type_traits>
+
+namespace infinicore {
+
+// Base hash_combine for arithmetic types
+template <typename T>
+std::enable_if_t<std::is_arithmetic_v<T>, void>
+hash_combine(size_t &seed, const T &value) {
+    seed ^= std::hash<T>{}(value) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
+}
+
+// Specialization for Tensor
+inline void hash_combine(size_t &seed, Tensor tensor) {
+    hash_combine(seed, static_cast<size_t>(tensor->dtype()));
+    for (Size shape : tensor->shape()) {
+        hash_combine(seed, shape);
+    }
+    for (Stride stride : tensor->strides()) {
+        hash_combine(seed, static_cast<size_t>(stride));
+    }
+}
+
+// Specialization for std::string
+inline void hash_combine(size_t &seed, const std::string &str) {
+    hash_combine(seed, std::hash<std::string>{}(str));
+}
+
+// Specialization for const char*
+inline void hash_combine(size_t &seed, const char *str) {
+    hash_combine(seed, std::string(str));
+}
+
+// Variadic template for multiple arguments
+template <typename First, typename... Rest>
+void hash_combine(size_t &seed, const First &first, const Rest &...rest) {
+    hash_combine(seed, first);
+    hash_combine(seed, rest...);
+}
+
+// Base case for variadic template
+inline void hash_combine(size_t &seed) {
+    // Base case - do nothing
+}
+
+// Convenience function to hash multiple values
+template <typename... Types>
+size_t hash_combine(const Types &...values) {
+    size_t seed = 0;
+    hash_combine(seed, values...);
+    return seed;
+}
+
+} // namespace infinicore

include/infinicore/context/context.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "../device.hpp"
+#include "../memory.hpp"
+
+#include <infiniop.h>
+#include <infinirt.h>
+
+#include <memory>
+
+namespace infinicore {
+
+namespace context {
+void setDevice(Device device);
+Device getDevice();
+size_t getDeviceCount(Device::Type type);
+
+infinirtStream_t getStream();
+infiniopHandle_t getInfiniopHandle();
+
+void syncStream();
+void syncDevice();
+
+std::shared_ptr<Memory> allocateMemory(size_t size);
+std::shared_ptr<Memory> allocateHostMemory(size_t size);
+std::shared_ptr<Memory> allocatePinnedHostMemory(size_t size);
+
+void memcpyH2D(void *dst, const void *src, size_t size);
+void memcpyD2H(void *dst, const void *src, size_t size);
+void memcpyD2D(void *dst, const void *src, size_t size);
+void memcpyH2H(void *dst, const void *src, size_t size);
+
+} // namespace context
+
+} // namespace infinicore

include/infinicore/device.hpp

@@ -1,30 +1,42 @@
-#ifndef __INFINICORE_DEVICE_API_HPP__
-#define __INFINICORE_DEVICE_API_HPP__
+#pragma once
 
 #include <cstdint>
 #include <string>
 
+#include "infinicore.h"
+
 namespace infinicore {
 
 class Device {
 public:
     using Index = std::size_t;
 
     enum class Type {
-        cpu,
-        cuda,
-        meta,
+        CPU = INFINI_DEVICE_CPU,
+        NVIDIA = INFINI_DEVICE_NVIDIA,
+        CAMBRICON = INFINI_DEVICE_CAMBRICON,
+        ASCEND = INFINI_DEVICE_ASCEND,
+        METAX = INFINI_DEVICE_METAX,
+        MOORE = INFINI_DEVICE_MOORE,
+        ILUVATAR = INFINI_DEVICE_ILUVATAR,
+        KUNLUN = INFINI_DEVICE_KUNLUN,
+        SUGON = INFINI_DEVICE_SUGON,
+        COUNT = INFINI_DEVICE_TYPE_COUNT,
     };
 
-    Device(const Type &type, const Index &index = 0);
+    Device(const Type &type = Type::CPU, const Index &index = 0);
+
+    const Type &getType() const;
 
-    const Type &get_type() const;
+    const Index &getIndex() const;
 
-    const Index &get_index() const;
+    std::string toString() const;
 
-    std::string to_string() const;
+    static std::string toString(const Type &type);
 
-    static std::string to_string(const Type &type);
+    bool operator==(const Device &other) const;
+
+    bool operator!=(const Device &other) const;
 
 private:
     Type type_;
@@ -33,5 +45,3 @@ class Device {
 };
 
 } // namespace infinicore
-
-#endif

include/infinicore/dtype.hpp

@@ -1,22 +1,33 @@
-#ifndef __INFINICORE_DTYPE_API_HPP__
-#define __INFINICORE_DTYPE_API_HPP__
+#pragma once
 
 #include <infinicore.h>
+#include <string>
 
 namespace infinicore {
 
 enum class DataType {
-    bfloat16 = INFINI_DTYPE_BF16,
-    float16 = INFINI_DTYPE_F16,
-    float32 = INFINI_DTYPE_F32,
-    float64 = INFINI_DTYPE_F64,
-    int32 = INFINI_DTYPE_I32,
-    int64 = INFINI_DTYPE_I64,
-    uint8 = INFINI_DTYPE_U8,
+    BYTE = INFINI_DTYPE_BYTE,
+    BOOL = INFINI_DTYPE_BOOL,
+    I8 = INFINI_DTYPE_I8,
+    I16 = INFINI_DTYPE_I16,
+    I32 = INFINI_DTYPE_I32,
+    I64 = INFINI_DTYPE_I64,
+    U8 = INFINI_DTYPE_U8,
+    U16 = INFINI_DTYPE_U16,
+    U32 = INFINI_DTYPE_U32,
+    U64 = INFINI_DTYPE_U64,
+    F8 = INFINI_DTYPE_F8,
+    F16 = INFINI_DTYPE_F16,
+    F32 = INFINI_DTYPE_F32,
+    F64 = INFINI_DTYPE_F64,
+    C16 = INFINI_DTYPE_C16,
+    C32 = INFINI_DTYPE_C32,
+    C64 = INFINI_DTYPE_C64,
+    C128 = INFINI_DTYPE_C128,
+    BF16 = INFINI_DTYPE_BF16,
 };
 
-std::string to_string(const DataType &dtype);
+std::string toString(const DataType &dtype);
+size_t dsize(const DataType &dtype);
 
 } // namespace infinicore
-
-#endif