|
| 1 | +#include "arg.h" |
| 2 | +#include "common.h" |
| 3 | +#include "log.h" |
| 4 | +#include "llama.h" |
| 5 | +#include "../src/llama-ext.h" |
| 6 | +#include "ggml.h" |
| 7 | + |
| 8 | +#include <array> |
| 9 | +#include <vector> |
| 10 | +#include <set> |
| 11 | +#include <fstream> |
| 12 | +#include <iostream> |
| 13 | + |
| 14 | +struct input_tensor { |
| 15 | + ggml_type type; |
| 16 | + std::array<int64_t, 4> ne; |
| 17 | + std::array<size_t, 4> nb; |
| 18 | + |
| 19 | + input_tensor(ggml_type type, int64_t * ne, size_t * nb): type(type) { |
| 20 | + memcpy(this->ne.data(), ne, 4 * sizeof(int64_t)); |
| 21 | + memcpy(this->nb.data(), nb, 4 * sizeof(size_t)); |
| 22 | + } |
| 23 | + |
| 24 | + bool operator<(const input_tensor &b) const { |
| 25 | + return std::tie(type, ne, nb) < |
| 26 | + std::tie(b.type, b.ne, b.nb); |
| 27 | + } |
| 28 | + |
| 29 | + void serialize(std::ostream& out) const { |
| 30 | + out << type << ' '; |
| 31 | + for (size_t i = 0; i < 4; i++) { |
| 32 | + out << ne[i] << ' '; |
| 33 | + } |
| 34 | + for (size_t i = 0; i < 4; i++) { |
| 35 | + out << nb[i] << ' '; |
| 36 | + } |
| 37 | + } |
| 38 | +}; |
| 39 | + |
| 40 | +struct test_object { |
| 41 | + ggml_op op; |
| 42 | + ggml_type type; |
| 43 | + std::array<int64_t, 4> ne; |
| 44 | + std::vector<int32_t> op_params; |
| 45 | + std::vector<input_tensor> sources; |
| 46 | + std::string name; |
| 47 | + |
| 48 | + void serialize(std::ostream& out) const { |
| 49 | + out << op << ' ' << type << ' '; |
| 50 | + for (size_t i = 0; i < 4; i++) { |
| 51 | + out << ne[i] << ' '; |
| 52 | + } |
| 53 | + |
| 54 | + out << op_params.size() << ' '; |
| 55 | + for (size_t i = 0; i < op_params.size(); i++) { |
| 56 | + out << op_params[i] << ' '; |
| 57 | + } |
| 58 | + |
| 59 | + out << sources.size() << ' '; |
| 60 | + for (size_t s = 0; s < sources.size(); s++) { |
| 61 | + sources[s].serialize(out); |
| 62 | + } |
| 63 | + |
| 64 | + if (!name.empty()) { |
| 65 | + out << name; |
| 66 | + } else { |
| 67 | + out << '-'; |
| 68 | + } |
| 69 | + |
| 70 | + out << '\n'; |
| 71 | + } |
| 72 | + |
| 73 | + bool operator<(const test_object &b) const { |
| 74 | + return std::tie(op, type, ne, op_params, sources) < |
| 75 | + std::tie(b.op, b.type, b.ne, b.op_params, b.sources); |
| 76 | + } |
| 77 | +}; |
| 78 | + |
| 79 | +static void extract_graph_ops(ggml_cgraph * cgraph, const char * label, std::set<test_object> & tests) { |
| 80 | + int n_nodes = ggml_graph_n_nodes(cgraph); |
| 81 | + int n_skipped = 0; |
| 82 | + int n_before = (int) tests.size(); |
| 83 | + for (int i = 0; i < n_nodes; i++) { |
| 84 | + ggml_tensor * node = ggml_graph_node(cgraph, i); |
| 85 | + |
| 86 | + if (node->op == GGML_OP_NONE || node->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) { |
| 87 | + n_skipped++; |
| 88 | + continue; |
| 89 | + } |
| 90 | + |
| 91 | + test_object test; |
| 92 | + |
| 93 | + test.op = node->op; |
| 94 | + test.type = node->type; |
| 95 | + memcpy(&test.ne, node->ne, 4 * sizeof(int64_t)); |
| 96 | + |
| 97 | + test.op_params.resize(GGML_MAX_OP_PARAMS / sizeof(int32_t)); |
| 98 | + memcpy(test.op_params.data(), node->op_params, GGML_MAX_OP_PARAMS); |
| 99 | + |
| 100 | + for (size_t s = 0; s < GGML_MAX_SRC; s++) { |
| 101 | + if (node->src[s] == nullptr) { |
| 102 | + break; |
| 103 | + } |
| 104 | + |
| 105 | + test.sources.emplace_back(node->src[s]->type, node->src[s]->ne, node->src[s]->nb); |
| 106 | + } |
| 107 | + |
| 108 | + test.name = node->name; |
| 109 | + tests.insert(test); |
| 110 | + } |
| 111 | + |
| 112 | + int n_new = (int) tests.size() - n_before; |
| 113 | + LOG_INF("%s: %d unique ops, %d total nodes, %d skipped (view ops)\n", |
| 114 | + label, n_new, n_nodes, n_skipped); |
| 115 | +} |
| 116 | + |
| 117 | +int main(int argc, char ** argv) { |
| 118 | + common_params params; |
| 119 | + params.out_file = "tests.txt"; |
| 120 | + |
| 121 | + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS)) { |
| 122 | + return 1; |
| 123 | + } |
| 124 | + |
| 125 | + common_init(); |
| 126 | + |
| 127 | + // Load CPU-only |
| 128 | + ggml_backend_dev_t cpu_device = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); |
| 129 | + params.devices = { cpu_device, nullptr }; |
| 130 | + params.fit_params = false; |
| 131 | + params.n_gpu_layers = 0; |
| 132 | + |
| 133 | + params.warmup = false; |
| 134 | + |
| 135 | + auto init_result = common_init_from_params(params); |
| 136 | + |
| 137 | + llama_context * ctx = init_result->context(); |
| 138 | + |
| 139 | + const uint32_t n_seqs = llama_n_seq_max(ctx); |
| 140 | + const uint32_t n_tokens = std::min(llama_n_ctx(ctx), llama_n_ubatch(ctx)); |
| 141 | + |
| 142 | + std::set<test_object> tests; |
| 143 | + |
| 144 | + auto * gf_pp = llama_graph_reserve(ctx, n_tokens, n_seqs, n_tokens); |
| 145 | + if (!gf_pp) { |
| 146 | + throw std::runtime_error("failed to reserve prompt processing graph"); |
| 147 | + } |
| 148 | + extract_graph_ops(gf_pp, "pp", tests); |
| 149 | + |
| 150 | + auto * gf_tg = llama_graph_reserve(ctx, n_seqs, n_seqs, n_seqs); |
| 151 | + if (!gf_tg) { |
| 152 | + throw std::runtime_error("failed to reserve token generation graph"); |
| 153 | + } |
| 154 | + extract_graph_ops(gf_tg, "tg", tests); |
| 155 | + |
| 156 | + LOG_INF("%d unique ops total\n", (int) tests.size()); |
| 157 | + |
| 158 | + std::ofstream f(params.out_file); |
| 159 | + |
| 160 | + if (!f.is_open()) { |
| 161 | + throw std::runtime_error("Unable to open output file"); |
| 162 | + } |
| 163 | + |
| 164 | + for (const auto& test : tests) { |
| 165 | + test.serialize(f); |
| 166 | + } |
| 167 | + |
| 168 | + return 0; |
| 169 | +} |
0 commit comments