diff --git a/src/cc/bcc_btf.cc b/src/cc/bcc_btf.cc index 233899617d5a..03e1ce889c14 100644 --- a/src/cc/bcc_btf.cc +++ b/src/cc/bcc_btf.cc @@ -199,4 +199,12 @@ int BTF::get_btf_info(const char *fname, return 0; } +int BTF::get_map_tids(std::string map_name, + unsigned expected_ksize, unsigned expected_vsize, + unsigned *key_tid, unsigned *value_tid) { + return btf__get_map_kv_tids(btf_, map_name.c_str(), + expected_ksize, expected_vsize, + key_tid, value_tid); +} + } // namespace ebpf diff --git a/src/cc/bcc_btf.h b/src/cc/bcc_btf.h index 334e179f7602..ffa8307e823d 100644 --- a/src/cc/bcc_btf.h +++ b/src/cc/bcc_btf.h @@ -53,6 +53,9 @@ class BTF { unsigned *finfo_rec_size, void **line_info, unsigned *line_info_cnt, unsigned *linfo_rec_size); + int get_map_tids(std::string map_name, + unsigned expected_ksize, unsigned expected_vsize, + unsigned *key_tid, unsigned *value_tid); private: void adjust(uint8_t *btf_sec, uintptr_t btf_sec_size, diff --git a/src/cc/bpf_module.cc b/src/cc/bpf_module.cc index 4d33ee8ecca3..f002454705d9 100644 --- a/src/cc/bpf_module.cc +++ b/src/cc/bpf_module.cc @@ -71,8 +71,9 @@ class MyMemoryManager : public SectionMemoryManager { uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, unsigned SectionID, StringRef SectionName) override { - uint8_t *Addr = SectionMemoryManager::allocateCodeSection(Size, Alignment, SectionID, SectionName); - //printf("allocateCodeSection: %s Addr %p Size %ld Alignment %d SectionID %d\n", + // The programs need to change from fake fd to real map fd, so not allocate ReadOnly regions. + uint8_t *Addr = SectionMemoryManager::allocateDataSection(Size, Alignment, SectionID, SectionName, false); + //printf("allocateDataSection: %s Addr %p Size %ld Alignment %d SectionID %d\n", // SectionName.str().c_str(), (void *)Addr, Size, Alignment, SectionID); (*sections_)[SectionName.str()] = make_tuple(Addr, Size); return Addr; @@ -157,7 +158,7 @@ int BPFModule::free_bcc_memory() { int BPFModule::load_cfile(const string &file, bool in_memory, const char *cflags[], int ncflags) { ClangLoader clang_loader(&*ctx_, flags_); if (clang_loader.parse(&mod_, *ts_, file, in_memory, cflags, ncflags, id_, - *func_src_, mod_src_, maps_ns_)) + *func_src_, mod_src_, maps_ns_, fake_fd_map_)) return -1; return 0; } @@ -170,7 +171,7 @@ int BPFModule::load_cfile(const string &file, bool in_memory, const char *cflags int BPFModule::load_includes(const string &text) { ClangLoader clang_loader(&*ctx_, flags_); if (clang_loader.parse(&mod_, *ts_, text, true, nullptr, 0, "", *func_src_, - mod_src_, "")) + mod_src_, "", fake_fd_map_)) return -1; return 0; } @@ -266,6 +267,110 @@ void BPFModule::load_btf(std::map> btf_ = btf; } +int BPFModule::load_maps(std::map> §ions) { + // find .maps. sections and retrieve all map key/value type id's + std::map> map_tids; + if (btf_) { + for (auto section : sections) { + auto sec_name = section.first; + if (strncmp(".maps.", sec_name.c_str(), 6) == 0) { + std::string map_name = sec_name.substr(6); + unsigned key_tid = 0, value_tid = 0; + unsigned expected_ksize = 0, expected_vsize = 0; + + for (auto map : fake_fd_map_) { + std::string name; + + name = get<1>(map.second); + if (map_name == name) { + expected_ksize = get<2>(map.second); + expected_vsize = get<3>(map.second); + break; + } + } + + int ret = btf_->get_map_tids(map_name, expected_ksize, + expected_vsize, &key_tid, &value_tid); + if (ret) + continue; + + map_tids[map_name] = std::make_pair(key_tid, value_tid); + } + } + } + + // create maps + std::map map_fds; + for (auto map : fake_fd_map_) { + int fd, fake_fd, map_type, key_size, value_size, max_entries, map_flags; + const char *map_name; + + fake_fd = map.first; + map_type = get<0>(map.second); + map_name = get<1>(map.second).c_str(); + key_size = get<2>(map.second); + value_size = get<3>(map.second); + max_entries = get<4>(map.second); + map_flags = get<5>(map.second); + + struct bpf_create_map_attr attr = {}; + attr.map_type = (enum bpf_map_type)map_type; + attr.name = map_name; + attr.key_size = key_size; + attr.value_size = value_size; + attr.max_entries = max_entries; + attr.map_flags = map_flags; + + if (map_tids.find(map_name) != map_tids.end()) { + attr.btf_fd = btf_->get_fd(); + attr.btf_key_type_id = map_tids[map_name].first; + attr.btf_value_type_id = map_tids[map_name].second; + } + + fd = bcc_create_map_xattr(&attr); + if (fd < 0) { + fprintf(stderr, "could not open bpf map: %s\nis map type enabled in your kernel?\n", + map_name); + return -1; + } + + map_fds[fake_fd] = fd; + } + + // update map table fd's + for (auto it = ts_->begin(), up = ts_->end(); it != up; ++it) { + TableDesc &table = it->second; + if (map_fds.find(table.fake_fd) != map_fds.end()) { + table.fd = map_fds[table.fake_fd]; + table.fake_fd = 0; + } + } + + // update instructions + for (auto section : sections) { + auto sec_name = section.first; + if (strncmp(".bpf.fn.", sec_name.c_str(), 8) == 0) { + uint8_t *addr = get<0>(section.second); + uintptr_t size = get<1>(section.second); + struct bpf_insn *insns = (struct bpf_insn *)addr; + int i, num_insns; + + num_insns = size/sizeof(struct bpf_insn); + for (i = 0; i < num_insns; i++) { + if (insns[i].code == (BPF_LD | BPF_DW | BPF_IMM)) { + // change map_fd is it is a ld_pseudo */ + if (insns[i].src_reg == BPF_PSEUDO_MAP_FD && + map_fds.find(insns[i].imm) != map_fds.end()) + insns[i].imm = map_fds[insns[i].imm]; + i++; + } + } + } + } + + return 0; +} + int BPFModule::finalize() { Module *mod = &*mod_; std::map> tmp_sections, @@ -310,6 +415,8 @@ int BPFModule::finalize() { } load_btf(*sections_p); + if (load_maps(*sections_p)) + return -1; if (!rw_engine_enabled_) { // Setup sections_ correctly and then free llvm internal memory diff --git a/src/cc/bpf_module.h b/src/cc/bpf_module.h index ce31c6ad57e7..bd79455ceb5c 100644 --- a/src/cc/bpf_module.h +++ b/src/cc/bpf_module.h @@ -23,6 +23,7 @@ #include #include "bcc_exception.h" +#include "table_storage.h" namespace llvm { class ExecutionEngine; @@ -80,6 +81,7 @@ class BPFModule { StatusTuple snprintf(std::string fn_name, char *str, size_t sz, const void *val); void load_btf(std::map> §ions); + int load_maps(std::map> §ions); public: BPFModule(unsigned flags, TableStorage *ts = nullptr, bool rw_engine_enabled = true, @@ -156,6 +158,7 @@ class BPFModule { TableStorage *ts_; std::unique_ptr local_ts_; BTF *btf_; + fake_fd_map_def fake_fd_map_; }; } // namespace ebpf diff --git a/src/cc/export/helpers.h b/src/cc/export/helpers.h index c025da5a25d3..9954e0ce6726 100644 --- a/src/cc/export/helpers.h +++ b/src/cc/export/helpers.h @@ -53,6 +53,16 @@ R"********( */ #define SEC(NAME) __attribute__((section(NAME), used)) +// Associate map with its key/value types +#define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \ + struct ____btf_map_##name { \ + type_key key; \ + type_val value; \ + }; \ + struct ____btf_map_##name \ + __attribute__ ((section(".maps." #name), used)) \ + ____btf_map_##name = { } + // Changes to the macro require changes in BFrontendAction classes #define BPF_F_TABLE(_table_type, _key_type, _leaf_type, _name, _max_entries, _flags) \ struct _name##_table_t { \ @@ -70,7 +80,8 @@ struct _name##_table_t { \ int flags; \ }; \ __attribute__((section("maps/" _table_type))) \ -struct _name##_table_t _name = { .flags = (_flags), .max_entries = (_max_entries) } +struct _name##_table_t _name = { .flags = (_flags), .max_entries = (_max_entries) }; \ +BPF_ANNOTATE_KV_PAIR(_name, _key_type, _leaf_type) #define BPF_TABLE(_table_type, _key_type, _leaf_type, _name, _max_entries) \ BPF_F_TABLE(_table_type, _key_type, _leaf_type, _name, _max_entries, 0) diff --git a/src/cc/file_desc.h b/src/cc/file_desc.h index a55ba0b679c2..d4ab310db9db 100644 --- a/src/cc/file_desc.h +++ b/src/cc/file_desc.h @@ -49,8 +49,12 @@ class FileDesc { FileDesc &operator=(const FileDesc &that) = delete; FileDesc dup() const { - int dup_fd = ::dup(fd_); - return FileDesc(dup_fd); + if (fd_ >= 0) { + int dup_fd = ::dup(fd_); + return FileDesc(dup_fd); + } else { + return FileDesc(-1); + } } operator int() { return fd_; } diff --git a/src/cc/frontends/clang/b_frontend_action.cc b/src/cc/frontends/clang/b_frontend_action.cc index 01396f7bff66..8f734db48635 100644 --- a/src/cc/frontends/clang/b_frontend_action.cc +++ b/src/cc/frontends/clang/b_frontend_action.cc @@ -766,7 +766,7 @@ bool BTypeVisitor::VisitCallExpr(CallExpr *Call) { return false; } } - string fd = to_string(desc->second.fd); + string fd = to_string(desc->second.fd >= 0 ? desc->second.fd : desc->second.fake_fd); string prefix, suffix; string txt; auto rewrite_start = GET_BEGINLOC(Call); @@ -1230,14 +1230,10 @@ bool BTypeVisitor::VisitVarDecl(VarDecl *Decl) { } table.type = map_type; - table.fd = bcc_create_map(map_type, table.name.c_str(), - table.key_size, table.leaf_size, - table.max_entries, table.flags); - } - if (table.fd < 0) { - error(GET_BEGINLOC(Decl), "could not open bpf map: %0\nis %1 map type enabled in your kernel?") << - strerror(errno) << A->getName(); - return false; + table.fake_fd = fe_.get_next_fake_fd(); + fe_.add_map_def(table.fake_fd, std::make_tuple((int)map_type, std::string(table.name), + (int)table.key_size, (int)table.leaf_size, + (int)table.max_entries, table.flags)); } if (!table.is_extern) @@ -1351,7 +1347,8 @@ BFrontendAction::BFrontendAction(llvm::raw_ostream &os, unsigned flags, TableStorage &ts, const std::string &id, const std::string &main_path, FuncSource &func_src, std::string &mod_src, - const std::string &maps_ns) + const std::string &maps_ns, + fake_fd_map_def &fake_fd_map) : os_(os), flags_(flags), ts_(ts), @@ -1360,7 +1357,9 @@ BFrontendAction::BFrontendAction(llvm::raw_ostream &os, unsigned flags, rewriter_(new Rewriter), main_path_(main_path), func_src_(func_src), - mod_src_(mod_src) {} + mod_src_(mod_src), + next_fake_fd_(-1), + fake_fd_map_(fake_fd_map) {} bool BFrontendAction::is_rewritable_ext_func(FunctionDecl *D) { StringRef file_name = rewriter_->getSourceMgr().getFilename(GET_BEGINLOC(D)); diff --git a/src/cc/frontends/clang/b_frontend_action.h b/src/cc/frontends/clang/b_frontend_action.h index 37aea82051c5..bc6690cbd9e9 100644 --- a/src/cc/frontends/clang/b_frontend_action.h +++ b/src/cc/frontends/clang/b_frontend_action.h @@ -155,7 +155,8 @@ class BFrontendAction : public clang::ASTFrontendAction { BFrontendAction(llvm::raw_ostream &os, unsigned flags, TableStorage &ts, const std::string &id, const std::string &main_path, FuncSource &func_src, std::string &mod_src, - const std::string &maps_ns); + const std::string &maps_ns, + fake_fd_map_def &fake_fd_map); // Called by clang when the AST has been completed, here the output stream // will be flushed. @@ -170,6 +171,11 @@ class BFrontendAction : public clang::ASTFrontendAction { std::string maps_ns() const { return maps_ns_; } bool is_rewritable_ext_func(clang::FunctionDecl *D); void DoMiscWorkAround(); + // negative fake_fd to be different from real fd in bpf_pseudo_fd. + int get_next_fake_fd() { return next_fake_fd_--; } + void add_map_def(int fd, std::tuple map_def) { + fake_fd_map_[fd] = map_def; + } private: llvm::raw_ostream &os_; @@ -184,6 +190,8 @@ class BFrontendAction : public clang::ASTFrontendAction { FuncSource &func_src_; std::string &mod_src_; std::set m_; + int next_fake_fd_; + fake_fd_map_def &fake_fd_map_; }; } // namespace visitor diff --git a/src/cc/frontends/clang/loader.cc b/src/cc/frontends/clang/loader.cc index 8f091708f65e..3e579ba190b2 100644 --- a/src/cc/frontends/clang/loader.cc +++ b/src/cc/frontends/clang/loader.cc @@ -108,7 +108,8 @@ int ClangLoader::parse(unique_ptr *mod, TableStorage &ts, const string &file, bool in_memory, const char *cflags[], int ncflags, const std::string &id, FuncSource &func_src, std::string &mod_src, - const std::string &maps_ns) { + const std::string &maps_ns, + fake_fd_map_def &fake_fd_map) { string main_path = "/virtual/main.c"; unique_ptr main_buf; struct utsname un; @@ -205,7 +206,7 @@ int ClangLoader::parse(unique_ptr *mod, TableStorage &ts, #endif if (do_compile(mod, ts, in_memory, flags_cstr, flags_cstr_rem, main_path, - main_buf, id, func_src, mod_src, true, maps_ns)) { + main_buf, id, func_src, mod_src, true, maps_ns, fake_fd_map)) { #if BCC_BACKUP_COMPILE != 1 return -1; #else @@ -215,8 +216,9 @@ int ClangLoader::parse(unique_ptr *mod, TableStorage &ts, ts.DeletePrefix(Path({id})); func_src.clear(); mod_src.clear(); + fake_fd_map.clear(); if (do_compile(mod, ts, in_memory, flags_cstr, flags_cstr_rem, main_path, - main_buf, id, func_src, mod_src, false, maps_ns)) + main_buf, id, func_src, mod_src, false, maps_ns, fake_fd_map)) return -1; #endif } @@ -263,7 +265,8 @@ int ClangLoader::do_compile(unique_ptr *mod, TableStorage &ts, const unique_ptr &main_buf, const std::string &id, FuncSource &func_src, std::string &mod_src, bool use_internal_bpfh, - const std::string &maps_ns) { + const std::string &maps_ns, + fake_fd_map_def &fake_fd_map) { using namespace clang; vector flags_cstr = flags_cstr_in; @@ -377,7 +380,7 @@ int ClangLoader::do_compile(unique_ptr *mod, TableStorage &ts, // capture the rewritten c file string out_str1; llvm::raw_string_ostream os1(out_str1); - BFrontendAction bact(os1, flags_, ts, id, main_path, func_src, mod_src, maps_ns); + BFrontendAction bact(os1, flags_, ts, id, main_path, func_src, mod_src, maps_ns, fake_fd_map); if (!compiler1.ExecuteAction(bact)) return -1; unique_ptr out_buf1 = llvm::MemoryBuffer::getMemBuffer(out_str1); diff --git a/src/cc/frontends/clang/loader.h b/src/cc/frontends/clang/loader.h index 1aeb6523a3d3..984ca2fdbf18 100644 --- a/src/cc/frontends/clang/loader.h +++ b/src/cc/frontends/clang/loader.h @@ -54,7 +54,8 @@ class ClangLoader { int parse(std::unique_ptr *mod, TableStorage &ts, const std::string &file, bool in_memory, const char *cflags[], int ncflags, const std::string &id, FuncSource &func_src, - std::string &mod_src, const std::string &maps_ns); + std::string &mod_src, const std::string &maps_ns, + fake_fd_map_def &fake_fd_map); private: int do_compile(std::unique_ptr *mod, TableStorage &ts, @@ -64,7 +65,8 @@ class ClangLoader { const std::unique_ptr &main_buf, const std::string &id, FuncSource &func_src, std::string &mod_src, bool use_internal_bpfh, - const std::string &maps_ns); + const std::string &maps_ns, + fake_fd_map_def &fake_fd_map); private: std::map> remapped_headers_; diff --git a/src/cc/libbpf.c b/src/cc/libbpf.c index c4452317b2a2..612bdd18cd00 100644 --- a/src/cc/libbpf.c +++ b/src/cc/libbpf.c @@ -206,6 +206,16 @@ int bcc_create_map_xattr(struct bpf_create_map_attr *attr) memcpy(map_name, attr->name, min(name_len, BPF_OBJ_NAME_LEN - 1)); attr->name = map_name; int ret = bpf_create_map_xattr(attr); + + // kernel already supports btf if its loading is successful, + // but this map type may not support pretty print yet. + if (ret < 0 && attr->btf_key_type_id && errno == 524 /* ENOTSUPP */) { + attr->btf_fd = 0; + attr->btf_key_type_id = 0; + attr->btf_value_type_id = 0; + ret = bpf_create_map_xattr(attr); + } + if (ret < 0 && name_len && (errno == E2BIG || errno == EINVAL)) { map_name[0] = '\0'; ret = bpf_create_map_xattr(attr); diff --git a/src/cc/table_desc.h b/src/cc/table_desc.h index da0927f9441a..3cb9393ec60d 100644 --- a/src/cc/table_desc.h +++ b/src/cc/table_desc.h @@ -45,6 +45,7 @@ class TableDesc { TableDesc(const TableDesc &that) : name(that.name), fd(that.fd.dup()), + fake_fd(that.fake_fd), type(that.type), key_size(that.key_size), leaf_size(that.leaf_size), @@ -61,7 +62,8 @@ class TableDesc { public: TableDesc() - : type(0), + : fake_fd(0), + type(0), key_size(0), leaf_size(0), max_entries(0), @@ -88,6 +90,7 @@ class TableDesc { std::string name; FileDesc fd; + int fake_fd; int type; size_t key_size; // sizes are in bytes size_t leaf_size; diff --git a/src/cc/table_storage.h b/src/cc/table_storage.h index 87aaa3383148..2df99c59713e 100644 --- a/src/cc/table_storage.h +++ b/src/cc/table_storage.h @@ -27,6 +27,8 @@ namespace ebpf { +typedef std::map> fake_fd_map_def; + class TableStorageImpl; class TableStorageIteratorImpl;