| /* |
| SPDX-License-Identifier: GPL-2.0-only |
| |
| Copyright (C) 2019 Facebook |
| |
| Derived from ctf_encoder.c, which is: |
| |
| Copyright (C) Arnaldo Carvalho de Melo <acme@redhat.com> |
| Copyright (C) Red Hat Inc |
| */ |
| |
| #include <linux/btf.h> |
| #include "dwarves.h" |
| #include "elf_symtab.h" |
| #include "btf_encoder.h" |
| #include "gobuffer.h" |
| |
| #include <bpf/btf.h> |
| #include <bpf/libbpf.h> |
| #include <ctype.h> /* for isalpha() and isalnum() */ |
| #include <stdlib.h> /* for qsort() and bsearch() */ |
| #include <inttypes.h> |
| #include <limits.h> |
| |
| #include <sys/types.h> |
| #include <sys/stat.h> |
| #include <fcntl.h> |
| |
| #include <unistd.h> |
| |
| #include <errno.h> |
| #include <stdint.h> |
| #include <search.h> /* for tsearch(), tfind() and tdestroy() */ |
| #include <pthread.h> |
| |
| #define BTF_ENCODER_MAX_PROTO 512 |
| |
| /* state used to do later encoding of saved functions */ |
| struct btf_encoder_state { |
| uint32_t type_id_off; |
| bool got_proto; |
| char proto[BTF_ENCODER_MAX_PROTO]; |
| }; |
| |
| struct elf_function { |
| const char *name; |
| bool generated; |
| size_t prefixlen; |
| struct function *function; |
| struct btf_encoder_state state; |
| }; |
| |
| #define MAX_PERCPU_VAR_CNT 4096 |
| |
| struct var_info { |
| uint64_t addr; |
| const char *name; |
| uint32_t sz; |
| }; |
| |
| /* |
| * cu: cu being processed. |
| */ |
| struct btf_encoder { |
| struct list_head node; |
| struct btf *btf; |
| struct cu *cu; |
| struct gobuffer percpu_secinfo; |
| const char *filename; |
| struct elf_symtab *symtab; |
| uint32_t type_id_off; |
| int saved_func_cnt; |
| bool has_index_type, |
| need_index_type, |
| skip_encoding_vars, |
| raw_output, |
| verbose, |
| force, |
| gen_floats, |
| is_rel; |
| uint32_t array_index_id; |
| struct { |
| struct var_info vars[MAX_PERCPU_VAR_CNT]; |
| int var_cnt; |
| uint32_t shndx; |
| uint64_t base_addr; |
| uint64_t sec_sz; |
| } percpu; |
| struct { |
| struct elf_function *entries; |
| int allocated; |
| int cnt; |
| int suffix_cnt; /* number of .isra, .part etc */ |
| } functions; |
| }; |
| |
| static LIST_HEAD(encoders); |
| static pthread_mutex_t encoders__lock = PTHREAD_MUTEX_INITIALIZER; |
| |
| static void btf_encoder__add_saved_funcs(struct btf_encoder *encoder); |
| |
| /* mutex only needed for add/delete, as this can happen in multiple encoding |
| * threads. Traversal of the list is currently confined to thread collection. |
| */ |
| |
| #define btf_encoders__for_each_encoder(encoder) \ |
| list_for_each_entry(encoder, &encoders, node) |
| |
| static void btf_encoders__add(struct btf_encoder *encoder) |
| { |
| pthread_mutex_lock(&encoders__lock); |
| list_add_tail(&encoder->node, &encoders); |
| pthread_mutex_unlock(&encoders__lock); |
| } |
| |
| static void btf_encoders__delete(struct btf_encoder *encoder) |
| { |
| struct btf_encoder *existing = NULL; |
| |
| pthread_mutex_lock(&encoders__lock); |
| /* encoder may not have been added to list yet; check. */ |
| btf_encoders__for_each_encoder(existing) { |
| if (encoder == existing) |
| break; |
| } |
| if (encoder == existing) |
| list_del(&encoder->node); |
| pthread_mutex_unlock(&encoders__lock); |
| } |
| |
| #define PERCPU_SECTION ".data..percpu" |
| |
| /* |
| * This depends on the GNU extension to eliminate the stray comma in the zero |
| * arguments case. |
| * |
| * The difference between elf_errmsg(-1) and elf_errmsg(elf_errno()) is that the |
| * latter clears the current error. |
| */ |
| #define elf_error(fmt, ...) \ |
| fprintf(stderr, "%s: " fmt ": %s.\n", __func__, ##__VA_ARGS__, elf_errmsg(-1)) |
| |
| /* |
| * This depends on the GNU extension to eliminate the stray comma in the zero |
| * arguments case. |
| * |
| * The difference between elf_errmsg(-1) and elf_errmsg(elf_errno()) is that the |
| * latter clears the current error. |
| */ |
| #define elf_error(fmt, ...) \ |
| fprintf(stderr, "%s: " fmt ": %s.\n", __func__, ##__VA_ARGS__, elf_errmsg(-1)) |
| |
| static int btf_var_secinfo_cmp(const void *a, const void *b) |
| { |
| const struct btf_var_secinfo *av = a; |
| const struct btf_var_secinfo *bv = b; |
| |
| return av->offset - bv->offset; |
| } |
| |
| #define BITS_PER_BYTE 8 |
| #define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) |
| #define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK) |
| #define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3) |
| #define BITS_ROUNDUP_BYTES(bits) (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) |
| |
| static const char * const btf_kind_str[] = { |
| [BTF_KIND_UNKN] = "UNKNOWN", |
| [BTF_KIND_INT] = "INT", |
| [BTF_KIND_PTR] = "PTR", |
| [BTF_KIND_ARRAY] = "ARRAY", |
| [BTF_KIND_STRUCT] = "STRUCT", |
| [BTF_KIND_UNION] = "UNION", |
| [BTF_KIND_ENUM] = "ENUM", |
| [BTF_KIND_FWD] = "FWD", |
| [BTF_KIND_TYPEDEF] = "TYPEDEF", |
| [BTF_KIND_VOLATILE] = "VOLATILE", |
| [BTF_KIND_CONST] = "CONST", |
| [BTF_KIND_RESTRICT] = "RESTRICT", |
| [BTF_KIND_FUNC] = "FUNC", |
| [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO", |
| [BTF_KIND_VAR] = "VAR", |
| [BTF_KIND_DATASEC] = "DATASEC", |
| [BTF_KIND_FLOAT] = "FLOAT", |
| [BTF_KIND_DECL_TAG] = "DECL_TAG", |
| [BTF_KIND_TYPE_TAG] = "TYPE_TAG", |
| [BTF_KIND_ENUM64] = "ENUM64", |
| }; |
| |
| static const char *btf__printable_name(const struct btf *btf, uint32_t offset) |
| { |
| if (!offset) |
| return "(anon)"; |
| else |
| return btf__str_by_offset(btf, offset); |
| } |
| |
| static const char * btf__int_encoding_str(uint8_t encoding) |
| { |
| if (encoding == 0) |
| return "(none)"; |
| else if (encoding == BTF_INT_SIGNED) |
| return "SIGNED"; |
| else if (encoding == BTF_INT_CHAR) |
| return "CHAR"; |
| else if (encoding == BTF_INT_BOOL) |
| return "BOOL"; |
| else |
| return "UNKN"; |
| } |
| |
| __attribute ((format (printf, 5, 6))) |
| static void btf__log_err(const struct btf *btf, int kind, const char *name, |
| bool output_cr, const char *fmt, ...) |
| { |
| fprintf(stderr, "[%u] %s %s", btf__type_cnt(btf), |
| btf_kind_str[kind], name ?: "(anon)"); |
| |
| if (fmt && *fmt) { |
| va_list ap; |
| |
| fprintf(stderr, " "); |
| va_start(ap, fmt); |
| vfprintf(stderr, fmt, ap); |
| va_end(ap); |
| } |
| |
| if (output_cr) |
| fprintf(stderr, "\n"); |
| } |
| |
| __attribute ((format (printf, 5, 6))) |
| static void btf_encoder__log_type(const struct btf_encoder *encoder, const struct btf_type *t, |
| bool err, bool output_cr, const char *fmt, ...) |
| { |
| const struct btf *btf = encoder->btf; |
| uint8_t kind; |
| FILE *out; |
| |
| if (!encoder->verbose && !err) |
| return; |
| |
| kind = BTF_INFO_KIND(t->info); |
| out = err ? stderr : stdout; |
| |
| fprintf(out, "[%u] %s %s", |
| btf__type_cnt(btf) - 1, btf_kind_str[kind], |
| btf__printable_name(btf, t->name_off)); |
| |
| if (fmt && *fmt) { |
| va_list ap; |
| |
| fprintf(out, " "); |
| va_start(ap, fmt); |
| vfprintf(out, fmt, ap); |
| va_end(ap); |
| } |
| |
| if (output_cr) |
| fprintf(out, "\n"); |
| } |
| |
| __attribute ((format (printf, 5, 6))) |
| static void btf_encoder__log_member(const struct btf_encoder *encoder, const struct btf_type *t, |
| const struct btf_member *member, bool err, const char *fmt, ...) |
| { |
| const struct btf *btf = encoder->btf; |
| FILE *out; |
| |
| if (!encoder->verbose && !err) |
| return; |
| |
| out = err ? stderr : stdout; |
| |
| if (btf_kflag(t)) |
| fprintf(out, "\t%s type_id=%u bitfield_size=%u bits_offset=%u", |
| btf__printable_name(btf, member->name_off), |
| member->type, |
| BTF_MEMBER_BITFIELD_SIZE(member->offset), |
| BTF_MEMBER_BIT_OFFSET(member->offset)); |
| else |
| fprintf(out, "\t%s type_id=%u bits_offset=%u", |
| btf__printable_name(btf, member->name_off), |
| member->type, |
| member->offset); |
| |
| if (fmt && *fmt) { |
| va_list ap; |
| |
| fprintf(out, " "); |
| va_start(ap, fmt); |
| vfprintf(out, fmt, ap); |
| va_end(ap); |
| } |
| |
| fprintf(out, "\n"); |
| } |
| |
| __attribute ((format (printf, 6, 7))) |
| static void btf_encoder__log_func_param(struct btf_encoder *encoder, const char *name, uint32_t type, |
| bool err, bool is_last_param, const char *fmt, ...) |
| { |
| FILE *out; |
| |
| if (!encoder->verbose && !err) |
| return; |
| |
| out = err ? stderr : stdout; |
| |
| if (is_last_param && !type) |
| fprintf(out, "vararg)\n"); |
| else |
| fprintf(out, "%u %s%s", type, name, is_last_param ? ")\n" : ", "); |
| |
| if (fmt && *fmt) { |
| va_list ap; |
| |
| fprintf(out, " "); |
| va_start(ap, fmt); |
| vfprintf(out, fmt, ap); |
| va_end(ap); |
| } |
| } |
| |
| static int32_t btf_encoder__add_float(struct btf_encoder *encoder, const struct base_type *bt, const char *name) |
| { |
| int32_t id = btf__add_float(encoder->btf, name, BITS_ROUNDUP_BYTES(bt->bit_size)); |
| |
| if (id < 0) { |
| btf__log_err(encoder->btf, BTF_KIND_FLOAT, name, true, "Error emitting BTF type"); |
| } else { |
| const struct btf_type *t; |
| |
| t = btf__type_by_id(encoder->btf, id); |
| btf_encoder__log_type(encoder, t, false, true, "size=%u nr_bits=%u", t->size, bt->bit_size); |
| } |
| |
| return id; |
| } |
| |
| static int32_t btf_encoder__add_base_type(struct btf_encoder *encoder, const struct base_type *bt, const char *name) |
| { |
| const struct btf_type *t; |
| uint8_t encoding = 0; |
| uint16_t byte_sz; |
| int32_t id; |
| |
| if (bt->is_signed) { |
| encoding = BTF_INT_SIGNED; |
| } else if (bt->is_bool) { |
| encoding = BTF_INT_BOOL; |
| } else if (bt->float_type && encoder->gen_floats) { |
| /* |
| * Encode floats as BTF_KIND_FLOAT if allowed, otherwise (in |
| * compatibility mode) encode them as BTF_KIND_INT - that's not |
| * fully correct, but that's what it used to be. |
| */ |
| if (bt->float_type == BT_FP_SINGLE || |
| bt->float_type == BT_FP_DOUBLE || |
| bt->float_type == BT_FP_LDBL) |
| return btf_encoder__add_float(encoder, bt, name); |
| fprintf(stderr, "Complex, interval and imaginary float types are not supported\n"); |
| return -1; |
| } |
| |
| /* dwarf5 may emit DW_ATE_[un]signed_{num} base types where |
| * {num} is not power of 2 and may exceed 128. Such attributes |
| * are mostly used to record operation for an actual parameter |
| * or variable. |
| * For example, |
| * DW_AT_location (indexed (0x3c) loclist = 0x00008fb0: |
| * [0xffffffff82808812, 0xffffffff82808817): |
| * DW_OP_breg0 RAX+0, |
| * DW_OP_convert (0x000e97d5) "DW_ATE_unsigned_64", |
| * DW_OP_convert (0x000e97df) "DW_ATE_unsigned_8", |
| * DW_OP_stack_value, |
| * DW_OP_piece 0x1, |
| * DW_OP_breg0 RAX+0, |
| * DW_OP_convert (0x000e97d5) "DW_ATE_unsigned_64", |
| * DW_OP_convert (0x000e97da) "DW_ATE_unsigned_32", |
| * DW_OP_lit8, |
| * DW_OP_shr, |
| * DW_OP_convert (0x000e97da) "DW_ATE_unsigned_32", |
| * DW_OP_convert (0x000e97e4) "DW_ATE_unsigned_24", |
| * DW_OP_stack_value, DW_OP_piece 0x3 |
| * DW_AT_name ("ebx") |
| * DW_AT_decl_file ("/linux/arch/x86/events/intel/core.c") |
| * |
| * In the above example, at some point, one unsigned_32 value |
| * is right shifted by 8 and the result is converted to unsigned_32 |
| * and then unsigned_24. |
| * |
| * BTF does not need such DW_OP_* information so let us sanitize |
| * these non-regular int types to avoid libbpf/kernel complaints. |
| */ |
| byte_sz = BITS_ROUNDUP_BYTES(bt->bit_size); |
| if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) { |
| name = "__SANITIZED_FAKE_INT__"; |
| byte_sz = 4; |
| } |
| |
| id = btf__add_int(encoder->btf, name, byte_sz, encoding); |
| if (id < 0) { |
| btf__log_err(encoder->btf, BTF_KIND_INT, name, true, "Error emitting BTF type"); |
| } else { |
| t = btf__type_by_id(encoder->btf, id); |
| btf_encoder__log_type(encoder, t, false, true, "size=%u nr_bits=%u encoding=%s%s", |
| t->size, bt->bit_size, btf__int_encoding_str(encoding), |
| id < 0 ? " Error in emitting BTF" : "" ); |
| } |
| |
| return id; |
| } |
| |
| static int32_t btf_encoder__add_ref_type(struct btf_encoder *encoder, uint16_t kind, uint32_t type, |
| const char *name, bool kind_flag) |
| { |
| struct btf *btf = encoder->btf; |
| const struct btf_type *t; |
| int32_t id; |
| |
| switch (kind) { |
| case BTF_KIND_PTR: |
| id = btf__add_ptr(btf, type); |
| break; |
| case BTF_KIND_VOLATILE: |
| id = btf__add_volatile(btf, type); |
| break; |
| case BTF_KIND_CONST: |
| id = btf__add_const(btf, type); |
| break; |
| case BTF_KIND_RESTRICT: |
| id = btf__add_restrict(btf, type); |
| break; |
| case BTF_KIND_TYPEDEF: |
| id = btf__add_typedef(btf, name, type); |
| break; |
| case BTF_KIND_TYPE_TAG: |
| id = btf__add_type_tag(btf, name, type); |
| break; |
| case BTF_KIND_FWD: |
| id = btf__add_fwd(btf, name, kind_flag); |
| break; |
| case BTF_KIND_FUNC: |
| id = btf__add_func(btf, name, BTF_FUNC_STATIC, type); |
| break; |
| default: |
| btf__log_err(btf, kind, name, true, "Unexpected kind for reference"); |
| return -1; |
| } |
| |
| if (id > 0) { |
| t = btf__type_by_id(btf, id); |
| if (kind == BTF_KIND_FWD) |
| btf_encoder__log_type(encoder, t, false, true, "%s", kind_flag ? "union" : "struct"); |
| else |
| btf_encoder__log_type(encoder, t, false, true, "type_id=%u", t->type); |
| } else { |
| btf__log_err(btf, kind, name, true, "Error emitting BTF type"); |
| } |
| return id; |
| } |
| |
| static int32_t btf_encoder__add_array(struct btf_encoder *encoder, uint32_t type, uint32_t index_type, uint32_t nelems) |
| { |
| struct btf *btf = encoder->btf; |
| const struct btf_type *t; |
| const struct btf_array *array; |
| int32_t id; |
| |
| id = btf__add_array(btf, index_type, type, nelems); |
| if (id > 0) { |
| t = btf__type_by_id(btf, id); |
| array = btf_array(t); |
| btf_encoder__log_type(encoder, t, false, true, "type_id=%u index_type_id=%u nr_elems=%u", |
| array->type, array->index_type, array->nelems); |
| } else { |
| btf__log_err(btf, BTF_KIND_ARRAY, NULL, true, |
| "type_id=%u index_type_id=%u nr_elems=%u Error emitting BTF type", |
| type, index_type, nelems); |
| } |
| return id; |
| } |
| |
| static int btf_encoder__add_field(struct btf_encoder *encoder, const char *name, uint32_t type, uint32_t bitfield_size, uint32_t offset) |
| { |
| struct btf *btf = encoder->btf; |
| const struct btf_type *t; |
| const struct btf_member *m; |
| int err; |
| |
| err = btf__add_field(btf, name, type, offset, bitfield_size); |
| t = btf__type_by_id(btf, btf__type_cnt(btf) - 1); |
| if (err) { |
| fprintf(stderr, "[%u] %s %s's field '%s' offset=%u bit_size=%u type=%u Error emitting field\n", |
| btf__type_cnt(btf) - 1, btf_kind_str[btf_kind(t)], |
| btf__printable_name(btf, t->name_off), |
| name, offset, bitfield_size, type); |
| } else { |
| m = &btf_members(t)[btf_vlen(t) - 1]; |
| btf_encoder__log_member(encoder, t, m, false, NULL); |
| } |
| return err; |
| } |
| |
| static int32_t btf_encoder__add_struct(struct btf_encoder *encoder, uint8_t kind, const char *name, uint32_t size) |
| { |
| struct btf *btf = encoder->btf; |
| const struct btf_type *t; |
| int32_t id; |
| |
| switch (kind) { |
| case BTF_KIND_STRUCT: |
| id = btf__add_struct(btf, name, size); |
| break; |
| case BTF_KIND_UNION: |
| id = btf__add_union(btf, name, size); |
| break; |
| default: |
| btf__log_err(btf, kind, name, true, "Unexpected kind of struct"); |
| return -1; |
| } |
| |
| if (id < 0) { |
| btf__log_err(btf, kind, name, true, "Error emitting BTF type"); |
| } else { |
| t = btf__type_by_id(btf, id); |
| btf_encoder__log_type(encoder, t, false, true, "size=%u", t->size); |
| } |
| |
| return id; |
| } |
| |
| #if LIBBPF_MAJOR_VERSION < 1 |
| static inline int libbpf_err(int ret) |
| { |
| if (ret < 0) |
| errno = -ret; |
| return ret; |
| } |
| |
| static |
| int btf__add_enum64(struct btf *btf __maybe_unused, const char *name __maybe_unused, |
| __u32 byte_sz __maybe_unused, bool is_signed __maybe_unused) |
| { |
| return libbpf_err(-ENOTSUP); |
| } |
| |
| static |
| int btf__add_enum64_value(struct btf *btf __maybe_unused, const char *name __maybe_unused, |
| __u64 value __maybe_unused) |
| { |
| return libbpf_err(-ENOTSUP); |
| } |
| #endif |
| |
| static int32_t btf_encoder__add_enum(struct btf_encoder *encoder, const char *name, struct type *etype, |
| struct conf_load *conf_load) |
| { |
| struct btf *btf = encoder->btf; |
| const struct btf_type *t; |
| int32_t id, size; |
| bool is_enum32; |
| |
| size = BITS_ROUNDUP_BYTES(etype->size); |
| is_enum32 = size <= 4 || conf_load->skip_encoding_btf_enum64; |
| if (is_enum32) |
| id = btf__add_enum(btf, name, size); |
| else |
| id = btf__add_enum64(btf, name, size, etype->is_signed_enum); |
| if (id > 0) { |
| t = btf__type_by_id(btf, id); |
| btf_encoder__log_type(encoder, t, false, true, "size=%u", t->size); |
| } else { |
| btf__log_err(btf, is_enum32 ? BTF_KIND_ENUM : BTF_KIND_ENUM64, name, true, |
| "size=%u Error emitting BTF type", size); |
| } |
| return id; |
| } |
| |
| static int btf_encoder__add_enum_val(struct btf_encoder *encoder, const char *name, int64_t value, |
| struct type *etype, struct conf_load *conf_load) |
| { |
| const char *fmt_str; |
| int err; |
| |
| /* If enum64 is not allowed, generate enum32 with unsigned int value. In enum64-supported |
| * libbpf library, btf__add_enum_value() will set the kflag (sign bit) in common_type |
| * if the value is negative. |
| */ |
| if (conf_load->skip_encoding_btf_enum64) |
| err = btf__add_enum_value(encoder->btf, name, (uint32_t)value); |
| else if (etype->size > 32) |
| err = btf__add_enum64_value(encoder->btf, name, value); |
| else |
| err = btf__add_enum_value(encoder->btf, name, value); |
| |
| if (!err) { |
| if (encoder->verbose) { |
| if (conf_load->skip_encoding_btf_enum64) { |
| printf("\t%s val=%u\n", name, (uint32_t)value); |
| } else { |
| fmt_str = etype->is_signed_enum ? "\t%s val=%lld\n" : "\t%s val=%llu\n"; |
| printf(fmt_str, name, (unsigned long long)value); |
| } |
| } |
| } else { |
| if (conf_load->skip_encoding_btf_enum64) { |
| fprintf(stderr, "\t%s val=%u Error emitting BTF enum value\n", name, (uint32_t)value); |
| } else { |
| fmt_str = etype->is_signed_enum ? "\t%s val=%lld Error emitting BTF enum value\n" |
| : "\t%s val=%llu Error emitting BTF enum value\n"; |
| fprintf(stderr, fmt_str, name, (unsigned long long)value); |
| } |
| } |
| return err; |
| } |
| |
| static int32_t btf_encoder__add_func_param(struct btf_encoder *encoder, const char *name, uint32_t type, bool is_last_param) |
| { |
| int err = btf__add_func_param(encoder->btf, name, type); |
| |
| if (!err) { |
| btf_encoder__log_func_param(encoder, name, type, false, is_last_param, NULL); |
| return 0; |
| } else { |
| btf_encoder__log_func_param(encoder, name, type, true, is_last_param, "Error adding func param"); |
| return -1; |
| } |
| } |
| |
| static int32_t btf_encoder__tag_type(struct btf_encoder *encoder, uint32_t tag_type) |
| { |
| if (tag_type == 0) |
| return 0; |
| |
| return encoder->type_id_off + tag_type; |
| } |
| |
| static int32_t btf_encoder__add_func_proto(struct btf_encoder *encoder, struct ftype *ftype) |
| { |
| struct btf *btf = encoder->btf; |
| const struct btf_type *t; |
| struct parameter *param; |
| uint16_t nr_params, param_idx; |
| int32_t id, type_id; |
| |
| /* add btf_type for func_proto */ |
| nr_params = ftype->nr_parms + (ftype->unspec_parms ? 1 : 0); |
| type_id = btf_encoder__tag_type(encoder, ftype->tag.type); |
| |
| id = btf__add_func_proto(btf, type_id); |
| if (id > 0) { |
| t = btf__type_by_id(btf, id); |
| btf_encoder__log_type(encoder, t, false, false, "return=%u args=(%s", t->type, !nr_params ? "void)\n" : ""); |
| } else { |
| btf__log_err(btf, BTF_KIND_FUNC_PROTO, NULL, true, |
| "return=%u vlen=%u Error emitting BTF type", |
| type_id, nr_params); |
| return id; |
| } |
| |
| /* add parameters */ |
| param_idx = 0; |
| ftype__for_each_parameter(ftype, param) { |
| const char *name = parameter__name(param); |
| |
| type_id = param->tag.type == 0 ? 0 : encoder->type_id_off + param->tag.type; |
| ++param_idx; |
| if (btf_encoder__add_func_param(encoder, name, type_id, param_idx == nr_params)) |
| return -1; |
| } |
| |
| ++param_idx; |
| if (ftype->unspec_parms) |
| if (btf_encoder__add_func_param(encoder, NULL, 0, param_idx == nr_params)) |
| return -1; |
| |
| return id; |
| } |
| |
| static int32_t btf_encoder__add_var(struct btf_encoder *encoder, uint32_t type, const char *name, uint32_t linkage) |
| { |
| struct btf *btf = encoder->btf; |
| const struct btf_type *t; |
| int32_t id; |
| |
| id = btf__add_var(btf, name, linkage, type); |
| if (id > 0) { |
| t = btf__type_by_id(btf, id); |
| btf_encoder__log_type(encoder, t, false, true, "type=%u linkage=%u", t->type, btf_var(t)->linkage); |
| } else { |
| btf__log_err(btf, BTF_KIND_VAR, name, true, |
| "type=%u linkage=%u Error emitting BTF type", |
| type, linkage); |
| } |
| return id; |
| } |
| |
| static int32_t btf_encoder__add_var_secinfo(struct btf_encoder *encoder, uint32_t type, |
| uint32_t offset, uint32_t size) |
| { |
| struct btf_var_secinfo si = { |
| .type = type, |
| .offset = offset, |
| .size = size, |
| }; |
| return gobuffer__add(&encoder->percpu_secinfo, &si, sizeof(si)); |
| } |
| |
| int32_t btf_encoder__add_encoder(struct btf_encoder *encoder, struct btf_encoder *other) |
| { |
| struct gobuffer *var_secinfo_buf = &other->percpu_secinfo; |
| size_t sz = gobuffer__size(var_secinfo_buf); |
| uint16_t nr_var_secinfo = sz / sizeof(struct btf_var_secinfo); |
| uint32_t type_id; |
| uint32_t next_type_id = btf__type_cnt(encoder->btf); |
| int32_t i, id; |
| struct btf_var_secinfo *vsi; |
| |
| if (encoder == other) |
| return 0; |
| |
| btf_encoder__add_saved_funcs(other); |
| |
| for (i = 0; i < nr_var_secinfo; i++) { |
| vsi = (struct btf_var_secinfo *)var_secinfo_buf->entries + i; |
| type_id = next_type_id + vsi->type - 1; /* Type ID starts from 1 */ |
| id = btf_encoder__add_var_secinfo(encoder, type_id, vsi->offset, vsi->size); |
| if (id < 0) |
| return id; |
| } |
| |
| return btf__add_btf(encoder->btf, other->btf); |
| } |
| |
| static int32_t btf_encoder__add_datasec(struct btf_encoder *encoder, const char *section_name) |
| { |
| struct gobuffer *var_secinfo_buf = &encoder->percpu_secinfo; |
| struct btf *btf = encoder->btf; |
| size_t sz = gobuffer__size(var_secinfo_buf); |
| uint16_t nr_var_secinfo = sz / sizeof(struct btf_var_secinfo); |
| struct btf_var_secinfo *last_vsi, *vsi; |
| const struct btf_type *t; |
| uint32_t datasec_sz; |
| int32_t err, id, i; |
| |
| qsort(var_secinfo_buf->entries, nr_var_secinfo, |
| sizeof(struct btf_var_secinfo), btf_var_secinfo_cmp); |
| |
| last_vsi = (struct btf_var_secinfo *)var_secinfo_buf->entries + nr_var_secinfo - 1; |
| datasec_sz = last_vsi->offset + last_vsi->size; |
| |
| id = btf__add_datasec(btf, section_name, datasec_sz); |
| if (id < 0) { |
| btf__log_err(btf, BTF_KIND_DATASEC, section_name, true, |
| "size=%u vlen=%u Error emitting BTF type", |
| datasec_sz, nr_var_secinfo); |
| } else { |
| t = btf__type_by_id(btf, id); |
| btf_encoder__log_type(encoder, t, false, true, "size=%u vlen=%u", t->size, nr_var_secinfo); |
| } |
| |
| for (i = 0; i < nr_var_secinfo; i++) { |
| vsi = (struct btf_var_secinfo *)var_secinfo_buf->entries + i; |
| err = btf__add_datasec_var_info(btf, vsi->type, vsi->offset, vsi->size); |
| if (!err) { |
| if (encoder->verbose) |
| printf("\ttype=%u offset=%u size=%u\n", |
| vsi->type, vsi->offset, vsi->size); |
| } else { |
| fprintf(stderr, "\ttype=%u offset=%u size=%u Error emitting BTF datasec var info\n", |
| vsi->type, vsi->offset, vsi->size); |
| return -1; |
| } |
| } |
| |
| return id; |
| } |
| |
| static int32_t btf_encoder__add_decl_tag(struct btf_encoder *encoder, const char *value, uint32_t type, |
| int component_idx) |
| { |
| struct btf *btf = encoder->btf; |
| const struct btf_type *t; |
| int32_t id; |
| |
| id = btf__add_decl_tag(btf, value, type, component_idx); |
| if (id > 0) { |
| t = btf__type_by_id(btf, id); |
| btf_encoder__log_type(encoder, t, false, true, "type_id=%u component_idx=%d", |
| t->type, component_idx); |
| } else { |
| btf__log_err(btf, BTF_KIND_DECL_TAG, value, true, "component_idx=%d Error emitting BTF type", |
| component_idx); |
| } |
| |
| return id; |
| } |
| |
| static bool proto__get(struct function *func, char *proto, size_t len) |
| { |
| const struct conf_fprintf conf = { |
| .name_spacing = 23, |
| .type_spacing = 26, |
| .emit_stats = 0, |
| .no_parm_names = 1, |
| .skip_emitting_errors = 1, |
| .skip_emitting_modifier = 1, |
| }; |
| |
| return function__prototype_conf(func, func->priv, &conf, proto, len) != NULL; |
| } |
| |
| static bool funcs__match(struct btf_encoder *encoder, struct elf_function *func, struct function *f2) |
| { |
| char proto[BTF_ENCODER_MAX_PROTO]; |
| struct function *f1 = func->function; |
| const char *name; |
| |
| if (!f1) |
| return false; |
| |
| name = function__name(f1); |
| |
| if (f1->proto.nr_parms != f2->proto.nr_parms) { |
| if (encoder->verbose) |
| printf("function mismatch for '%s'(%s): %d params != %d params\n", |
| name, f1->alias ?: name, |
| f1->proto.nr_parms, f2->proto.nr_parms); |
| return false; |
| } |
| if (f1->proto.nr_parms == 0) |
| return true; |
| |
| if (f1->proto.tag.type == f2->proto.tag.type) |
| return true; |
| |
| if (!func->state.got_proto) |
| func->state.got_proto = proto__get(f1, func->state.proto, sizeof(func->state.proto)); |
| |
| if (proto__get(f2, proto, sizeof(proto))) { |
| if (strcmp(func->state.proto, proto) != 0) { |
| if (encoder->verbose) |
| printf("function mismatch for '%s'('%s'): '%s' != '%s'\n", |
| name, f1->alias ?: name, |
| func->state.proto, proto); |
| return false; |
| } |
| } |
| return true; |
| } |
| |
| static int32_t btf_encoder__save_func(struct btf_encoder *encoder, struct function *fn, struct elf_function *func) |
| { |
| fn->priv = encoder->cu; |
| if (func->function) { |
| struct function *existing = func->function; |
| |
| /* If saving and we find an existing entry, we want to merge |
| * observations across both functions, checking that the |
| * "seen optimized parameters", "inconsistent prototype" |
| * and "unexpected register" status is reflected in the |
| * the func entry. |
| * If the entry is new, record encoder state required |
| * to add the local function later (encoder + type_id_off) |
| * such that we can add the function later. |
| */ |
| existing->proto.optimized_parms |= fn->proto.optimized_parms; |
| existing->proto.unexpected_reg |= fn->proto.unexpected_reg; |
| if (!existing->proto.unexpected_reg && !existing->proto.inconsistent_proto && |
| !funcs__match(encoder, func, fn)) |
| existing->proto.inconsistent_proto = 1; |
| } else { |
| func->state.type_id_off = encoder->type_id_off; |
| func->function = fn; |
| encoder->saved_func_cnt++; |
| } |
| return 0; |
| } |
| |
| static int32_t btf_encoder__add_func(struct btf_encoder *encoder, struct function *fn) |
| { |
| int btf_fnproto_id, btf_fn_id, tag_type_id; |
| struct llvm_annotation *annot; |
| const char *name; |
| |
| btf_fnproto_id = btf_encoder__add_func_proto(encoder, &fn->proto); |
| name = function__name(fn); |
| btf_fn_id = btf_encoder__add_ref_type(encoder, BTF_KIND_FUNC, btf_fnproto_id, name, false); |
| if (btf_fnproto_id < 0 || btf_fn_id < 0) { |
| printf("error: failed to encode function '%s'\n", function__name(fn)); |
| return -1; |
| } |
| list_for_each_entry(annot, &fn->annots, node) { |
| tag_type_id = btf_encoder__add_decl_tag(encoder, annot->value, btf_fn_id, |
| annot->component_idx); |
| if (tag_type_id < 0) { |
| fprintf(stderr, "error: failed to encode tag '%s' to func %s with component_idx %d\n", |
| annot->value, name, annot->component_idx); |
| return -1; |
| } |
| } |
| return 0; |
| } |
| |
| static void btf_encoder__add_saved_funcs(struct btf_encoder *encoder) |
| { |
| int i; |
| |
| for (i = 0; i < encoder->functions.cnt; i++) { |
| struct elf_function *func = &encoder->functions.entries[i]; |
| struct function *fn = func->function; |
| struct btf_encoder *other_encoder; |
| |
| if (!fn || fn->proto.processed) |
| continue; |
| |
| /* merge optimized-out status across encoders; since each |
| * encoder has the same elf symbol table we can use the |
| * same index to access the same elf symbol. |
| */ |
| btf_encoders__for_each_encoder(other_encoder) { |
| struct function *other_fn; |
| |
| if (other_encoder == encoder) |
| continue; |
| |
| other_fn = other_encoder->functions.entries[i].function; |
| if (!other_fn) |
| continue; |
| fn->proto.optimized_parms |= other_fn->proto.optimized_parms; |
| fn->proto.unexpected_reg |= other_fn->proto.unexpected_reg; |
| if (other_fn->proto.inconsistent_proto) |
| fn->proto.inconsistent_proto = 1; |
| if (!fn->proto.unexpected_reg && !fn->proto.inconsistent_proto && |
| !funcs__match(encoder, func, other_fn)) |
| fn->proto.inconsistent_proto = 1; |
| other_fn->proto.processed = 1; |
| } |
| /* do not exclude functions with optimized-out parameters; they |
| * may still be _called_ with the right parameter values, they |
| * just do not _use_ them. Only exclude functions with |
| * unexpected register use or multiple inconsistent prototypes. |
| */ |
| if (fn->proto.unexpected_reg || fn->proto.inconsistent_proto) { |
| if (encoder->verbose) { |
| const char *name = function__name(fn); |
| |
| printf("skipping addition of '%s'(%s) due to %s\n", |
| name, fn->alias ?: name, |
| fn->proto.unexpected_reg ? "unexpected register used for parameter" : |
| "multiple inconsistent function prototypes"); |
| } |
| } else { |
| encoder->type_id_off = func->state.type_id_off; |
| btf_encoder__add_func(encoder, fn); |
| } |
| fn->proto.processed = 1; |
| } |
| } |
| |
| /* |
| * This corresponds to the same macro defined in |
| * include/linux/kallsyms.h |
| */ |
| #define KSYM_NAME_LEN 128 |
| |
| static int functions_cmp(const void *_a, const void *_b) |
| { |
| const struct elf_function *a = _a; |
| const struct elf_function *b = _b; |
| |
| /* if search key allows prefix match, verify target has matching |
| * prefix len and prefix matches. |
| */ |
| if (a->prefixlen && a->prefixlen == b->prefixlen) |
| return strncmp(a->name, b->name, b->prefixlen); |
| return strcmp(a->name, b->name); |
| } |
| |
| #ifndef max |
| #define max(x, y) ((x) < (y) ? (y) : (x)) |
| #endif |
| |
| static int btf_encoder__collect_function(struct btf_encoder *encoder, GElf_Sym *sym) |
| { |
| struct elf_function *new; |
| const char *name; |
| |
| if (elf_sym__type(sym) != STT_FUNC) |
| return 0; |
| name = elf_sym__name(sym, encoder->symtab); |
| if (!name) |
| return 0; |
| |
| if (encoder->functions.cnt == encoder->functions.allocated) { |
| encoder->functions.allocated = max(1000, encoder->functions.allocated * 3 / 2); |
| new = realloc(encoder->functions.entries, encoder->functions.allocated * sizeof(*encoder->functions.entries)); |
| if (!new) { |
| /* |
| * The cleanup - delete_functions is called |
| * in btf_encoder__encode_cu error path. |
| */ |
| return -1; |
| } |
| encoder->functions.entries = new; |
| } |
| |
| encoder->functions.entries[encoder->functions.cnt].name = name; |
| if (strchr(name, '.')) { |
| const char *suffix = strchr(name, '.'); |
| |
| encoder->functions.suffix_cnt++; |
| encoder->functions.entries[encoder->functions.cnt].prefixlen = suffix - name; |
| } |
| encoder->functions.entries[encoder->functions.cnt].generated = false; |
| encoder->functions.entries[encoder->functions.cnt].function = NULL; |
| encoder->functions.entries[encoder->functions.cnt].state.got_proto = false; |
| encoder->functions.entries[encoder->functions.cnt].state.proto[0] = '\0'; |
| encoder->functions.entries[encoder->functions.cnt].state.type_id_off = 0; |
| encoder->functions.cnt++; |
| return 0; |
| } |
| |
| static struct elf_function *btf_encoder__find_function(const struct btf_encoder *encoder, |
| const char *name, size_t prefixlen) |
| { |
| struct elf_function key = { .name = name, .prefixlen = prefixlen }; |
| |
| return bsearch(&key, encoder->functions.entries, encoder->functions.cnt, sizeof(key), functions_cmp); |
| } |
| |
| static bool btf_name_char_ok(char c, bool first) |
| { |
| if (c == '_' || c == '.') |
| return true; |
| |
| return first ? isalpha(c) : isalnum(c); |
| } |
| |
| /* Check whether the given name is valid in vmlinux btf. */ |
| static bool btf_name_valid(const char *p) |
| { |
| const char *limit; |
| |
| if (!btf_name_char_ok(*p, true)) |
| return false; |
| |
| /* set a limit on identifier length */ |
| limit = p + KSYM_NAME_LEN; |
| p++; |
| while (*p && p < limit) { |
| if (!btf_name_char_ok(*p, false)) |
| return false; |
| p++; |
| } |
| |
| return !*p; |
| } |
| |
| static void dump_invalid_symbol(const char *msg, const char *sym, |
| int verbose, bool force) |
| { |
| if (force) { |
| if (verbose) |
| fprintf(stderr, "PAHOLE: Warning: %s, ignored (sym: '%s').\n", |
| msg, sym); |
| return; |
| } |
| |
| fprintf(stderr, "PAHOLE: Error: %s (sym: '%s').\n", msg, sym); |
| fprintf(stderr, "PAHOLE: Error: Use '--btf_encode_force' to ignore such symbols and force emit the btf.\n"); |
| } |
| |
| static int tag__check_id_drift(struct btf_encoder *encoder, const struct tag *tag, |
| uint32_t core_id, uint32_t btf_type_id) |
| { |
| if (btf_type_id != (core_id + encoder->type_id_off)) { |
| fprintf(stderr, |
| "%s: %s id drift, core_id: %u, btf_type_id: %u, type_id_off: %u\n", |
| __func__, dwarf_tag_name(tag->tag), |
| core_id, btf_type_id, encoder->type_id_off); |
| return -1; |
| } |
| |
| return 0; |
| } |
| |
| static int32_t btf_encoder__add_struct_type(struct btf_encoder *encoder, struct tag *tag) |
| { |
| struct type *type = tag__type(tag); |
| struct class_member *pos; |
| const char *name = type__name(type); |
| int32_t type_id; |
| uint8_t kind; |
| |
| kind = (tag->tag == DW_TAG_union_type) ? |
| BTF_KIND_UNION : BTF_KIND_STRUCT; |
| |
| type_id = btf_encoder__add_struct(encoder, kind, name, type->size); |
| if (type_id < 0) |
| return type_id; |
| |
| type__for_each_data_member(type, pos) { |
| /* |
| * dwarf_loader uses DWARF's recommended bit offset addressing |
| * scheme, which conforms to BTF requirement, so no conversion |
| * is required. |
| */ |
| name = class_member__name(pos); |
| if (btf_encoder__add_field(encoder, name, encoder->type_id_off + pos->tag.type, |
| pos->bitfield_size, pos->bit_offset)) |
| return -1; |
| } |
| |
| return type_id; |
| } |
| |
| static uint32_t array_type__nelems(struct tag *tag) |
| { |
| int i; |
| uint32_t nelem = 1; |
| struct array_type *array = tag__array_type(tag); |
| |
| for (i = array->dimensions - 1; i >= 0; --i) |
| nelem *= array->nr_entries[i]; |
| |
| return nelem; |
| } |
| |
| static int32_t btf_encoder__add_enum_type(struct btf_encoder *encoder, struct tag *tag, |
| struct conf_load *conf_load) |
| { |
| struct type *etype = tag__type(tag); |
| struct enumerator *pos; |
| const char *name = type__name(etype); |
| int32_t type_id; |
| |
| type_id = btf_encoder__add_enum(encoder, name, etype, conf_load); |
| if (type_id < 0) |
| return type_id; |
| |
| type__for_each_enumerator(etype, pos) { |
| name = enumerator__name(pos); |
| if (btf_encoder__add_enum_val(encoder, name, pos->value, etype, conf_load)) |
| return -1; |
| } |
| |
| return type_id; |
| } |
| |
| static int btf_encoder__encode_tag(struct btf_encoder *encoder, struct tag *tag, |
| struct conf_load *conf_load) |
| { |
| /* single out type 0 as it represents special type "void" */ |
| uint32_t ref_type_id = tag->type == 0 ? 0 : encoder->type_id_off + tag->type; |
| struct base_type *bt; |
| const char *name; |
| |
| switch (tag->tag) { |
| case DW_TAG_base_type: |
| bt = tag__base_type(tag); |
| name = __base_type__name(bt); |
| return btf_encoder__add_base_type(encoder, bt, name); |
| case DW_TAG_const_type: |
| return btf_encoder__add_ref_type(encoder, BTF_KIND_CONST, ref_type_id, NULL, false); |
| case DW_TAG_pointer_type: |
| return btf_encoder__add_ref_type(encoder, BTF_KIND_PTR, ref_type_id, NULL, false); |
| case DW_TAG_restrict_type: |
| return btf_encoder__add_ref_type(encoder, BTF_KIND_RESTRICT, ref_type_id, NULL, false); |
| case DW_TAG_volatile_type: |
| return btf_encoder__add_ref_type(encoder, BTF_KIND_VOLATILE, ref_type_id, NULL, false); |
| case DW_TAG_typedef: |
| name = namespace__name(tag__namespace(tag)); |
| return btf_encoder__add_ref_type(encoder, BTF_KIND_TYPEDEF, ref_type_id, name, false); |
| case DW_TAG_LLVM_annotation: |
| name = tag__btf_type_tag(tag)->value; |
| return btf_encoder__add_ref_type(encoder, BTF_KIND_TYPE_TAG, ref_type_id, name, false); |
| case DW_TAG_structure_type: |
| case DW_TAG_union_type: |
| case DW_TAG_class_type: |
| name = namespace__name(tag__namespace(tag)); |
| if (tag__type(tag)->declaration) |
| return btf_encoder__add_ref_type(encoder, BTF_KIND_FWD, 0, name, tag->tag == DW_TAG_union_type); |
| else |
| return btf_encoder__add_struct_type(encoder, tag); |
| case DW_TAG_array_type: |
| /* TODO: Encode one dimension at a time. */ |
| encoder->need_index_type = true; |
| return btf_encoder__add_array(encoder, ref_type_id, encoder->array_index_id, array_type__nelems(tag)); |
| case DW_TAG_enumeration_type: |
| return btf_encoder__add_enum_type(encoder, tag, conf_load); |
| case DW_TAG_subroutine_type: |
| return btf_encoder__add_func_proto(encoder, tag__ftype(tag)); |
| case DW_TAG_unspecified_type: |
| /* Just don't encode this for now, converting anything with this type to void (0) instead. |
| * |
| * If we end up needing to encode this, one possible hack is to do as follows, as "const void". |
| * |
| * Returning zero means we skipped encoding a DWARF type. |
| */ |
| // btf_encoder__add_ref_type(encoder, BTF_KIND_CONST, 0, NULL, false); |
| return 0; |
| default: |
| fprintf(stderr, "Unsupported DW_TAG_%s(0x%x): type: 0x%x\n", |
| dwarf_tag_name(tag->tag), tag->tag, ref_type_id); |
| return -1; |
| } |
| } |
| |
| static int btf_encoder__write_raw_file(struct btf_encoder *encoder) |
| { |
| const char *filename = encoder->filename; |
| uint32_t raw_btf_size; |
| const void *raw_btf_data; |
| int fd, err; |
| |
| raw_btf_data = btf__raw_data(encoder->btf, &raw_btf_size); |
| if (raw_btf_data == NULL) { |
| fprintf(stderr, "%s: btf__raw_data failed!\n", __func__); |
| return -1; |
| } |
| |
| fd = open(filename, O_WRONLY | O_CREAT, 0640); |
| if (fd < 0) { |
| fprintf(stderr, "%s: Couldn't open %s for writing the raw BTF info: %s\n", __func__, filename, strerror(errno)); |
| return -1; |
| } |
| err = write(fd, raw_btf_data, raw_btf_size); |
| if (err < 0) |
| fprintf(stderr, "%s: Couldn't write the raw BTF info to %s: %s\n", __func__, filename, strerror(errno)); |
| |
| close(fd); |
| |
| if ((uint32_t)err != raw_btf_size) { |
| fprintf(stderr, "%s: Could only write %d bytes to %s of raw BTF info out of %d, aborting\n", __func__, err, filename, raw_btf_size); |
| unlink(filename); |
| err = -1; |
| } else { |
| /* go from bytes written == raw_btf_size to an indication that all went fine */ |
| err = 0; |
| } |
| |
| return err; |
| } |
| |
| static int btf_encoder__write_elf(struct btf_encoder *encoder) |
| { |
| struct btf *btf = encoder->btf; |
| const char *filename = encoder->filename; |
| GElf_Shdr shdr_mem, *shdr; |
| Elf_Data *btf_data = NULL; |
| Elf_Scn *scn = NULL; |
| Elf *elf = NULL; |
| const void *raw_btf_data; |
| uint32_t raw_btf_size; |
| int fd, err = -1; |
| size_t strndx; |
| |
| fd = open(filename, O_RDWR); |
| if (fd < 0) { |
| fprintf(stderr, "Cannot open %s\n", filename); |
| return -1; |
| } |
| |
| if (elf_version(EV_CURRENT) == EV_NONE) { |
| elf_error("Cannot set libelf version"); |
| goto out; |
| } |
| |
| elf = elf_begin(fd, ELF_C_RDWR, NULL); |
| if (elf == NULL) { |
| elf_error("Cannot update ELF file"); |
| goto out; |
| } |
| |
| elf_flagelf(elf, ELF_C_SET, ELF_F_DIRTY); |
| |
| /* |
| * First we look if there was already a .BTF section to overwrite. |
| */ |
| |
| elf_getshdrstrndx(elf, &strndx); |
| while ((scn = elf_nextscn(elf, scn)) != NULL) { |
| shdr = gelf_getshdr(scn, &shdr_mem); |
| if (shdr == NULL) |
| continue; |
| char *secname = elf_strptr(elf, strndx, shdr->sh_name); |
| if (strcmp(secname, ".BTF") == 0) { |
| btf_data = elf_getdata(scn, btf_data); |
| break; |
| } |
| } |
| |
| raw_btf_data = btf__raw_data(btf, &raw_btf_size); |
| |
| if (btf_data) { |
| /* Existing .BTF section found */ |
| btf_data->d_buf = (void *)raw_btf_data; |
| btf_data->d_size = raw_btf_size; |
| elf_flagdata(btf_data, ELF_C_SET, ELF_F_DIRTY); |
| |
| if (elf_update(elf, ELF_C_NULL) >= 0 && |
| elf_update(elf, ELF_C_WRITE) >= 0) |
| err = 0; |
| else |
| elf_error("elf_update failed"); |
| } else { |
| const char *llvm_objcopy; |
| char tmp_fn[PATH_MAX]; |
| char cmd[PATH_MAX * 2]; |
| |
| llvm_objcopy = getenv("LLVM_OBJCOPY"); |
| if (!llvm_objcopy) |
| llvm_objcopy = "llvm-objcopy"; |
| |
| /* Use objcopy to add a .BTF section */ |
| snprintf(tmp_fn, sizeof(tmp_fn), "%s.btf", filename); |
| close(fd); |
| fd = creat(tmp_fn, S_IRUSR | S_IWUSR); |
| if (fd == -1) { |
| fprintf(stderr, "%s: open(%s) failed!\n", __func__, |
| tmp_fn); |
| goto out; |
| } |
| |
| if (write(fd, raw_btf_data, raw_btf_size) != raw_btf_size) { |
| fprintf(stderr, "%s: write of %d bytes to '%s' failed: %d!\n", |
| __func__, raw_btf_size, tmp_fn, errno); |
| goto unlink; |
| } |
| |
| snprintf(cmd, sizeof(cmd), "%s --add-section .BTF=%s %s", |
| llvm_objcopy, tmp_fn, filename); |
| if (system(cmd)) { |
| fprintf(stderr, "%s: failed to add .BTF section to '%s': %d!\n", |
| __func__, filename, errno); |
| goto unlink; |
| } |
| |
| err = 0; |
| unlink: |
| unlink(tmp_fn); |
| } |
| |
| out: |
| if (fd != -1) |
| close(fd); |
| if (elf) |
| elf_end(elf); |
| return err; |
| } |
| |
| int btf_encoder__encode(struct btf_encoder *encoder) |
| { |
| int err; |
| |
| /* for single-threaded case, saved funcs are added here */ |
| btf_encoder__add_saved_funcs(encoder); |
| |
| if (gobuffer__size(&encoder->percpu_secinfo) != 0) |
| btf_encoder__add_datasec(encoder, PERCPU_SECTION); |
| |
| /* Empty file, nothing to do, so... done! */ |
| if (btf__type_cnt(encoder->btf) == 1) |
| return 0; |
| |
| if (btf__dedup(encoder->btf, NULL)) { |
| fprintf(stderr, "%s: btf__dedup failed!\n", __func__); |
| return -1; |
| } |
| |
| if (encoder->raw_output) |
| err = btf_encoder__write_raw_file(encoder); |
| else |
| err = btf_encoder__write_elf(encoder); |
| |
| return err; |
| } |
| |
| static int percpu_var_cmp(const void *_a, const void *_b) |
| { |
| const struct var_info *a = _a; |
| const struct var_info *b = _b; |
| |
| if (a->addr == b->addr) |
| return 0; |
| return a->addr < b->addr ? -1 : 1; |
| } |
| |
| static bool btf_encoder__percpu_var_exists(struct btf_encoder *encoder, uint64_t addr, uint32_t *sz, const char **name) |
| { |
| struct var_info key = { .addr = addr }; |
| const struct var_info *p = bsearch(&key, encoder->percpu.vars, encoder->percpu.var_cnt, |
| sizeof(encoder->percpu.vars[0]), percpu_var_cmp); |
| if (!p) |
| return false; |
| |
| *sz = p->sz; |
| *name = p->name; |
| return true; |
| } |
| |
| static int btf_encoder__collect_percpu_var(struct btf_encoder *encoder, GElf_Sym *sym, size_t sym_sec_idx) |
| { |
| const char *sym_name; |
| uint64_t addr; |
| uint32_t size; |
| |
| /* compare a symbol's shndx to determine if it's a percpu variable */ |
| if (sym_sec_idx != encoder->percpu.shndx) |
| return 0; |
| if (elf_sym__type(sym) != STT_OBJECT) |
| return 0; |
| |
| addr = elf_sym__value(sym); |
| |
| size = elf_sym__size(sym); |
| if (!size) |
| return 0; /* ignore zero-sized symbols */ |
| |
| sym_name = elf_sym__name(sym, encoder->symtab); |
| if (!btf_name_valid(sym_name)) { |
| dump_invalid_symbol("Found symbol of invalid name when encoding btf", |
| sym_name, encoder->verbose, encoder->force); |
| if (encoder->force) |
| return 0; |
| return -1; |
| } |
| |
| if (encoder->verbose) |
| printf("Found per-CPU symbol '%s' at address 0x%" PRIx64 "\n", sym_name, addr); |
| |
| /* Make sure addr is section-relative. For kernel modules (which are |
| * ET_REL files) this is already the case. For vmlinux (which is an |
| * ET_EXEC file) we need to subtract the section address. |
| */ |
| if (!encoder->is_rel) |
| addr -= encoder->percpu.base_addr; |
| |
| if (encoder->percpu.var_cnt == MAX_PERCPU_VAR_CNT) { |
| fprintf(stderr, "Reached the limit of per-CPU variables: %d\n", |
| MAX_PERCPU_VAR_CNT); |
| return -1; |
| } |
| encoder->percpu.vars[encoder->percpu.var_cnt].addr = addr; |
| encoder->percpu.vars[encoder->percpu.var_cnt].sz = size; |
| encoder->percpu.vars[encoder->percpu.var_cnt].name = sym_name; |
| encoder->percpu.var_cnt++; |
| |
| return 0; |
| } |
| |
| static int btf_encoder__collect_symbols(struct btf_encoder *encoder, bool collect_percpu_vars) |
| { |
| Elf32_Word sym_sec_idx; |
| uint32_t core_id; |
| GElf_Sym sym; |
| |
| /* cache variables' addresses, preparing for searching in symtab. */ |
| encoder->percpu.var_cnt = 0; |
| |
| /* search within symtab for percpu variables */ |
| elf_symtab__for_each_symbol_index(encoder->symtab, core_id, sym, sym_sec_idx) { |
| if (collect_percpu_vars && btf_encoder__collect_percpu_var(encoder, &sym, sym_sec_idx)) |
| return -1; |
| if (btf_encoder__collect_function(encoder, &sym)) |
| return -1; |
| } |
| |
| if (collect_percpu_vars) { |
| if (encoder->percpu.var_cnt) |
| qsort(encoder->percpu.vars, encoder->percpu.var_cnt, sizeof(encoder->percpu.vars[0]), percpu_var_cmp); |
| |
| if (encoder->verbose) |
| printf("Found %d per-CPU variables!\n", encoder->percpu.var_cnt); |
| } |
| |
| if (encoder->functions.cnt) { |
| qsort(encoder->functions.entries, encoder->functions.cnt, sizeof(encoder->functions.entries[0]), |
| functions_cmp); |
| if (encoder->verbose) |
| printf("Found %d functions!\n", encoder->functions.cnt); |
| } |
| |
| return 0; |
| } |
| |
| static bool ftype__has_arg_names(const struct ftype *ftype) |
| { |
| struct parameter *param; |
| |
| ftype__for_each_parameter(ftype, param) { |
| if (parameter__name(param) == NULL) |
| return false; |
| } |
| return true; |
| } |
| |
| static int btf_encoder__encode_cu_variables(struct btf_encoder *encoder) |
| { |
| struct cu *cu = encoder->cu; |
| uint32_t core_id; |
| struct tag *pos; |
| int err = -1; |
| |
| if (encoder->percpu.shndx == 0 || !encoder->symtab) |
| return 0; |
| |
| if (encoder->verbose) |
| printf("search cu '%s' for percpu global variables.\n", cu->name); |
| |
| cu__for_each_variable(cu, core_id, pos) { |
| struct variable *var = tag__variable(pos); |
| uint32_t size, type, linkage; |
| const char *name, *dwarf_name; |
| struct llvm_annotation *annot; |
| const struct tag *tag; |
| uint64_t addr; |
| int id; |
| |
| if (var->declaration && !var->spec) |
| continue; |
| |
| /* percpu variables are allocated in global space */ |
| if (variable__scope(var) != VSCOPE_GLOBAL && !var->spec) |
| continue; |
| |
| /* addr has to be recorded before we follow spec */ |
| addr = var->ip.addr; |
| dwarf_name = variable__name(var); |
| |
| /* Make sure addr is section-relative. DWARF, unlike ELF, |
| * always contains virtual symbol addresses, so subtract |
| * the section address unconditionally. |
| */ |
| if (addr < encoder->percpu.base_addr || addr >= encoder->percpu.base_addr + encoder->percpu.sec_sz) |
| continue; |
| addr -= encoder->percpu.base_addr; |
| |
| if (!btf_encoder__percpu_var_exists(encoder, addr, &size, &name)) |
| continue; /* not a per-CPU variable */ |
| |
| /* A lot of "special" DWARF variables (e.g, __UNIQUE_ID___xxx) |
| * have addr == 0, which is the same as, say, valid |
| * fixed_percpu_data per-CPU variable. To distinguish between |
| * them, additionally compare DWARF and ELF symbol names. If |
| * DWARF doesn't provide proper name, pessimistically assume |
| * bad variable. |
| * |
| * Examples of such special variables are: |
| * |
| * 1. __ADDRESSABLE(sym), which are forcely emitted as symbols. |
| * 2. __UNIQUE_ID(prefix), which are introduced to generate unique ids. |
| * 3. __exitcall(fn), functions which are labeled as exit calls. |
| * |
| * This is relevant only for vmlinux image, as for kernel |
| * modules per-CPU data section has non-zero offset so all |
| * per-CPU symbols have non-zero values. |
| */ |
| if (var->ip.addr == 0) { |
| if (!dwarf_name || strcmp(dwarf_name, name)) |
| continue; |
| } |
| |
| if (var->spec) |
| var = var->spec; |
| |
| if (var->ip.tag.type == 0) { |
| fprintf(stderr, "error: found variable '%s' in CU '%s' that has void type\n", |
| name, cu->name); |
| if (encoder->force) |
| continue; |
| err = -1; |
| break; |
| } |
| |
| tag = cu__type(cu, var->ip.tag.type); |
| if (tag__size(tag, cu) == 0) { |
| if (encoder->verbose) |
| fprintf(stderr, "Ignoring zero-sized per-CPU variable '%s'...\n", dwarf_name ?: "<missing name>"); |
| continue; |
| } |
| |
| type = var->ip.tag.type + encoder->type_id_off; |
| linkage = var->external ? BTF_VAR_GLOBAL_ALLOCATED : BTF_VAR_STATIC; |
| |
| if (encoder->verbose) { |
| printf("Variable '%s' from CU '%s' at address 0x%" PRIx64 " encoded\n", |
| name, cu->name, addr); |
| } |
| |
| /* add a BTF_KIND_VAR in encoder->types */ |
| id = btf_encoder__add_var(encoder, type, name, linkage); |
| if (id < 0) { |
| fprintf(stderr, "error: failed to encode variable '%s' at addr 0x%" PRIx64 "\n", |
| name, addr); |
| goto out; |
| } |
| |
| list_for_each_entry(annot, &var->annots, node) { |
| int tag_type_id = btf_encoder__add_decl_tag(encoder, annot->value, id, annot->component_idx); |
| if (tag_type_id < 0) { |
| fprintf(stderr, "error: failed to encode tag '%s' to variable '%s' with component_idx %d\n", |
| annot->value, name, annot->component_idx); |
| goto out; |
| } |
| } |
| |
| /* |
| * add a BTF_VAR_SECINFO in encoder->percpu_secinfo, which will be added into |
| * encoder->types later when we add BTF_VAR_DATASEC. |
| */ |
| id = btf_encoder__add_var_secinfo(encoder, id, addr, size); |
| if (id < 0) { |
| fprintf(stderr, "error: failed to encode section info for variable '%s' at addr 0x%" PRIx64 "\n", |
| name, addr); |
| goto out; |
| } |
| } |
| |
| err = 0; |
| out: |
| return err; |
| } |
| |
| struct btf_encoder *btf_encoder__new(struct cu *cu, const char *detached_filename, struct btf *base_btf, bool skip_encoding_vars, bool force, bool gen_floats, bool verbose) |
| { |
| struct btf_encoder *encoder = zalloc(sizeof(*encoder)); |
| |
| if (encoder) { |
| encoder->raw_output = detached_filename != NULL; |
| encoder->filename = strdup(encoder->raw_output ? detached_filename : cu->filename); |
| if (encoder->filename == NULL) |
| goto out_delete; |
| |
| encoder->btf = btf__new_empty_split(base_btf); |
| if (encoder->btf == NULL) |
| goto out_delete; |
| |
| encoder->force = force; |
| encoder->gen_floats = gen_floats; |
| encoder->skip_encoding_vars = skip_encoding_vars; |
| encoder->verbose = verbose; |
| encoder->has_index_type = false; |
| encoder->need_index_type = false; |
| encoder->array_index_id = 0; |
| |
| GElf_Ehdr ehdr; |
| |
| if (gelf_getehdr(cu->elf, &ehdr) == NULL) { |
| if (encoder->verbose) |
| elf_error("cannot get ELF header"); |
| goto out_delete; |
| } |
| |
| encoder->is_rel = ehdr.e_type == ET_REL; |
| |
| switch (ehdr.e_ident[EI_DATA]) { |
| case ELFDATA2LSB: |
| btf__set_endianness(encoder->btf, BTF_LITTLE_ENDIAN); |
| break; |
| case ELFDATA2MSB: |
| btf__set_endianness(encoder->btf, BTF_BIG_ENDIAN); |
| break; |
| default: |
| fprintf(stderr, "%s: unknown ELF endianness.\n", __func__); |
| goto out_delete; |
| } |
| |
| encoder->symtab = elf_symtab__new(NULL, cu->elf); |
| if (!encoder->symtab) { |
| if (encoder->verbose) |
| printf("%s: '%s' doesn't have symtab.\n", __func__, cu->filename); |
| goto out; |
| } |
| |
| /* find percpu section's shndx */ |
| |
| GElf_Shdr shdr; |
| Elf_Scn *sec = elf_section_by_name(cu->elf, &shdr, PERCPU_SECTION, NULL); |
| |
| if (!sec) { |
| if (encoder->verbose) |
| printf("%s: '%s' doesn't have '%s' section\n", __func__, cu->filename, PERCPU_SECTION); |
| } else { |
| encoder->percpu.shndx = elf_ndxscn(sec); |
| encoder->percpu.base_addr = shdr.sh_addr; |
| encoder->percpu.sec_sz = shdr.sh_size; |
| } |
| |
| if (btf_encoder__collect_symbols(encoder, !encoder->skip_encoding_vars)) |
| goto out_delete; |
| |
| if (encoder->verbose) |
| printf("File %s:\n", cu->filename); |
| btf_encoders__add(encoder); |
| } |
| out: |
| return encoder; |
| |
| out_delete: |
| btf_encoder__delete(encoder); |
| return NULL; |
| } |
| |
| void btf_encoder__delete(struct btf_encoder *encoder) |
| { |
| if (encoder == NULL) |
| return; |
| |
| btf_encoders__delete(encoder); |
| __gobuffer__delete(&encoder->percpu_secinfo); |
| zfree(&encoder->filename); |
| btf__free(encoder->btf); |
| encoder->btf = NULL; |
| elf_symtab__delete(encoder->symtab); |
| |
| encoder->functions.allocated = encoder->functions.cnt = 0; |
| free(encoder->functions.entries); |
| encoder->functions.entries = NULL; |
| |
| free(encoder); |
| } |
| |
| int btf_encoder__encode_cu(struct btf_encoder *encoder, struct cu *cu, struct conf_load *conf_load) |
| { |
| struct llvm_annotation *annot; |
| int btf_type_id, tag_type_id, skipped_types = 0; |
| uint32_t core_id; |
| struct function *fn; |
| struct tag *pos; |
| int err = 0; |
| |
| encoder->cu = cu; |
| encoder->type_id_off = btf__type_cnt(encoder->btf) - 1; |
| |
| if (!encoder->has_index_type) { |
| /* cu__find_base_type_by_name() takes "type_id_t *id" */ |
| type_id_t id; |
| if (cu__find_base_type_by_name(cu, "int", &id)) { |
| encoder->has_index_type = true; |
| encoder->array_index_id = encoder->type_id_off + id; |
| } else { |
| encoder->has_index_type = false; |
| encoder->array_index_id = encoder->type_id_off + cu->types_table.nr_entries; |
| } |
| } |
| |
| cu__for_each_type(cu, core_id, pos) { |
| btf_type_id = btf_encoder__encode_tag(encoder, pos, conf_load); |
| |
| if (btf_type_id == 0) { |
| ++skipped_types; |
| continue; |
| } |
| |
| if (btf_type_id < 0 || |
| tag__check_id_drift(encoder, pos, core_id, btf_type_id + skipped_types)) { |
| err = -1; |
| goto out; |
| } |
| } |
| |
| if (encoder->need_index_type && !encoder->has_index_type) { |
| struct base_type bt = {}; |
| |
| bt.name = 0; |
| bt.bit_size = 32; |
| bt.is_signed = true; |
| btf_encoder__add_base_type(encoder, &bt, "int"); |
| encoder->has_index_type = true; |
| } |
| |
| cu__for_each_type(cu, core_id, pos) { |
| struct namespace *ns; |
| const char *tag_name; |
| |
| switch (pos->tag) { |
| case DW_TAG_structure_type: |
| tag_name = "struct"; |
| break; |
| case DW_TAG_union_type: |
| tag_name = "union"; |
| break; |
| case DW_TAG_typedef: |
| tag_name = "typedef"; |
| break; |
| default: |
| continue; |
| } |
| |
| btf_type_id = encoder->type_id_off + core_id; |
| ns = tag__namespace(pos); |
| list_for_each_entry(annot, &ns->annots, node) { |
| tag_type_id = btf_encoder__add_decl_tag(encoder, annot->value, btf_type_id, annot->component_idx); |
| if (tag_type_id < 0) { |
| fprintf(stderr, "error: failed to encode tag '%s' to %s '%s' with component_idx %d\n", |
| annot->value, tag_name, namespace__name(ns), annot->component_idx); |
| goto out; |
| } |
| } |
| } |
| |
| cu__for_each_function(cu, core_id, fn) { |
| struct elf_function *func = NULL; |
| bool save = false; |
| |
| /* |
| * Skip functions that: |
| * - are marked as declarations |
| * - do not have full argument names |
| * - are not in ftrace list (if it's available) |
| * - are not external (in case ftrace filter is not available) |
| */ |
| if (fn->declaration) |
| continue; |
| if (!ftype__has_arg_names(&fn->proto)) |
| continue; |
| if (encoder->functions.cnt) { |
| const char *name; |
| |
| name = function__name(fn); |
| if (!name) |
| continue; |
| |
| /* prefer exact function name match... */ |
| func = btf_encoder__find_function(encoder, name, 0); |
| if (func) { |
| if (func->generated) |
| continue; |
| if (conf_load->skip_encoding_btf_inconsistent_proto) |
| save = true; |
| else |
| func->generated = true; |
| } else if (encoder->functions.suffix_cnt && |
| conf_load->btf_gen_optimized) { |
| /* falling back to name.isra.0 match if no exact |
| * match is found; only bother if we found any |
| * .suffix function names. The function |
| * will be saved and added once we ensure |
| * it does not have optimized-out parameters |
| * in any cu. |
| */ |
| func = btf_encoder__find_function(encoder, name, |
| strlen(name)); |
| if (func) { |
| save = true; |
| if (encoder->verbose) |
| printf("matched function '%s' with '%s'%s\n", |
| name, func->name, |
| fn->proto.optimized_parms ? |
| ", has optimized-out parameters" : |
| fn->proto.unexpected_reg ? ", has unexpected register use by params" : |
| ""); |
| fn->alias = func->name; |
| } |
| } |
| if (!func) |
| continue; |
| } else { |
| if (!fn->external) |
| continue; |
| } |
| |
| if (save) |
| err = btf_encoder__save_func(encoder, fn, func); |
| else |
| err = btf_encoder__add_func(encoder, fn); |
| if (err) |
| goto out; |
| } |
| |
| if (!encoder->skip_encoding_vars) |
| err = btf_encoder__encode_cu_variables(encoder); |
| |
| /* It is only safe to delete this CU if we have not stashed any static |
| * functions for later addition. |
| */ |
| if (!err) |
| err = encoder->saved_func_cnt > 0 ? LSK__KEEPIT : LSK__DELETE; |
| out: |
| encoder->cu = NULL; |
| return err; |
| } |
| |
| struct btf *btf_encoder__btf(struct btf_encoder *encoder) |
| { |
| return encoder->btf; |
| } |