diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 4dc00411f..b965f4a07 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,9 +1,10 @@ add_subdirectory(dom) -include_directories( . linux ) +include_directories( . ) link_libraries(simdjson-windows-headers test-data) link_libraries(simdjson) +link_libraries(counters) if(SIMDJSON_STATIC_REFLECTION) add_compile_definitions(SIMDJSON_STATIC_REFLECTION=1) endif(SIMDJSON_STATIC_REFLECTION) diff --git a/benchmark/apple/apple_arm_events.h b/benchmark/apple/apple_arm_events.h deleted file mode 100644 index 2156414e3..000000000 --- a/benchmark/apple/apple_arm_events.h +++ /dev/null @@ -1,1107 +0,0 @@ - -// Original design from: -// ============================================================================= -// XNU kperf/kpc -// Available for 64-bit Intel/Apple Silicon, macOS/iOS, with root privileges -// -// References: -// -// XNU source (since xnu 2422.1.72): -// https://github.com/apple/darwin-xnu/blob/main/osfmk/kern/kpc.h -// https://github.com/apple/darwin-xnu/blob/main/bsd/kern/kern_kpc.c -// -// Lightweight PET (Profile Every Thread, since xnu 3789.1.32): -// https://github.com/apple/darwin-xnu/blob/main/osfmk/kperf/pet.c -// https://github.com/apple/darwin-xnu/blob/main/osfmk/kperf/kperf_kpc.c -// -// System Private frameworks (since macOS 10.11, iOS 8.0): -// /System/Library/PrivateFrameworks/kperf.framework -// /System/Library/PrivateFrameworks/kperfdata.framework -// -// Xcode framework (since Xcode 7.0): -// /Applications/Xcode.app/Contents/SharedFrameworks/DVTInstrumentsFoundation.framework -// -// CPU database (plist files) -// macOS (since macOS 10.11): -// /usr/share/kpep/.plist -// iOS (copied from Xcode, since iOS 10.0, Xcode 8.0): -// /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform -// /DeviceSupport//DeveloperDiskImage.dmg/usr/share/kpep/.plist -// -// -// Created by YaoYuan on 2021. -// Released into the public domain (unlicense.org). -// ============================================================================= - -#ifndef M1CYCLES_H -#define M1CYCLES_H - -#include -#include -#include -#include -#include - -#include // for dlopen() and dlsym() -#include // for mach_absolute_time() -#include // for kdebug trace decode -#include // for sysctl() -#include // for usleep() - -struct performance_counters { - double cycles; - double branches; - double missed_branches; - double instructions; - performance_counters(uint64_t c, uint64_t b, uint64_t m, uint64_t i) - : cycles(c), branches(b), missed_branches(m), instructions(i) {} - performance_counters(double c, double b, double m, double i) - : cycles(c), branches(b), missed_branches(m), instructions(i) {} - performance_counters(double init) - : cycles(init), branches(init), missed_branches(init), - instructions(init) {} - - inline performance_counters &operator-=(const performance_counters &other) { - cycles -= other.cycles; - branches -= other.branches; - missed_branches -= other.missed_branches; - instructions -= other.instructions; - return *this; - } - inline performance_counters &min(const performance_counters &other) { - cycles = other.cycles < cycles ? other.cycles : cycles; - branches = other.branches < branches ? other.branches : branches; - missed_branches = other.missed_branches < missed_branches - ? other.missed_branches - : missed_branches; - instructions = - other.instructions < instructions ? other.instructions : instructions; - return *this; - } - inline performance_counters &operator+=(const performance_counters &other) { - cycles += other.cycles; - branches += other.branches; - missed_branches += other.missed_branches; - instructions += other.instructions; - return *this; - } - - inline performance_counters &operator/=(double numerator) { - cycles /= numerator; - branches /= numerator; - missed_branches /= numerator; - instructions /= numerator; - return *this; - } -}; - -inline performance_counters operator-(const performance_counters &a, - const performance_counters &b) { - return performance_counters(a.cycles - b.cycles, a.branches - b.branches, - a.missed_branches - b.missed_branches, - a.instructions - b.instructions); -} - -typedef float f32; -typedef double f64; -typedef int8_t i8; -typedef uint8_t u8; -typedef int16_t i16; -typedef uint16_t u16; -typedef int32_t i32; -typedef uint32_t u32; -typedef int64_t i64; -typedef uint64_t u64; -typedef size_t usize; - -// ----------------------------------------------------------------------------- -// header (reverse engineered) -// This framework wraps some sysctl calls to communicate with the kpc in kernel. -// Most functions requires root privileges, or process is "blessed". -// ----------------------------------------------------------------------------- - -// Cross-platform class constants. -#define KPC_CLASS_FIXED (0) -#define KPC_CLASS_CONFIGURABLE (1) -#define KPC_CLASS_POWER (2) -#define KPC_CLASS_RAWPMU (3) - -// Cross-platform class mask constants. -#define KPC_CLASS_FIXED_MASK (1u << KPC_CLASS_FIXED) // 1 -#define KPC_CLASS_CONFIGURABLE_MASK (1u << KPC_CLASS_CONFIGURABLE) // 2 -#define KPC_CLASS_POWER_MASK (1u << KPC_CLASS_POWER) // 4 -#define KPC_CLASS_RAWPMU_MASK (1u << KPC_CLASS_RAWPMU) // 8 - -// PMU version constants. -#define KPC_PMU_ERROR (0) // Error -#define KPC_PMU_INTEL_V3 (1) // Intel -#define KPC_PMU_ARM_APPLE (2) // ARM64 -#define KPC_PMU_INTEL_V2 (3) // Old Intel -#define KPC_PMU_ARM_V2 (4) // Old ARM - -// The maximum number of counters we could read from every class in one go. -// ARMV7: FIXED: 1, CONFIGURABLE: 4 -// ARM32: FIXED: 2, CONFIGURABLE: 6 -// ARM64: FIXED: 2, CONFIGURABLE: CORE_NCTRS - FIXED (6 or 8) -// x86: 32 -#define KPC_MAX_COUNTERS 32 - -// Bits for defining what to do on an action. -// Defined in https://github.com/apple/darwin-xnu/blob/main/osfmk/kperf/action.h -#define KPERF_SAMPLER_TH_INFO (1U << 0) -#define KPERF_SAMPLER_TH_SNAPSHOT (1U << 1) -#define KPERF_SAMPLER_KSTACK (1U << 2) -#define KPERF_SAMPLER_USTACK (1U << 3) -#define KPERF_SAMPLER_PMC_THREAD (1U << 4) -#define KPERF_SAMPLER_PMC_CPU (1U << 5) -#define KPERF_SAMPLER_PMC_CONFIG (1U << 6) -#define KPERF_SAMPLER_MEMINFO (1U << 7) -#define KPERF_SAMPLER_TH_SCHEDULING (1U << 8) -#define KPERF_SAMPLER_TH_DISPATCH (1U << 9) -#define KPERF_SAMPLER_TK_SNAPSHOT (1U << 10) -#define KPERF_SAMPLER_SYS_MEM (1U << 11) -#define KPERF_SAMPLER_TH_INSCYC (1U << 12) -#define KPERF_SAMPLER_TK_INFO (1U << 13) - -// Maximum number of kperf action ids. -#define KPERF_ACTION_MAX (32) - -// Maximum number of kperf timer ids. -#define KPERF_TIMER_MAX (8) - -// x86/arm config registers are 64-bit -typedef u64 kpc_config_t; - -/// Print current CPU identification string to the buffer (same as snprintf), -/// such as "cpu_7_8_10b282dc_46". This string can be used to locate the PMC -/// database in /usr/share/kpep. -/// @return string's length, or negative value if error occurs. -/// @note This method does not requires root privileges. -/// @details sysctl get(hw.cputype), get(hw.cpusubtype), -/// get(hw.cpufamily), get(machdep.cpu.model) -static int (*kpc_cpu_string)(char *buf, usize buf_size); - -/// Get the version of KPC that's being run. -/// @return See `PMU version constants` above. -/// @details sysctl get(kpc.pmu_version) -static u32 (*kpc_pmu_version)(void); - -/// Get running PMC classes. -/// @return See `class mask constants` above, -/// 0 if error occurs or no class is set. -/// @details sysctl get(kpc.counting) -static u32 (*kpc_get_counting)(void); - -/// Set PMC classes to enable counting. -/// @param classes See `class mask constants` above, set 0 to shutdown counting. -/// @return 0 for success. -/// @details sysctl set(kpc.counting) -static int (*kpc_set_counting)(u32 classes); - -/// Get running PMC classes for current thread. -/// @return See `class mask constants` above, -/// 0 if error occurs or no class is set. -/// @details sysctl get(kpc.thread_counting) -static u32 (*kpc_get_thread_counting)(void); - -/// Set PMC classes to enable counting for current thread. -/// @param classes See `class mask constants` above, set 0 to shutdown counting. -/// @return 0 for success. -/// @details sysctl set(kpc.thread_counting) -static int (*kpc_set_thread_counting)(u32 classes); - -/// Get how many config registers there are for a given mask. -/// For example: Intel may returns 1 for `KPC_CLASS_FIXED_MASK`, -/// returns 4 for `KPC_CLASS_CONFIGURABLE_MASK`. -/// @param classes See `class mask constants` above. -/// @return 0 if error occurs or no class is set. -/// @note This method does not requires root privileges. -/// @details sysctl get(kpc.config_count) -static u32 (*kpc_get_config_count)(u32 classes); - -/// Get config registers. -/// @param classes see `class mask constants` above. -/// @param config Config buffer to receive values, should not smaller than -/// kpc_get_config_count(classes) * sizeof(kpc_config_t). -/// @return 0 for success. -/// @details sysctl get(kpc.config_count), get(kpc.config) -static int (*kpc_get_config)(u32 classes, kpc_config_t *config); - -/// Set config registers. -/// @param classes see `class mask constants` above. -/// @param config Config buffer, should not smaller than -/// kpc_get_config_count(classes) * sizeof(kpc_config_t). -/// @return 0 for success. -/// @details sysctl get(kpc.config_count), set(kpc.config) -static int (*kpc_set_config)(u32 classes, kpc_config_t *config); - -/// Get how many counters there are for a given mask. -/// For example: Intel may returns 3 for `KPC_CLASS_FIXED_MASK`, -/// returns 4 for `KPC_CLASS_CONFIGURABLE_MASK`. -/// @param classes See `class mask constants` above. -/// @note This method does not requires root privileges. -/// @details sysctl get(kpc.counter_count) -static u32 (*kpc_get_counter_count)(u32 classes); - -/// Get counter accumulations. -/// If `all_cpus` is true, the buffer count should not smaller than -/// (cpu_count * counter_count). Otherwise, the buffer count should not smaller -/// than (counter_count). -/// @see kpc_get_counter_count(), kpc_cpu_count(). -/// @param all_cpus true for all CPUs, false for current cpu. -/// @param classes See `class mask constants` above. -/// @param curcpu A pointer to receive current cpu id, can be NULL. -/// @param buf Buffer to receive counter's value. -/// @return 0 for success. -/// @details sysctl get(hw.ncpu), get(kpc.counter_count), get(kpc.counters) -static int (*kpc_get_cpu_counters)(bool all_cpus, u32 classes, int *curcpu, - u64 *buf); - -/// Get counter accumulations for current thread. -/// @param tid Thread id, should be 0. -/// @param buf_count The number of buf's elements (not bytes), -/// should not smaller than kpc_get_counter_count(). -/// @param buf Buffer to receive counter's value. -/// @return 0 for success. -/// @details sysctl get(kpc.thread_counters) -static int (*kpc_get_thread_counters)(u32 tid, u32 buf_count, u64 *buf); - -/// Acquire/release the counters used by the Power Manager. -/// @param val 1:acquire, 0:release -/// @return 0 for success. -/// @details sysctl set(kpc.force_all_ctrs) -static int (*kpc_force_all_ctrs_set)(int val); - -/// Get the state of all_ctrs. -/// @return 0 for success. -/// @details sysctl get(kpc.force_all_ctrs) -static int (*kpc_force_all_ctrs_get)(int *val_out); - -/// Set number of actions, should be `KPERF_ACTION_MAX`. -/// @details sysctl set(kperf.action.count) -static int (*kperf_action_count_set)(u32 count); - -/// Get number of actions. -/// @details sysctl get(kperf.action.count) -static int (*kperf_action_count_get)(u32 *count); - -/// Set what to sample when a trigger fires an action, e.g. -/// `KPERF_SAMPLER_PMC_CPU`. -/// @details sysctl set(kperf.action.samplers) -static int (*kperf_action_samplers_set)(u32 actionid, u32 sample); - -/// Get what to sample when a trigger fires an action. -/// @details sysctl get(kperf.action.samplers) -static int (*kperf_action_samplers_get)(u32 actionid, u32 *sample); - -/// Apply a task filter to the action, -1 to disable filter. -/// @details sysctl set(kperf.action.filter_by_task) -static int (*kperf_action_filter_set_by_task)(u32 actionid, i32 port); - -/// Apply a pid filter to the action, -1 to disable filter. -/// @details sysctl set(kperf.action.filter_by_pid) -static int (*kperf_action_filter_set_by_pid)(u32 actionid, i32 pid); - -/// Set number of time triggers, should be `KPERF_TIMER_MAX`. -/// @details sysctl set(kperf.timer.count) -static int (*kperf_timer_count_set)(u32 count); - -/// Get number of time triggers. -/// @details sysctl get(kperf.timer.count) -static int (*kperf_timer_count_get)(u32 *count); - -/// Set timer number and period. -/// @details sysctl set(kperf.timer.period) -static int (*kperf_timer_period_set)(u32 actionid, u64 tick); - -/// Get timer number and period. -/// @details sysctl get(kperf.timer.period) -static int (*kperf_timer_period_get)(u32 actionid, u64 *tick); - -/// Set timer number and actionid. -/// @details sysctl set(kperf.timer.action) -static int (*kperf_timer_action_set)(u32 actionid, u32 timerid); - -/// Get timer number and actionid. -/// @details sysctl get(kperf.timer.action) -static int (*kperf_timer_action_get)(u32 actionid, u32 *timerid); - -/// Set which timer ID does PET (Profile Every Thread). -/// @details sysctl set(kperf.timer.pet_timer) -static int (*kperf_timer_pet_set)(u32 timerid); - -/// Get which timer ID does PET (Profile Every Thread). -/// @details sysctl get(kperf.timer.pet_timer) -static int (*kperf_timer_pet_get)(u32 *timerid); - -/// Enable or disable sampling. -/// @details sysctl set(kperf.sampling) -static int (*kperf_sample_set)(u32 enabled); - -/// Get is currently sampling. -/// @details sysctl get(kperf.sampling) -static int (*kperf_sample_get)(u32 *enabled); - -/// Reset kperf: stop sampling, kdebug, timers and actions. -/// @return 0 for success. -static int (*kperf_reset)(void); - -/// Nanoseconds to CPU ticks. -static u64 (*kperf_ns_to_ticks)(u64 ns); - -/// CPU ticks to nanoseconds. -static u64 (*kperf_ticks_to_ns)(u64 ticks); - -/// CPU ticks frequency (mach_absolute_time). -static u64 (*kperf_tick_frequency)(void); - -/// Get lightweight PET mode (not in kperf.framework). -static int kperf_lightweight_pet_get(u32 *enabled) { - if (!enabled) - return -1; - usize size = 4; - return sysctlbyname("kperf.lightweight_pet", enabled, &size, NULL, 0); -} - -/// Set lightweight PET mode (not in kperf.framework). -static int kperf_lightweight_pet_set(u32 enabled) { - return sysctlbyname("kperf.lightweight_pet", NULL, NULL, &enabled, 4); -} - -// ----------------------------------------------------------------------------- -// header (reverse engineered) -// This framework provides some functions to access the local CPU database. -// These functions do not require root privileges. -// ----------------------------------------------------------------------------- - -// KPEP CPU architecture constants. -#define KPEP_ARCH_I386 0 -#define KPEP_ARCH_X86_64 1 -#define KPEP_ARCH_ARM 2 -#define KPEP_ARCH_ARM64 3 - -/// KPEP event (size: 48/28 bytes on 64/32 bit OS) -typedef struct kpep_event { - const char *name; ///< Unique name of a event, such as "INST_RETIRED.ANY". - const char *description; ///< Description for this event. - const char *errata; ///< Errata, currently NULL. - const char *alias; ///< Alias name, such as "Instructions", "Cycles". - const char *fallback; ///< Fallback event name for fixed counter. - u32 mask; - u8 number; - u8 umask; - u8 reserved; - u8 is_fixed; -} kpep_event; - -/// KPEP database (size: 144/80 bytes on 64/32 bit OS) -typedef struct kpep_db { - const char *name; ///< Database name, such as "haswell". - const char *cpu_id; ///< Plist name, such as "cpu_7_8_10b282dc". - const char *marketing_name; ///< Marketing name, such as "Intel Haswell". - void *plist_data; ///< Plist data (CFDataRef), currently NULL. - void *event_map; ///< All events (CFDict). - kpep_event - *event_arr; ///< Event struct buffer (sizeof(kpep_event) * events_count). - kpep_event **fixed_event_arr; ///< Fixed counter events (sizeof(kpep_event *) - ///< * fixed_counter_count) - void *alias_map; ///< All aliases (CFDict). - usize reserved_1; - usize reserved_2; - usize reserved_3; - usize event_count; ///< All events count. - usize alias_count; - usize fixed_counter_count; - usize config_counter_count; - usize power_counter_count; - u32 architecture; ///< see `KPEP CPU architecture constants` above. - u32 fixed_counter_bits; - u32 config_counter_bits; - u32 power_counter_bits; -} kpep_db; - -/// KPEP config (size: 80/44 bytes on 64/32 bit OS) -typedef struct kpep_config { - kpep_db *db; - kpep_event **ev_arr; ///< (sizeof(kpep_event *) * counter_count), init NULL - usize *ev_map; ///< (sizeof(usize *) * counter_count), init 0 - usize *ev_idx; ///< (sizeof(usize *) * counter_count), init -1 - u32 *flags; ///< (sizeof(u32 *) * counter_count), init 0 - u64 *kpc_periods; ///< (sizeof(u64 *) * counter_count), init 0 - usize event_count; /// kpep_config_events_count() - usize counter_count; - u32 classes; ///< See `class mask constants` above. - u32 config_counter; - u32 power_counter; - u32 reserved; -} kpep_config; - -/// Error code for kpep_config_xxx() and kpep_db_xxx() functions. -typedef enum { - KPEP_CONFIG_ERROR_NONE = 0, - KPEP_CONFIG_ERROR_INVALID_ARGUMENT = 1, - KPEP_CONFIG_ERROR_OUT_OF_MEMORY = 2, - KPEP_CONFIG_ERROR_IO = 3, - KPEP_CONFIG_ERROR_BUFFER_TOO_SMALL = 4, - KPEP_CONFIG_ERROR_CUR_SYSTEM_UNKNOWN = 5, - KPEP_CONFIG_ERROR_DB_PATH_INVALID = 6, - KPEP_CONFIG_ERROR_DB_NOT_FOUND = 7, - KPEP_CONFIG_ERROR_DB_ARCH_UNSUPPORTED = 8, - KPEP_CONFIG_ERROR_DB_VERSION_UNSUPPORTED = 9, - KPEP_CONFIG_ERROR_DB_CORRUPT = 10, - KPEP_CONFIG_ERROR_EVENT_NOT_FOUND = 11, - KPEP_CONFIG_ERROR_CONFLICTING_EVENTS = 12, - KPEP_CONFIG_ERROR_COUNTERS_NOT_FORCED = 13, - KPEP_CONFIG_ERROR_EVENT_UNAVAILABLE = 14, - KPEP_CONFIG_ERROR_ERRNO = 15, - KPEP_CONFIG_ERROR_MAX -} kpep_config_error_code; - -/// Error description for kpep_config_error_code. -static const char *kpep_config_error_names[KPEP_CONFIG_ERROR_MAX] = { - "none", - "invalid argument", - "out of memory", - "I/O", - "buffer too small", - "current system unknown", - "database path invalid", - "database not found", - "database architecture unsupported", - "database version unsupported", - "database corrupt", - "event not found", - "conflicting events", - "all counters must be forced", - "event unavailable", - "check errno"}; - -/// Error description. -static const char *kpep_config_error_desc(int code) { - if (0 <= code && code < KPEP_CONFIG_ERROR_MAX) { - return kpep_config_error_names[code]; - } - return "unknown error"; -} - -/// Create a config. -/// @param db A kpep db, see kpep_db_create() -/// @param cfg_ptr A pointer to receive the new config. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_create)(kpep_db *db, kpep_config **cfg_ptr); - -/// Free the config. -static void (*kpep_config_free)(kpep_config *cfg); - -/// Add an event to config. -/// @param cfg The config. -/// @param ev_ptr A event pointer. -/// @param flag 0: all, 1: user space only -/// @param err Error bitmap pointer, can be NULL. -/// If return value is `CONFLICTING_EVENTS`, this bitmap contains -/// the conflicted event indices, e.g. "1 << 2" means index 2. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_add_event)(kpep_config *cfg, kpep_event **ev_ptr, - u32 flag, u32 *err); - -/// Remove event at index. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_remove_event)(kpep_config *cfg, usize idx); - -/// Force all counters. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_force_counters)(kpep_config *cfg); - -/// Get events count. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_events_count)(kpep_config *cfg, usize *count_ptr); - -/// Get all event pointers. -/// @param buf A buffer to receive event pointers. -/// @param buf_size The buffer's size in bytes, should not smaller than -/// kpep_config_events_count() * sizeof(void *). -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_events)(kpep_config *cfg, kpep_event **buf, - usize buf_size); - -/// Get kpc register configs. -/// @param buf A buffer to receive kpc register configs. -/// @param buf_size The buffer's size in bytes, should not smaller than -/// kpep_config_kpc_count() * sizeof(kpc_config_t). -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_kpc)(kpep_config *cfg, kpc_config_t *buf, - usize buf_size); - -/// Get kpc register config count. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_kpc_count)(kpep_config *cfg, usize *count_ptr); - -/// Get kpc classes. -/// @param classes See `class mask constants` above. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_kpc_classes)(kpep_config *cfg, u32 *classes_ptr); - -/// Get the index mapping from event to counter. -/// @param buf A buffer to receive indexes. -/// @param buf_size The buffer's size in bytes, should not smaller than -/// kpep_config_events_count() * sizeof(kpc_config_t). -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_config_kpc_map)(kpep_config *cfg, usize *buf, usize buf_size); - -/// Open a kpep database file in "/usr/share/kpep/" or "/usr/local/share/kpep/". -/// @param name File name, for example "haswell", "cpu_100000c_1_92fb37c8". -/// Pass NULL for current CPU. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_db_create)(const char *name, kpep_db **db_ptr); - -/// Free the kpep database. -static void (*kpep_db_free)(kpep_db *db); - -/// Get the database's name. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_db_name)(kpep_db *db, const char **name); - -/// Get the event alias count. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_db_aliases_count)(kpep_db *db, usize *count); - -/// Get all alias. -/// @param buf A buffer to receive all alias strings. -/// @param buf_size The buffer's size in bytes, -/// should not smaller than kpep_db_aliases_count() * sizeof(void *). -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_db_aliases)(kpep_db *db, const char **buf, usize buf_size); - -/// Get counters count for given classes. -/// @param classes 1: Fixed, 2: Configurable. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_db_counters_count)(kpep_db *db, u8 classes, usize *count); - -/// Get all event count. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_db_events_count)(kpep_db *db, usize *count); - -/// Get all events. -/// @param buf A buffer to receive all event pointers. -/// @param buf_size The buffer's size in bytes, -/// should not smaller than kpep_db_events_count() * sizeof(void *). -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_db_events)(kpep_db *db, kpep_event **buf, usize buf_size); - -/// Get one event by name. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_db_event)(kpep_db *db, const char *name, kpep_event **ev_ptr); - -/// Get event's name. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_event_name)(kpep_event *ev, const char **name_ptr); - -/// Get event's alias. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_event_alias)(kpep_event *ev, const char **alias_ptr); - -/// Get event's description. -/// @return kpep_config_error_code, 0 for success. -static int (*kpep_event_description)(kpep_event *ev, const char **str_ptr); - -// ----------------------------------------------------------------------------- -// load kperf/kperfdata dynamic library -// ----------------------------------------------------------------------------- - -typedef struct { - const char *name; - void **impl; -} lib_symbol; - -#define lib_nelems(x) (sizeof(x) / sizeof((x)[0])) -#define lib_symbol_def(name) \ - { \ -#name, (void **)&name \ - } - -static const lib_symbol lib_symbols_kperf[] = { - lib_symbol_def(kpc_pmu_version), - lib_symbol_def(kpc_cpu_string), - lib_symbol_def(kpc_set_counting), - lib_symbol_def(kpc_get_counting), - lib_symbol_def(kpc_set_thread_counting), - lib_symbol_def(kpc_get_thread_counting), - lib_symbol_def(kpc_get_config_count), - lib_symbol_def(kpc_get_counter_count), - lib_symbol_def(kpc_set_config), - lib_symbol_def(kpc_get_config), - lib_symbol_def(kpc_get_cpu_counters), - lib_symbol_def(kpc_get_thread_counters), - lib_symbol_def(kpc_force_all_ctrs_set), - lib_symbol_def(kpc_force_all_ctrs_get), - lib_symbol_def(kperf_action_count_set), - lib_symbol_def(kperf_action_count_get), - lib_symbol_def(kperf_action_samplers_set), - lib_symbol_def(kperf_action_samplers_get), - lib_symbol_def(kperf_action_filter_set_by_task), - lib_symbol_def(kperf_action_filter_set_by_pid), - lib_symbol_def(kperf_timer_count_set), - lib_symbol_def(kperf_timer_count_get), - lib_symbol_def(kperf_timer_period_set), - lib_symbol_def(kperf_timer_period_get), - lib_symbol_def(kperf_timer_action_set), - lib_symbol_def(kperf_timer_action_get), - lib_symbol_def(kperf_sample_set), - lib_symbol_def(kperf_sample_get), - lib_symbol_def(kperf_reset), - lib_symbol_def(kperf_timer_pet_set), - lib_symbol_def(kperf_timer_pet_get), - lib_symbol_def(kperf_ns_to_ticks), - lib_symbol_def(kperf_ticks_to_ns), - lib_symbol_def(kperf_tick_frequency), -}; - -static const lib_symbol lib_symbols_kperfdata[] = { - lib_symbol_def(kpep_config_create), - lib_symbol_def(kpep_config_free), - lib_symbol_def(kpep_config_add_event), - lib_symbol_def(kpep_config_remove_event), - lib_symbol_def(kpep_config_force_counters), - lib_symbol_def(kpep_config_events_count), - lib_symbol_def(kpep_config_events), - lib_symbol_def(kpep_config_kpc), - lib_symbol_def(kpep_config_kpc_count), - lib_symbol_def(kpep_config_kpc_classes), - lib_symbol_def(kpep_config_kpc_map), - lib_symbol_def(kpep_db_create), - lib_symbol_def(kpep_db_free), - lib_symbol_def(kpep_db_name), - lib_symbol_def(kpep_db_aliases_count), - lib_symbol_def(kpep_db_aliases), - lib_symbol_def(kpep_db_counters_count), - lib_symbol_def(kpep_db_events_count), - lib_symbol_def(kpep_db_events), - lib_symbol_def(kpep_db_event), - lib_symbol_def(kpep_event_name), - lib_symbol_def(kpep_event_alias), - lib_symbol_def(kpep_event_description), -}; - -#define lib_path_kperf "/System/Library/PrivateFrameworks/kperf.framework/kperf" -#define lib_path_kperfdata \ - "/System/Library/PrivateFrameworks/kperfdata.framework/kperfdata" - -static bool lib_inited = false; -static bool lib_has_err = false; -static char lib_err_msg[256]; - -static void *lib_handle_kperf = NULL; -static void *lib_handle_kperfdata = NULL; - -static void lib_deinit(void) { - lib_inited = false; - lib_has_err = false; - if (lib_handle_kperf) - dlclose(lib_handle_kperf); - if (lib_handle_kperfdata) - dlclose(lib_handle_kperfdata); - lib_handle_kperf = NULL; - lib_handle_kperfdata = NULL; - for (usize i = 0; i < lib_nelems(lib_symbols_kperf); i++) { - const lib_symbol *symbol = &lib_symbols_kperf[i]; - *symbol->impl = NULL; - } - for (usize i = 0; i < lib_nelems(lib_symbols_kperfdata); i++) { - const lib_symbol *symbol = &lib_symbols_kperfdata[i]; - *symbol->impl = NULL; - } -} - -static bool lib_init(void) { -#define return_err() \ - do { \ - lib_deinit(); \ - lib_inited = true; \ - lib_has_err = true; \ - return false; \ - } while (false) - - if (lib_inited) - return !lib_has_err; - - // load dynamic library - lib_handle_kperf = dlopen(lib_path_kperf, RTLD_LAZY); - if (!lib_handle_kperf) { - snprintf(lib_err_msg, sizeof(lib_err_msg), - "Failed to load kperf.framework, message: %s.", dlerror()); - return_err(); - } - lib_handle_kperfdata = dlopen(lib_path_kperfdata, RTLD_LAZY); - if (!lib_handle_kperfdata) { - snprintf(lib_err_msg, sizeof(lib_err_msg), - "Failed to load kperfdata.framework, message: %s.", dlerror()); - return_err(); - } - - // load symbol address from dynamic library - for (usize i = 0; i < lib_nelems(lib_symbols_kperf); i++) { - const lib_symbol *symbol = &lib_symbols_kperf[i]; - *symbol->impl = dlsym(lib_handle_kperf, symbol->name); - if (!*symbol->impl) { - snprintf(lib_err_msg, sizeof(lib_err_msg), - "Failed to load kperf function: %s.", symbol->name); - return_err(); - } - } - for (usize i = 0; i < lib_nelems(lib_symbols_kperfdata); i++) { - const lib_symbol *symbol = &lib_symbols_kperfdata[i]; - *symbol->impl = dlsym(lib_handle_kperfdata, symbol->name); - if (!*symbol->impl) { - snprintf(lib_err_msg, sizeof(lib_err_msg), - "Failed to load kperfdata function: %s.", symbol->name); - return_err(); - } - } - - lib_inited = true; - lib_has_err = false; - return true; - -#undef return_err -} - -// ----------------------------------------------------------------------------- -// kdebug private structs -// https://github.com/apple/darwin-xnu/blob/main/bsd/sys_private/kdebug_private.h -// ----------------------------------------------------------------------------- - -/* - * Ensure that both LP32 and LP64 variants of arm64 use the same kd_buf - * structure. - */ -#if defined(__arm64__) -typedef uint64_t kd_buf_argtype; -#else -typedef uintptr_t kd_buf_argtype; -#endif - -typedef struct { - uint64_t timestamp; - kd_buf_argtype arg1; - kd_buf_argtype arg2; - kd_buf_argtype arg3; - kd_buf_argtype arg4; - kd_buf_argtype arg5; /* the thread ID */ - uint32_t debugid; /* see */ - -/* - * Ensure that both LP32 and LP64 variants of arm64 use the same kd_buf - * structure. - */ -#if defined(__LP64__) || defined(__arm64__) - uint32_t cpuid; /* cpu index, from 0 */ - kd_buf_argtype unused; -#endif -} kd_buf; - -/* bits for the type field of kd_regtype */ -#define KDBG_CLASSTYPE 0x10000 -#define KDBG_SUBCLSTYPE 0x20000 -#define KDBG_RANGETYPE 0x40000 -#define KDBG_TYPENONE 0x80000 -#define KDBG_CKTYPES 0xF0000 - -/* only trace at most 4 types of events, at the code granularity */ -#define KDBG_VALCHECK 0x00200000U - -typedef struct { - unsigned int type; - unsigned int value1; - unsigned int value2; - unsigned int value3; - unsigned int value4; -} kd_regtype; - -typedef struct { - /* number of events that can fit in the buffers */ - int nkdbufs; - /* set if trace is disabled */ - int nolog; - /* kd_ctrl_page.flags */ - unsigned int flags; - /* number of threads in thread map */ - int nkdthreads; - /* the owning pid */ - int bufid; -} kbufinfo_t; - -// ----------------------------------------------------------------------------- -// kdebug utils -// ----------------------------------------------------------------------------- - -/// Clean up trace buffers and reset ktrace/kdebug/kperf. -/// @return 0 on success. -static int kdebug_reset(void) { - int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDREMOVE}; - return sysctl(mib, 3, NULL, NULL, NULL, 0); -} - -/// Disable and reinitialize the trace buffers. -/// @return 0 on success. -static int kdebug_reinit(void) { - int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDSETUP}; - return sysctl(mib, 3, NULL, NULL, NULL, 0); -} - -/// Set debug filter. -static int kdebug_setreg(kd_regtype *kdr) { - int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDSETREG}; - usize size = sizeof(kd_regtype); - return sysctl(mib, 3, kdr, &size, NULL, 0); -} - -/// Set maximum number of trace entries (kd_buf). -/// Only allow allocation up to half the available memory (sane_size). -/// @return 0 on success. -static int kdebug_trace_setbuf(int nbufs) { - int mib[4] = {CTL_KERN, KERN_KDEBUG, KERN_KDSETBUF, nbufs}; - return sysctl(mib, 4, NULL, NULL, NULL, 0); -} - -/// Enable or disable kdebug trace. -/// Trace buffer must already be initialized. -/// @return 0 on success. -static int kdebug_trace_enable(bool enable) { - int mib[4] = {CTL_KERN, KERN_KDEBUG, KERN_KDENABLE, enable}; - return sysctl(mib, 4, NULL, 0, NULL, 0); -} - -/// Retrieve trace buffer information from kernel. -/// @return 0 on success. -static int kdebug_get_bufinfo(kbufinfo_t *info) { - if (!info) - return -1; - int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDGETBUF}; - size_t needed = sizeof(kbufinfo_t); - return sysctl(mib, 3, info, &needed, NULL, 0); -} - -/// Retrieve trace buffers from kernel. -/// @param buf Memory to receive buffer data, array of `kd_buf`. -/// @param len Length of `buf` in bytes. -/// @param count Number of trace entries (kd_buf) obtained. -/// @return 0 on success. -static int kdebug_trace_read(void *buf, usize len, usize *count) { - if (count) - *count = 0; - if (!buf || !len) - return -1; - - // Note: the input and output units are not the same. - // input: bytes - // output: number of kd_buf - int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDREADTR}; - int ret = sysctl(mib, 3, buf, &len, NULL, 0); - if (ret != 0) - return ret; - *count = len; - return 0; -} - -/// Block until there are new buffers filled or `timeout_ms` have passed. -/// @param timeout_ms timeout milliseconds, 0 means wait forever. -/// @param suc set true if new buffers filled. -/// @return 0 on success. -static int kdebug_wait(usize timeout_ms, bool *suc) { - if (timeout_ms == 0) - return -1; - int mib[3] = {CTL_KERN, KERN_KDEBUG, KERN_KDBUFWAIT}; - usize val = timeout_ms; - int ret = sysctl(mib, 3, NULL, &val, NULL, 0); - if (suc) - *suc = !!val; - return ret; -} - -// ----------------------------------------------------------------------------- -// Demo -// ----------------------------------------------------------------------------- - -#define EVENT_NAME_MAX 8 -typedef struct { - const char *alias; /// name for print - const char *names[EVENT_NAME_MAX]; /// name from pmc db -} event_alias; - -/// Event names from /usr/share/kpep/.plist -static const event_alias profile_events[] = { - {"cycles", - { - "FIXED_CYCLES", // Apple A7-A15 - "CPU_CLK_UNHALTED.THREAD", // Intel Core 1th-10th - "CPU_CLK_UNHALTED.CORE", // Intel Yonah, Merom - }}, - {"instructions", - { - "FIXED_INSTRUCTIONS", // Apple A7-A15 - "INST_RETIRED.ANY" // Intel Yonah, Merom, Core 1th-10th - }}, - {"branches", - { - "INST_BRANCH", // Apple A7-A15 - "BR_INST_RETIRED.ALL_BRANCHES", // Intel Core 1th-10th - "INST_RETIRED.ANY", // Intel Yonah, Merom - }}, - {"branch-misses", - { - "BRANCH_MISPRED_NONSPEC", // Apple A7-A15, since iOS 15, macOS 12 - "BRANCH_MISPREDICT", // Apple A7-A14 - "BR_MISP_RETIRED.ALL_BRANCHES", // Intel Core 2th-10th - "BR_INST_RETIRED.MISPRED", // Intel Yonah, Merom - }}, -}; - -static kpep_event *get_event(kpep_db *db, const event_alias *alias) { - for (usize j = 0; j < EVENT_NAME_MAX; j++) { - const char *name = alias->names[j]; - if (!name) - break; - kpep_event *ev = NULL; - if (kpep_db_event(db, name, &ev) == 0) { - return ev; - } - } - return NULL; -} - -struct AppleEvents { - kpc_config_t regs[KPC_MAX_COUNTERS] = {0}; - usize counter_map[KPC_MAX_COUNTERS] = {0}; - u64 counters_0[KPC_MAX_COUNTERS] = {0}; - u64 counters_1[KPC_MAX_COUNTERS] = {0}; - static constexpr usize ev_count = - sizeof(profile_events) / sizeof(profile_events[0]); - bool init = false; - bool worked = false; - inline bool setup_performance_counters() { - if (init) { - return worked; - } - init = true; - - // load dylib - if (!lib_init()) { - printf("Error: %s\n", lib_err_msg); - return (worked = false); - } - - // check permission - int force_ctrs = 0; - if (kpc_force_all_ctrs_get(&force_ctrs)) { - return (worked = false); - } - int ret; - // load pmc db - kpep_db *db = NULL; - if ((ret = kpep_db_create(NULL, &db))) { - printf("Error: cannot load pmc database: %d.\n", ret); - return (worked = false); - } - - // create a config - kpep_config *cfg = NULL; - if ((ret = kpep_config_create(db, &cfg))) { - printf("Failed to create kpep config: %d (%s).\n", ret, - kpep_config_error_desc(ret)); - return (worked = false); - } - if ((ret = kpep_config_force_counters(cfg))) { - printf("Failed to force counters: %d (%s).\n", ret, - kpep_config_error_desc(ret)); - return (worked = false); - } - - // get events - kpep_event *ev_arr[ev_count] = {0}; - for (usize i = 0; i < ev_count; i++) { - const event_alias *alias = profile_events + i; - ev_arr[i] = get_event(db, alias); - if (!ev_arr[i]) { - printf("Cannot find event: %s.\n", alias->alias); - return (worked = false); - } - } - - // add event to config - for (usize i = 0; i < ev_count; i++) { - kpep_event *ev = ev_arr[i]; - if ((ret = kpep_config_add_event(cfg, &ev, 0, NULL))) { - printf("Failed to add event: %d (%s).\n", ret, - kpep_config_error_desc(ret)); - return (worked = false); - } - } - - // prepare buffer and config - u32 classes = 0; - usize reg_count = 0; - if ((ret = kpep_config_kpc_classes(cfg, &classes))) { - printf("Failed get kpc classes: %d (%s).\n", ret, - kpep_config_error_desc(ret)); - return (worked = false); - } - if ((ret = kpep_config_kpc_count(cfg, ®_count))) { - printf("Failed get kpc count: %d (%s).\n", ret, - kpep_config_error_desc(ret)); - return (worked = false); - } - if ((ret = kpep_config_kpc_map(cfg, counter_map, sizeof(counter_map)))) { - printf("Failed get kpc map: %d (%s).\n", ret, - kpep_config_error_desc(ret)); - return (worked = false); - } - if ((ret = kpep_config_kpc(cfg, regs, sizeof(regs)))) { - printf("Failed get kpc registers: %d (%s).\n", ret, - kpep_config_error_desc(ret)); - return (worked = false); - } - - // set config to kernel - if ((ret = kpc_force_all_ctrs_set(1))) { - printf("Failed force all ctrs: %d.\n", ret); - return (worked = false); - } - if ((classes & KPC_CLASS_CONFIGURABLE_MASK) && reg_count) { - if ((ret = kpc_set_config(classes, regs))) { - printf("Failed set kpc config: %d.\n", ret); - return (worked = false); - } - } - - // start counting - if ((ret = kpc_set_counting(classes))) { - printf("Failed set counting: %d.\n", ret); - return (worked = false); - } - if ((ret = kpc_set_thread_counting(classes))) { - printf("Failed set thread counting: %d.\n", ret); - return (worked = false); - } - - return (worked = true); - } - - inline performance_counters get_counters() { - static bool warned = false; - int ret; - // get counters before - if ((ret = kpc_get_thread_counters(0, KPC_MAX_COUNTERS, counters_0))) { - if (!warned) { - - printf("Failed get thread counters before: %d.\n", ret); - warned = true; - } - return 1; - } - return performance_counters { - counters_0[counter_map[0]], counters_0[counter_map[2]], - counters_0[counter_map[3]], counters_0[counter_map[1]]}; - } -}; - -#endif diff --git a/benchmark/benchfeatures.cpp b/benchmark/benchfeatures.cpp index fd042f85d..3ebdd74da 100644 --- a/benchmark/benchfeatures.cpp +++ b/benchmark/benchfeatures.cpp @@ -1,4 +1,5 @@ -#include "event_counter.h" +#include +using namespace counters; #include #include @@ -25,7 +26,6 @@ #include #include -#include "linux-perf-events.h" #ifdef __linux__ #include #endif @@ -204,12 +204,8 @@ struct feature_benchmarker { } // Rate of 1-7-structural misses per 8-structural flip double struct1_7_miss_rate(BenchmarkStage stage) const { -#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - return 1; -#else if (!has_events()) { return 1; } return struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses() / double(struct7_miss.stats->blocks_with_1_structural_flipped); -#endif } // Extra cost of an 8-15 structural block over a 1-7 structural block double struct8_15_cost(BenchmarkStage stage) const { @@ -221,12 +217,8 @@ struct feature_benchmarker { } // Rate of 8-15-structural misses per 8-structural flip double struct8_15_miss_rate(BenchmarkStage stage) const { -#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - return 1; -#else if (!has_events()) { return 1; } return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / double(struct15_miss.stats->blocks_with_8_structurals_flipped); -#endif } // Extra cost of a 16+-structural block over an 8-15 structural block (actual varies based on # of structurals!) @@ -239,12 +231,8 @@ struct feature_benchmarker { } // Rate of 16-structural misses per 16-structural flip double struct16_miss_rate(BenchmarkStage stage) const { -#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - return 1; -#else if (!has_events()) { return 1; } return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / double(struct23_miss.stats->blocks_with_16_structurals_flipped); -#endif } @@ -258,12 +246,8 @@ struct feature_benchmarker { } // Rate of UTF-8 misses per UTF-8 flip double utf8_miss_rate(BenchmarkStage stage) const { -#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - return 1; -#else if (!has_events()) { return 1; } return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / double(utf8_miss.stats->blocks_with_utf8_flipped); -#endif } // Extra cost of having escapes in a block double escape_cost(BenchmarkStage stage) const { @@ -275,12 +259,8 @@ struct feature_benchmarker { } // Rate of escape misses per escape flip double escape_miss_rate(BenchmarkStage stage) const { -#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - return 1; -#else if (!has_events()) { return 1; } return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / double(escape_miss.stats->blocks_with_escapes_flipped); -#endif } @@ -378,22 +358,6 @@ struct feature_benchmarker { } }; -#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS -void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) { - double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks); - double calc = features.calc_expected(stage, results); - double calc_misses = features.calc_expected_misses(stage, results); - double calc_miss_cost = features.calc_expected_miss_cost(stage, results); - printf(" | %-8s ", benchmark_stage_name(stage)); - printf("| %-15s ", filename); - printf("| %8.3g ", features.calc_expected_feature_cost(stage, results)); - printf("| %8.3g ", calc_miss_cost); - printf("| %8.3g ", calc); - printf("| %8.3g ", actual); - printf("| %+8.3g ", actual - calc); - printf("| %13llu ", (long long unsigned)(calc_misses)); -} -#else void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) { double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks); double calc = features.calc_expected(stage, results); @@ -417,7 +381,6 @@ void print_file_effectiveness(BenchmarkStage stage, const char* filename, const } printf("|\n"); } -#endif int main(int argc, char *argv[]) { // Read options diff --git a/benchmark/benchmark.h b/benchmark/benchmark.h index cdcf7331f..d40d2c692 100644 --- a/benchmark/benchmark.h +++ b/benchmark/benchmark.h @@ -1,7 +1,8 @@ #ifndef _BENCHMARK_H_ #define _BENCHMARK_H_ -#include "event_counter.h" +#include +using namespace counters; /* * Prints the best number of operations per cycle where diff --git a/benchmark/benchmarker.h b/benchmark/benchmarker.h index b90a5a407..9c330bd23 100644 --- a/benchmark/benchmarker.h +++ b/benchmark/benchmarker.h @@ -1,7 +1,8 @@ #ifndef __BENCHMARKER_H #define __BENCHMARKER_H -#include "event_counter.h" +#include +using namespace counters; #include "simdjson.h" #include @@ -28,11 +29,9 @@ #include #include -#include "linux-perf-events.h" #ifdef __linux__ #include #endif -#include "simdjson.h" #include @@ -423,18 +422,12 @@ struct benchmarker { stage.instructions() / static_cast(stats->structurals), stage.instructions() / static_cast(stage.cycles()) ); -#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - // NOTE: removed cycles/miss because it is a somewhat misleading stat - printf("%s%-13s: %7.0f branch misses (%6.2f%%) - %.0f cache misses (%6.2f%%) - %.2f cache references\n", + printf("%s%-13s: %7.0f branch misses (%6.2f%%)\n", prefix, "Misses", stage.branch_misses(), - percent(stage.branch_misses(), all_stages_without_allocation.branch_misses()), - stage.cache_misses(), - percent(stage.cache_misses(), all_stages_without_allocation.cache_misses()), - stage.cache_references() + percent(stage.branch_misses(), all_stages_without_allocation.branch_misses()) ); -#endif } } diff --git a/benchmark/car_builder/benchmark_car_builder.cpp b/benchmark/car_builder/benchmark_car_builder.cpp index 66b8d996e..ecbc42eda 100644 --- a/benchmark/car_builder/benchmark_car_builder.cpp +++ b/benchmark/car_builder/benchmark_car_builder.cpp @@ -1,4 +1,5 @@ -#include "event_counter.h" +#include +using namespace counters; #include #include diff --git a/benchmark/dom/CMakeLists.txt b/benchmark/dom/CMakeLists.txt index 0528cdf69..664ce3ee8 100644 --- a/benchmark/dom/CMakeLists.txt +++ b/benchmark/dom/CMakeLists.txt @@ -1,11 +1,12 @@ -include_directories( .. ../linux ) +include_directories( .. ) link_libraries(simdjson-windows-headers test-data) link_libraries(simdjson) +link_libraries(counters) add_executable(perfdiff perfdiff.cpp) add_executable(parse parse.cpp) add_executable(parse_stream parse_stream.cpp) -add_executable(statisticalmodel statisticalmodel.cpp) + add_executable(parse_noutf8validation parse.cpp) target_compile_definitions(parse_noutf8validation PRIVATE SIMDJSON_SKIPUTF8VALIDATION) diff --git a/benchmark/dom/parse.cpp b/benchmark/dom/parse.cpp index d26622da1..7b88403fb 100644 --- a/benchmark/dom/parse.cpp +++ b/benchmark/dom/parse.cpp @@ -1,4 +1,5 @@ -#include "event_counter.h" +#include +using namespace counters; #include #include @@ -24,7 +25,6 @@ #include #include -#include "linux-perf-events.h" #ifdef __linux__ #include #endif diff --git a/benchmark/dom/statisticalmodel.cpp b/benchmark/dom/statisticalmodel.cpp deleted file mode 100644 index 2d7c9a7cc..000000000 --- a/benchmark/dom/statisticalmodel.cpp +++ /dev/null @@ -1,207 +0,0 @@ -#include -#include -#include "simdjson.h" -#ifdef __linux__ -#include "linux-perf-events.h" -#endif - -size_t count_nonasciibytes(const uint8_t *input, size_t length) { - size_t count = 0; - for (size_t i = 0; i < length; i++) { - count += input[i] >> 7; - } - return count; -} - -size_t count_backslash(const uint8_t *input, size_t length) { - size_t count = 0; - for (size_t i = 0; i < length; i++) { - count += (input[i] == '\\') ? 1 : 0; - } - return count; -} - -struct stat_s { - size_t integer_count; - size_t float_count; - size_t string_count; - size_t backslash_count; - size_t non_ascii_byte_count; - size_t object_count; - size_t array_count; - size_t null_count; - size_t true_count; - size_t false_count; - size_t byte_count; - size_t structural_indexes_count; - bool valid; -}; - -using stat_t = struct stat_s; - - - -simdjson_inline void simdjson_process_atom(stat_t &s, - simdjson::dom::element element) { - if (element.is()) { - s.integer_count++; - } else if(element.is()) { - s.string_count++; - } else if(element.is()) { - s.float_count++; - } else if (element.is()) { - bool v; - simdjson::error_code error; - if ((error = element.get(v))) { std::cerr << error << std::endl; abort(); } - if (v) { - s.true_count++; - } else { - s.false_count++; - } - } else if (element.is_null()) { - s.null_count++; - } -} - -void simdjson_recurse(stat_t &s, simdjson::dom::element element) { - simdjson::error_code error; - if (element.is()) { - s.array_count++; - simdjson::dom::array array; - if ((error = element.get(array))) { std::cerr << error << std::endl; abort(); } - for (auto child : array) { - if (child.is() || child.is()) { - simdjson_recurse(s, child); - } else { - simdjson_process_atom(s, child); - } - } - } else if (element.is()) { - s.object_count++; - simdjson::dom::object object; - if ((error = element.get(object))) { std::cerr << error << std::endl; abort(); } - for (auto field : object) { - s.string_count++; // for key - if (field.value.is() || field.value.is()) { - simdjson_recurse(s, field.value); - } else { - simdjson_process_atom(s, field.value); - } - } - } else { - simdjson_process_atom(s, element); - } -} - -stat_t simdjson_compute_stats(const simdjson::padded_string &p) { - stat_t answer{}; - simdjson::dom::parser parser; - simdjson::dom::element doc; - auto error = parser.parse(p).get(doc); - if (error) { - answer.valid = false; - return answer; - } - answer.valid = true; - answer.backslash_count = - count_backslash(reinterpret_cast(p.data()), p.size()); - answer.non_ascii_byte_count = count_nonasciibytes( - reinterpret_cast(p.data()), p.size()); - answer.byte_count = p.size(); - answer.structural_indexes_count = parser.implementation->n_structural_indexes; - simdjson_recurse(answer, doc); - return answer; -} - -int main(int argc, char *argv[]) { -#ifndef _MSC_VER - int c; - while ((c = getopt(argc, argv, "")) != -1) { - switch (c) { - - default: - abort(); - } - } -#else - int optind = 1; -#endif - if (optind >= argc) { - std::cerr << "Reads json, prints stats. " << std::endl; - std::cerr << "Usage: " << argv[0] << " " << std::endl; - - exit(1); - } - const char *filename = argv[optind]; - if (optind + 1 < argc) { - std::cerr << "warning: ignoring everything after " << argv[optind + 1] - << std::endl; - } - simdjson::padded_string p; - auto error = simdjson::padded_string::load(filename).get(p); - if (error) { - std::cerr << "Could not load the file " << filename << std::endl; - return EXIT_FAILURE; - } - stat_t s = simdjson_compute_stats(p); - if (!s.valid) { - std::cerr << "not a valid JSON" << std::endl; - return EXIT_FAILURE; - } - - printf("# integer_count float_count string_count backslash_count " - "non_ascii_byte_count object_count array_count null_count true_count " - "false_count byte_count structural_indexes_count "); -#ifdef __linux__ - printf(" stage1_cycle_count stage1_instruction_count stage2_cycle_count " - " stage2_instruction_count stage3_cycle_count " - "stage3_instruction_count "); -#else - printf("(you are not under linux, so perf counters are disaabled)"); -#endif - printf("\n"); - printf("%zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu ", s.integer_count, - s.float_count, s.string_count, s.backslash_count, - s.non_ascii_byte_count, s.object_count, s.array_count, s.null_count, - s.true_count, s.false_count, s.byte_count, s.structural_indexes_count); -#ifdef __linux__ - simdjson::dom::parser parser; - simdjson::error_code alloc_error = parser.allocate(p.size()); - if (alloc_error) { - std::cerr << alloc_error << std::endl; - return EXIT_FAILURE; - } - const uint32_t iterations = p.size() < 1 * 1000 * 1000 ? 1000 : 50; - std::vector evts; - evts.push_back(PERF_COUNT_HW_CPU_CYCLES); - evts.push_back(PERF_COUNT_HW_INSTRUCTIONS); - LinuxEvents unified(evts); - unsigned long cy1 = 0, cy2 = 0; - unsigned long cl1 = 0, cl2 = 0; - std::vector results; - results.resize(evts.size()); - for (uint32_t i = 0; i < iterations; i++) { - unified.start(); - // The default template is simdjson::architecture::NATIVE. - bool isok = (parser.implementation->stage1((const uint8_t *)p.data(), p.size(), simdjson::stage1_mode::regular) == simdjson::SUCCESS); - unified.end(results); - - cy1 += results[0]; - cl1 += results[1]; - - unified.start(); - isok = isok && (parser.implementation->stage2(parser.doc) == simdjson::SUCCESS); - unified.end(results); - - cy2 += results[0]; - cl2 += results[1]; - if (!isok) { - std::cerr << "failure?" << std::endl; - } - } - printf("%f %f %f %f ", static_cast(cy1) / static_cast(iterations), static_cast(cl1) / static_cast(iterations), - static_cast(cy2) / static_cast(iterations), static_cast(cl2) / static_cast(iterations)); -#endif // __linux__ - printf("\n"); - return EXIT_SUCCESS; -} diff --git a/benchmark/event_counter.h b/benchmark/event_counter.h deleted file mode 100644 index d4fef0659..000000000 --- a/benchmark/event_counter.h +++ /dev/null @@ -1,201 +0,0 @@ -#ifndef __EVENT_COUNTER_H -#define __EVENT_COUNTER_H - -#ifndef SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS -#ifdef __aarch64__ -// on ARM, we use just cycles and instructions -#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 1 -#else -// elsewhere, we try to use four counters. -#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 0 -#endif -#endif -#include -#include -#ifndef _MSC_VER -#include -#endif -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __linux__ -#include "linux-perf-events.h" -#include -#endif - -#if __APPLE__ && __aarch64__ -#include "apple/apple_arm_events.h" -#endif - -#include "simdjson.h" - -using std::string; -using std::vector; -using std::chrono::steady_clock; -using std::chrono::time_point; -using std::chrono::duration; - -struct event_count { - duration elapsed; - vector event_counts; - event_count() : elapsed(0), event_counts{0,0,0,0,0} {} - event_count(const duration _elapsed, const vector _event_counts) : elapsed(_elapsed), event_counts(_event_counts) {} - event_count(const event_count& other): elapsed(other.elapsed), event_counts(other.event_counts) { } - - // The types of counters (so we can read the getter more easily) - #if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - enum event_counter_types { - CPU_CYCLES, - INSTRUCTIONS - }; - #else - enum event_counter_types { - CPU_CYCLES, - INSTRUCTIONS, - BRANCH_MISSES, - CACHE_REFERENCES, - CACHE_MISSES - }; - #endif - double elapsed_sec() const { return duration(elapsed).count(); } - double elapsed_ns() const { return duration(elapsed).count(); } - double cycles() const { return static_cast(event_counts[CPU_CYCLES]); } - double instructions() const { return static_cast(event_counts[INSTRUCTIONS]); } -#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - double branch_misses() const { return static_cast(event_counts[BRANCH_MISSES]); } - double cache_references() const { return static_cast(event_counts[CACHE_REFERENCES]); } - double cache_misses() const { return static_cast(event_counts[CACHE_MISSES]); } -#endif - event_count& operator=(const event_count& other) { - this->elapsed = other.elapsed; - this->event_counts = other.event_counts; - return *this; - } - event_count operator+(const event_count& other) const { - return event_count(elapsed+other.elapsed, { - event_counts[0]+other.event_counts[0], - event_counts[1]+other.event_counts[1], - event_counts[2]+other.event_counts[2], - event_counts[3]+other.event_counts[3], - event_counts[4]+other.event_counts[4], - }); - } - - void operator+=(const event_count& other) { - *this = *this + other; - } -}; - -struct event_aggregate { - int iterations = 0; - event_count total{}; - event_count best{}; - event_count worst{}; - - event_aggregate() {} - - void operator<<(const event_count& other) { - if (iterations == 0 || other.elapsed < best.elapsed) { - best = other; - } - if (iterations == 0 || other.elapsed > worst.elapsed) { - worst = other; - } - iterations++; - total += other; - } - - double elapsed_sec() const { return total.elapsed_sec() / iterations; } - double total_elapsed_ns() const { return total.elapsed_ns(); } - double elapsed_ns() const { return total.elapsed_ns() / iterations; } - double cycles() const { return total.cycles() / iterations; } - double instructions() const { return total.instructions() / iterations; } -#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - double branch_misses() const { return total.branch_misses() / iterations; } - double cache_references() const { return total.cache_references() / iterations; } - double cache_misses() const { return total.cache_misses() / iterations; } -#endif -}; - -struct event_collector { - event_count count{}; - time_point start_clock{}; - -#if defined(__linux__) - LinuxEvents linux_events; - event_collector() : linux_events(vector{ - #if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS - PERF_COUNT_HW_CPU_CYCLES, - PERF_COUNT_HW_INSTRUCTIONS, - #else - PERF_COUNT_HW_CPU_CYCLES, - PERF_COUNT_HW_INSTRUCTIONS, - PERF_COUNT_HW_BRANCH_MISSES, - PERF_COUNT_HW_CACHE_REFERENCES, - PERF_COUNT_HW_CACHE_MISSES - #endif - }) {} - bool has_events() { - return linux_events.is_working(); - } -#elif __APPLE__ && __aarch64__ - AppleEvents apple_events; - performance_counters diff; - event_collector() : diff(0) { - apple_events.setup_performance_counters(); - } - bool has_events() { - return apple_events.setup_performance_counters(); - } -#else - event_collector() {} - bool has_events() { - return false; - } -#endif - - simdjson_inline void start() { -#if defined(__linux) - linux_events.start(); -#elif __APPLE__ && __aarch64__ - if(has_events()) { diff = apple_events.get_counters(); } -#endif - start_clock = steady_clock::now(); - } - simdjson_inline event_count& end() { - time_point end_clock = steady_clock::now(); -#if defined(__linux) - linux_events.end(count.event_counts); -#elif __APPLE__ && __aarch64__ - if(has_events()) { - performance_counters end = apple_events.get_counters(); - diff = end - diff; - } - count.event_counts[0] = diff.cycles; - count.event_counts[1] = diff.instructions; - count.event_counts[2] = diff.missed_branches; - count.event_counts[3] = 0; - count.event_counts[4] = 0; -#endif - count.elapsed = end_clock - start_clock; - return count; - } -}; - -#endif diff --git a/benchmark/from/benchmark_helpers.h b/benchmark/from/benchmark_helpers.h index b6273b1d1..8706ffc56 100644 --- a/benchmark/from/benchmark_helpers.h +++ b/benchmark/from/benchmark_helpers.h @@ -1,7 +1,8 @@ #ifndef BENCHMARK_HELPERS_H #define BENCHMARK_HELPERS_H -#include "event_counter.h" +#include +using namespace counters; #include event_collector collector; diff --git a/benchmark/json_benchmark/run_json_benchmark.h b/benchmark/json_benchmark/run_json_benchmark.h index 29917697c..6a12767a2 100644 --- a/benchmark/json_benchmark/run_json_benchmark.h +++ b/benchmark/json_benchmark/run_json_benchmark.h @@ -1,7 +1,8 @@ #pragma once #include "simdjson.h" -#include "event_counter.h" +#include +using namespace counters; #include namespace json_benchmark { @@ -58,11 +59,7 @@ template static void run_json_benchmark(benchmark::State if (collector.has_events()) { state.counters["instructions"] = events.instructions(); state.counters["cycles"] = events.cycles(); -#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS state.counters["branch_miss"] = events.branch_misses(); - state.counters["cache_miss"] = events.cache_misses(); - state.counters["cache_ref"] = events.cache_references(); -#endif state.counters["instructions_per_byte"] = events.instructions() / double(bench.bytes_per_iteration()); state.counters["instructions_per_cycle"] = events.instructions() / events.cycles(); state.counters["cycles_per_byte"] = events.cycles() / double(bench.bytes_per_iteration()); @@ -70,11 +67,7 @@ template static void run_json_benchmark(benchmark::State state.counters["best_instructions"] = events.best.instructions(); state.counters["best_cycles"] = events.best.cycles(); -#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS state.counters["best_branch_miss"] = events.best.branch_misses(); - state.counters["best_cache_miss"] = events.best.cache_misses(); - state.counters["best_cache_ref"] = events.best.cache_references(); -#endif state.counters["best_instructions_per_byte"] = events.best.instructions() / double(bench.bytes_per_iteration()); state.counters["best_instructions_per_cycle"] = events.best.instructions() / events.best.cycles(); @@ -95,11 +88,7 @@ template static void run_json_benchmark(benchmark::State if (collector.has_events()) { label << " instructions=" << setw(12) << uint64_t(events.best.instructions()) << setw(0); label << " cycles=" << setw(12) << uint64_t(events.best.cycles()) << setw(0); -#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS label << " branch_miss=" << setw(8) << uint64_t(events.best.branch_misses()) << setw(0); - label << " cache_miss=" << setw(8) << uint64_t(events.best.cache_misses()) << setw(0); - label << " cache_ref=" << setw(10) << uint64_t(events.best.cache_references()) << setw(0); -#endif } label << " items=" << setw(10) << bench.items_per_iteration() << setw(0); diff --git a/benchmark/large_amazon_cellphones/large_amazon_cellphones.h b/benchmark/large_amazon_cellphones/large_amazon_cellphones.h index 8a3b7119a..ec9aadefe 100644 --- a/benchmark/large_amazon_cellphones/large_amazon_cellphones.h +++ b/benchmark/large_amazon_cellphones/large_amazon_cellphones.h @@ -1,6 +1,7 @@ #pragma once #include "json_benchmark/string_runner.h" +#include #include #include diff --git a/benchmark/linux/linux-perf-events.h b/benchmark/linux/linux-perf-events.h deleted file mode 100644 index 4fe872d2c..000000000 --- a/benchmark/linux/linux-perf-events.h +++ /dev/null @@ -1,105 +0,0 @@ -#pragma once -#ifdef __linux__ - -#include // for __NR_perf_event_open -#include // for perf event constants -#include // for ioctl -#include // for syscall - -#include // for errno -#include // for memset -#include - -#include -#include - -template class LinuxEvents { - int fd; - bool working; - perf_event_attr attribs{}; - size_t num_events{}; - std::vector temp_result_vec{}; - std::vector ids{}; - -public: - explicit LinuxEvents(std::vector config_vec) : fd(0), working(true) { - memset(&attribs, 0, sizeof(attribs)); - attribs.type = TYPE; - attribs.size = sizeof(attribs); - attribs.disabled = 1; - attribs.exclude_kernel = 1; - attribs.exclude_hv = 1; - - attribs.sample_period = 0; - attribs.read_format = PERF_FORMAT_GROUP | PERF_FORMAT_ID; - const int pid = 0; // the current process - const int cpu = -1; // all CPUs - const unsigned long flags = 0; - - int group = -1; // no group - num_events = config_vec.size(); - ids.resize(config_vec.size()); - uint32_t i = 0; - for (auto config : config_vec) { - attribs.config = config; - int _fd = static_cast(syscall(__NR_perf_event_open, &attribs, pid, cpu, group, flags)); - if (_fd == -1) { - report_error("perf_event_open"); - } - ioctl(_fd, PERF_EVENT_IOC_ID, &ids[i++]); - if (group == -1) { - group = _fd; - fd = _fd; - } - } - - temp_result_vec.resize(num_events * 2 + 1); - } - - ~LinuxEvents() { if (fd != -1) { close(fd); } } - - inline void start() { - if (fd != -1) { - if (ioctl(fd, PERF_EVENT_IOC_RESET, PERF_IOC_FLAG_GROUP) == -1) { - report_error("ioctl(PERF_EVENT_IOC_RESET)"); - } - - if (ioctl(fd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP) == -1) { - report_error("ioctl(PERF_EVENT_IOC_ENABLE)"); - } - } - } - - inline void end(std::vector &results) { - if (fd != -1) { - if (ioctl(fd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP) == -1) { - report_error("ioctl(PERF_EVENT_IOC_DISABLE)"); - } - - if (read(fd, temp_result_vec.data(), temp_result_vec.size() * 8) == -1) { - report_error("read"); - } - } - // our actual results are in slots 1,3,5, ... of this structure - for (uint32_t i = 1; i < temp_result_vec.size(); i += 2) { - results[i / 2] = temp_result_vec[i]; - } - for (uint32_t i = 2; i < temp_result_vec.size(); i += 2) { - if(ids[i/2-1] != temp_result_vec[i]) { - report_error("event mismatch"); - } - } - - - } - - bool is_working() { - return working; - } - -private: - void report_error(const std::string &) { - working = false; - } -}; -#endif \ No newline at end of file diff --git a/benchmark/static_reflect/benchmark_utils/benchmark_helper.h b/benchmark/static_reflect/benchmark_utils/benchmark_helper.h index ee1fbf4fe..95a32172c 100644 --- a/benchmark/static_reflect/benchmark_utils/benchmark_helper.h +++ b/benchmark/static_reflect/benchmark_utils/benchmark_helper.h @@ -1,6 +1,7 @@ #ifndef BENCHMARK_HELPER_HPP #define BENCHMARK_HELPER_HPP -#include "event_counter.h" +#include +using namespace counters; #include inline event_collector &get_collector() { diff --git a/dependencies/CMakeLists.txt b/dependencies/CMakeLists.txt index ccb3ef637..93464f9b2 100644 --- a/dependencies/CMakeLists.txt +++ b/dependencies/CMakeLists.txt @@ -20,6 +20,14 @@ if(SIMDJSON_GOOGLE_BENCHMARKS) ) endif() +CPMAddPackage( + NAME counters + URL https://github.com/lemire/counters/archive/refs/tags/v3.1.0.zip + OPTIONS + "COUNTERS_BUILD_TESTS OFF" + "COUNTERS_INSTALL OFF" +) + CPMAddPackage( NAME simdjson-data URL https://github.com/simdjson/simdjson-data/archive/351949906abde446f0314bf79606fb5d884f5be7.zip