From dfe5d7eb3b76b55feb70e3c36fddeb964266ac85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20L=C3=BCke?= Date: Wed, 25 Dec 2024 03:19:29 +0100 Subject: [PATCH] mana: unfinished offset ptr stuff --- src/tools/mana_auto_replay.cc | 390 ++++++++++++++++++++++++++++++---- src/tools/mana_lib.hpp | 68 ++++++ src/tools/mana_replay_api.h | 94 ++++---- 3 files changed, 473 insertions(+), 79 deletions(-) create mode 100644 src/tools/mana_lib.hpp diff --git a/src/tools/mana_auto_replay.cc b/src/tools/mana_auto_replay.cc index af8d516..5eff86a 100644 --- a/src/tools/mana_auto_replay.cc +++ b/src/tools/mana_auto_replay.cc @@ -37,8 +37,10 @@ #include #include #include +#include #include "mana_replay_api.h" #include "internal/mana_arena.h" +#include "tools/mana_lib.hpp" CMRC_DECLARE(mnode::resources); @@ -131,25 +133,357 @@ std::optional make_parser_context(const mvlc::CrateConfig &crateC } } +MANA_DEFINE_PLUGIN_INIT(mana_default_init) { return nullptr; } +MANA_DEFINE_PLUGIN_SHUTDOWN(mana_default_shutdown) {} +MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_default_begin_run) {} +MANA_DEFINE_PLUGIN_END_RUN(mana_default_end_run) {} +MANA_DEFINE_PLUGIN_EVENT_DATA(mana_default_process_event) {} +MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_default_system_event) {} + +static mana_plugin_t DefaultPlugin = {.init = mana_default_init, + .shutdown = mana_default_shutdown, + .begin_run = mana_default_begin_run, + .end_run = mana_default_end_run, + .process_event = mana_default_process_event}; + +struct DataSource +{ + mvlc::util::DataFilter filter; + mvlc::util::CacheEntry fAddress; + mvlc::util::CacheEntry fValue; + mvlc::util::span dest; +}; + struct AnalysisContext { std::unique_ptr arena; - run_descriptor_t *runDescriptor = nullptr; + mana_run_descriptor_t *runDescriptor = nullptr; + mvlc::util::span eventStorages; + std::vector>> dataSources; // eventIndex, moduleIndex, sourceIndex + mana_plugin_t outputPlugin = DefaultPlugin; + void *outputPluginContext = nullptr; + + size_t runDescriptorBytes = 0; + size_t eventStoragesBytes = 0; + + void process_module_data(const mvlc::readout_parser::DataBlock &data, DataSource &source) + { + std::fill(std::begin(source.dest), std::end(source.dest), mnode::make_quiet_nan()); + for (const u32 *word = data.data, *end = data.data + data.size; word < end; ++word) + { + if (mvlc::util::matches(source.filter, *word)) + { + u32 address = mvlc::util::extract(source.fAddress, *word); + u32 value = mvlc::util::extract(source.fValue, *word); + assert(address < source.dest.size()); + source.dest[address] = value; + } + } + } + + void process_module_data(const mvlc::readout_parser::DataBlock &data, std::vector &sources) + { + for (auto &source: sources) + { + process_module_data(data, source); + } + } + + void process_module_data(const mvlc::readout_parser::ModuleData &moduleData, + std::vector &sources) + { + mvlc::readout_parser::DataBlock data = moduleData.hasDynamic ? dynamic_span(moduleData) + : prefix_span(moduleData); + process_module_data(data, sources); + + } + + void process_event_data(int crateIndex, int eventIndex, + const mvlc::readout_parser::ModuleData *moduleDataList, + unsigned moduleCount) + { + std::cout << fmt::format( + "ReadoutEvent: ctx={}, crateId={}, eventIndex={}, moduleCount={}\n", fmt::ptr(this), + crateIndex, eventIndex, moduleCount); + + auto &eventSources = dataSources.at(eventIndex); + + for (unsigned mi=0; mi make_mana(const mvlc::CrateConfig &crateConfig, +std::map +make_module_info_by_type(const nlohmann::json &jModuleDataSources) +{ + // module type name -> module info json + std::map moduleInfoByType; + for (const auto &mod_: jModuleDataSources["modules"]) + moduleInfoByType[mod_["type_name"]] = mod_; + return moduleInfoByType; +} + +std::vector> +match_modules(const mvlc::CrateConfig &crateConfig, + const std::map &moduleInfoByType) +{ + std::vector> moduleDataSources; + for (const auto &event: crateConfig.stacks) + { + std::vector eventModuleDataSources; + for (const auto &module_: event.getGroups()) + { + nlohmann::json jModule; + // match the meta data module type name against the concrete module name + for (const auto &[type_, info]: moduleInfoByType) + { + if (module_.name.find(type_) != std::string::npos) + { + spdlog::info("match: type={}, name={}", type_, module_.name); + jModule = info; + break; + } + } + + if (jModule.empty()) + { + spdlog::warn("No module info found for module name '{}'", module_.name); + } + + eventModuleDataSources.emplace_back(jModule); + } + moduleDataSources.emplace_back(eventModuleDataSources); + } + + return moduleDataSources; +} + +struct DataSourceInfo +{ + const std::string name; + const std::string filterString; + mvlc::util::FilterWithCaches filter; + + DataSourceInfo(const std::string &name_, const std::string &filterString_) + : name(name_) + , filterString(filterString_) + , filter(mvlc::util::make_filter_with_caches(filterString)) + { + } +}; + +std::vector> +make_event_ds_info(const mvlc::CrateConfig &crateConfig, + const std::vector> &moduleDataSources) +{ + std::vector> eventDsInfo; + + for (size_t eventIndex = 0; eventIndex < crateConfig.stacks.size(); ++eventIndex) + { + const auto &dataSources = moduleDataSources.at(eventIndex); + const auto &event = crateConfig.stacks.at(eventIndex); + const auto &readouts = event.getGroups(); + std::vector dsInfo; + + for (size_t moduleIndex = 0; moduleIndex < event.getGroups().size(); ++moduleIndex) + { + const auto &readout = readouts.at(moduleIndex); + const auto &ds = dataSources.at(moduleIndex); + + spdlog::info("readout.name={}, ds={}", readout.name, ds.dump()); + if (ds.contains("data_sources")) + { + for (const auto &filter: ds["data_sources"]) + { + auto name = + fmt::format("{}.{}.{}", event.getName(), readout.name, filter["name"]); + dsInfo.emplace_back(DataSourceInfo(name, filter["filter"])); + } + } + } + eventDsInfo.emplace_back(dsInfo); + } + + return eventDsInfo; +} + +template +T *push_offset_array(mana::Arena &arena, mana_offset_array_t &dest, size_t size) +{ + T *ptr = arena.push_t(size); + mana::set(dest.ptr, mana_custom, ptr); + dest.size_bytes = size * sizeof(T); + return ptr; +} + +template +T *push_typed_offset_array(mana::Arena &arena, mana_offset_array_t &dest, size_t size) +{ + T *ptr = arena.push_t(size); + mana::set(dest.ptr, ptr); + dest.size_bytes = size * sizeof(T); + return ptr; +} + +template size_t element_count(mana_offset_array_t &array) +{ + return array.size_bytes / sizeof(T); +} + +template mvlc::util::span get_span(mana_offset_array_t &array) +{ + auto ptr = reinterpret_cast(mana::get(array.ptr)); + auto size = element_count(array); + return {ptr, size}; +} + +std::pair +make_run_descriptor(mana::Arena &arena, const std::string &runName, + const std::vector> &eventDsInfo) +{ + auto rd = arena.push_t(); + mana::set(rd->name, arena.push_cstr(runName)); + auto eds = push_offset_array(arena, rd->events, eventDsInfo.size()); + rd->event_count = eventDsInfo.size(); + + for (const auto &eventDataSources: eventDsInfo) + { + auto ed = eds++; + auto ads = push_offset_array(arena, ed->data_arrays, + eventDataSources.size()); + ed->data_array_count = eventDataSources.size(); + + for (const auto &ds: eventDataSources) + { + auto ad = ads++; + mana::set(ad->name, arena.push_cstr(ds.name)); + ad->data_type = mana_float; + ad->size = 1; + if (auto c = get_cache_entry(ds.filter, 'A'); c) + ad->size = 1u << c->extractBits; + if (auto c = get_cache_entry(ds.filter, 'D'); c) + ad->bits = c->extractBits; + } + } + + return {rd, arena.cur_end() - reinterpret_cast(rd)}; +} + +std::pair, size_t> +make_event_storages(mana::Arena &arena, const std::vector> &eventDsInfo) +{ + auto eds = arena.push_t(eventDsInfo.size()); + + for (size_t eventIndex = 0; eventIndex < eventDsInfo.size(); ++eventIndex) + { + auto &ed = eds[eventIndex]; + auto &dsInfo = eventDsInfo[eventIndex]; + auto das = push_offset_array(arena, ed.data_arrays, dsInfo.size()); + + for (const auto &ds: dsInfo) + { + auto da = das++; + size_t size = 1; + if (auto c = get_cache_entry(ds.filter, 'A'); c) + size = 1u << c->extractBits; + push_typed_offset_array(arena, *da, size); + } + } + + return {{eds, eventDsInfo.size()}, arena.cur_end() - reinterpret_cast(eds)}; +} + +void dump(mana_run_descriptor_t *rd) +{ + auto eds = get_span(rd->events); + + spdlog::info("mana_run_descriptor @ {}, name={}, event_count={}", fmt::ptr(rd), + mana::get(rd->name), eds.size()); + + for (size_t eventIndex = 0; eventIndex < rd->event_count; ++eventIndex) + { + auto &ed = eds[eventIndex]; + spdlog::info(" event[{}]:", eventIndex); + + auto ads = get_span(ed.data_arrays); + + for (auto &ad: ads) + { + spdlog::info(" array: name={}, data_type={}, size={}, bits={}", + mana::get(ad.name), static_cast(ad.data_type), ad.size, + ad.bits); + } + } +} + +std::optional make_mana(const std::string runName, + const mvlc::CrateConfig &crateConfig, const nlohmann::json &jModuleDataSources) { try { - std::map moduleInfoByType; - for (const auto &mod_: jModuleDataSources["modules"]) - moduleInfoByType[mod_["type_name"]] = mod_; + // module type name -> module info json + auto moduleInfoByType = make_module_info_by_type(jModuleDataSources); AnalysisContext ctx; ctx.arena = std::make_unique(); + // auto arena = ctx.arena.get(); + + // eventIndex -> moduleIndex -> module info json + auto moduleDataSources = match_modules(crateConfig, moduleInfoByType); + auto eventDsInfo = make_event_ds_info(crateConfig, moduleDataSources); + auto arena = ctx.arena.get(); + auto [runDescriptor, runDescriptorSize] = make_run_descriptor(*arena, runName, eventDsInfo); + + spdlog::info( + "runDescriptor @ {}, size={}, allocations={}, capacity={}, pad_waste={}, used_size={}", + fmt::ptr(runDescriptor), runDescriptorSize, arena->allocations(), arena->capacity(), + arena->pad_waste(), arena->used()); + + if (runDescriptor) + dump(runDescriptor); + + auto [eventStorages, eventStoragesSize] = make_event_storages(*arena, eventDsInfo); + + spdlog::info( + "eventStorages @ {}, size={}, allocations={}, capacity={}, pad_waste={}, used_size={}", + fmt::ptr(&eventStorages.front()), eventStoragesSize, arena->allocations(), + arena->capacity(), arena->pad_waste(), arena->used()); + + for (size_t eventIndex = 0; eventIndex < eventStorages.size(); ++eventIndex) + { + auto &es = eventStorages[eventIndex]; + auto arrays = get_span(es.data_arrays); + spdlog::info("eventStorage: event={}, arrays={}", eventIndex, arrays.size()); + for (auto &array: arrays) + { + spdlog::info(" array @ {}, size_bytes={}, size={}", + fmt::ptr(mana::get(array.ptr)), array.size_bytes, + element_count(array)); + } + } + + ctx.runDescriptor = runDescriptor; + ctx.eventStorages = eventStorages; + ctx.runDescriptorBytes = runDescriptorSize; + ctx.eventStoragesBytes = eventStoragesSize; + return ctx; } catch (const std::exception &e) @@ -159,22 +493,6 @@ std::optional make_mana(const mvlc::CrateConfig &crateConfig, } } -void event_data_callback(AnalysisContext *ctx, int crateIndex, int eventIndex, - const mvlc::readout_parser::ModuleData *moduleDataList, - unsigned moduleCount) -{ - assert(moduleDataList); - std::cout << fmt::format("ReadoutEvent: ctx={}, crateId={}, eventIndex={}, moduleCount={}\n", - fmt::ptr(ctx), crateIndex, eventIndex, moduleCount); -} - -void system_event_callback(AnalysisContext *ctx, int crateIndex, const u32 *header, u32 size) -{ - assert(header); - std::cout << fmt::format("SystemEvent: ctx={}, crateId={}, size={}\n", fmt::ptr(ctx), - crateIndex, size); -} - size_t process_one_buffer(size_t bufferNumber, ListfileContext &listfileContext, ParserContext &parserContext, AnalysisContext &analysisContext) { @@ -211,23 +529,21 @@ int main(int argc, char *argv[]) if (!listfileContext) return 1; - auto analysisContext = make_mana(listfileContext->crateConfig, jModuleDataSources); + auto analysisContext = make_mana(filename, listfileContext->crateConfig, jModuleDataSources); if (!analysisContext) return 1; - return 42; - auto event_data = [](void *ctx, int crateIndex, int eventIndex, const mvlc::readout_parser::ModuleData *moduleDataList, unsigned moduleCount) { - event_data_callback(static_cast(ctx), crateIndex, eventIndex, - moduleDataList, moduleCount); + reinterpret_cast(ctx)->process_event_data(crateIndex, eventIndex, + moduleDataList, moduleCount); }; auto system_event = [](void *ctx, int crateIndex, const u32 *header, u32 size) - { system_event_callback(static_cast(ctx), crateIndex, header, size); }; + { reinterpret_cast(ctx)->process_system_event(crateIndex, header, size); }; auto parserContext = make_parser_context(listfileContext->crateConfig, {event_data, system_event}); @@ -241,14 +557,16 @@ int main(int argc, char *argv[]) const std::chrono::milliseconds ReportInterval(500); mvlc::util::Stopwatch sw; - auto report = - [](const mvlc::util::Stopwatch sw, size_t bufferNumber, size_t totalBytesProcessed) + auto report = [&] { - auto s = sw.get_elapsed().count() / (1000.0 * 1000.0); - auto bytesPerSecond = totalBytesProcessed / s; - auto MiBPerSecond = bytesPerSecond / (1u << 20); - std::cout << fmt::format("Processed {} buffers, {} bytes. t={} s, rate={} MiB/s\n", - bufferNumber, totalBytesProcessed, s, MiBPerSecond); + [](const mvlc::util::Stopwatch sw, size_t bufferNumber, size_t totalBytesProcessed) + { + auto s = sw.get_elapsed().count() / (1000.0 * 1000.0); + auto bytesPerSecond = totalBytesProcessed / s; + auto MiBPerSecond = bytesPerSecond / (1u << 20); + std::cout << fmt::format("Processed {} buffers, {} bytes. t={} s, rate={} MiB/s\n", + bufferNumber, totalBytesProcessed, s, MiBPerSecond); + }(sw, bufferNumber, totalBytesProcessed); }; do @@ -260,13 +578,13 @@ int main(int argc, char *argv[]) if (auto elapsed = sw.get_interval(); elapsed >= ReportInterval) { - report(sw, bufferNumber, totalBytesProcessed); + report(); sw.interval(); } } while (bytesProcessed > 0); - report(sw, bufferNumber, totalBytesProcessed); + report(); return 0; } diff --git a/src/tools/mana_lib.hpp b/src/tools/mana_lib.hpp new file mode 100644 index 0000000..0debee6 --- /dev/null +++ b/src/tools/mana_lib.hpp @@ -0,0 +1,68 @@ +#ifndef AAB5E4D2_A05B_4F2F_B76A_406A5A569D55 +#define AAB5E4D2_A05B_4F2F_B76A_406A5A569D55 + +#include +#include +#include "mana_replay_api.h" + +namespace mesytec::mnode::mana +{ + +inline bool is_null(const mana_offset_ptr_t &ptr) { return ptr.offset == 1; } +inline void set_raw(mana_offset_ptr_t &ptr, void *p) +{ + assert(p != reinterpret_cast(&ptr) + 1); + + if (p) + { + ptr.offset = reinterpret_cast(p) - reinterpret_cast(&ptr); + spdlog::info("set_raw: &ptr={}, p={}, offset={}", fmt::ptr(&ptr), fmt::ptr(p), ptr.offset); + } + else + { + ptr.offset = 1; + } +} + +inline void *get(mana_offset_ptr_t &ptr) +{ + if (is_null(ptr)) + return nullptr; + + return reinterpret_cast(&ptr) + ptr.offset; +} + +template T *get(mana_offset_ptr_t &ptr); // to catch unsupported types + +template T *get_(mana_offset_ptr_t &ptr, mana_data_type_t expected) +{ + if (ptr.data_type != expected) + return nullptr; + + return reinterpret_cast(get(ptr)); +} + +template <> u32 *get(mana_offset_ptr_t &ptr) { return get_(ptr, mana_uint32); } +template <> u64 *get(mana_offset_ptr_t &ptr) { return get_(ptr, mana_uint64); } +template <> s8 *get(mana_offset_ptr_t &ptr) { return get_(ptr, mana_sint8); } +template <> char *get(mana_offset_ptr_t &ptr) { return get_(ptr, mana_sint8); } +template <> float *get(mana_offset_ptr_t &ptr) { return get_(ptr, mana_float); } +template <> double *get(mana_offset_ptr_t &ptr) { return get_(ptr, mana_double); } + +inline void set(mana_offset_ptr_t &ptr, mana_data_type_t data_type, void *p) +{ + ptr.data_type = data_type; + set_raw(ptr, p); +} + +inline void set(mana_offset_ptr_t &ptr, std::nullptr_t) { set(ptr, mana_uint32, nullptr); } +inline void set(mana_offset_ptr_t &ptr, u32 *p) { set(ptr, mana_uint32, p); } +inline void set(mana_offset_ptr_t &ptr, u64 *p) { set(ptr, mana_uint64, p); } +inline void set(mana_offset_ptr_t &ptr, s8 *p) { set(ptr, mana_sint8, p); } +inline void set(mana_offset_ptr_t &ptr, char *p) { set(ptr, mana_sint8, p); } +inline void set(mana_offset_ptr_t &ptr, float *p) { set(ptr, mana_float, p); } +inline void set(mana_offset_ptr_t &ptr, double *p) { set(ptr, mana_double, p); } + +} + +#endif /* AAB5E4D2_A05B_4F2F_B76A_406A5A569D55 */ diff --git a/src/tools/mana_replay_api.h b/src/tools/mana_replay_api.h index fab9fec..5026a71 100644 --- a/src/tools/mana_replay_api.h +++ b/src/tools/mana_replay_api.h @@ -20,59 +20,53 @@ extern "C" typedef enum { - dt_uint32, - dt_float, -} data_type_t; + mana_custom, + mana_uint8, + mana_sint8, + mana_uint16, + mana_uint32, + mana_uint64, + mana_float, + mana_double, +} mana_data_type_t; typedef struct { - const char *array_name; - data_type_t data_type; - size_t array_size; - size_t valid_bits; // belongs into data_type_t -} array_descriptor_t; + mana_data_type_t data_type; + ptrdiff_t offset; +} mana_offset_ptr_t; typedef struct { - const char *module_name; - const char *module_type; - array_descriptor_t *data_sources; - size_t data_source_count; -} module_descriptor_t; + mana_offset_ptr_t ptr; + size_t size_bytes; +} mana_offset_array_t; typedef struct { - const char *event_name; - module_descriptor_t *modules; - size_t module_count; -} event_descriptor_t; - -typedef struct -{ - const char *listfile_name; - event_descriptor_t *events; - size_t event_count; -} run_descriptor_t; - -// data views - daq data - -typedef struct -{ - float *data; + mana_offset_ptr_t name; // mana_uint8 + mana_data_type_t data_type; size_t size; -} array_t; + size_t bits; +} mana_array_descriptor_t; typedef struct { - array_t *data_arrays; - size_t data_array_count; -} module_data_t; + mana_offset_array_t data_arrays; // mana_custom: mana_array_descriptor_t + size_t data_array_count; +} mana_event_descriptor_t; typedef struct { - module_data_t *modules; - size_t module_count; -} event_data_t; + mana_offset_ptr_t name; // mana_uint8 + mana_offset_array_t events; // mana_custom: mana_event_descriptor_t + size_t event_count; +} mana_run_descriptor_t; + +typedef struct +{ + mana_offset_array_t data_arrays; // mana_custom: mana_offset_array_t +} mana_event_data_t; #define MANA_DEFINE_PLUGIN_INIT(name) \ void *name() @@ -81,19 +75,33 @@ typedef struct void name(void *context) #define MANA_DEFINE_PLUGIN_BEGIN_RUN(name) \ - void name(void *context, run_descriptor_t run_descriptor) + void name(void *context, mana_run_descriptor_t *run_descriptor) #define MANA_DEFINE_PLUGIN_END_RUN(name) \ - void name(void *context, run_descriptor_t run_descriptor) + void name(void *context, mana_run_descriptor_t *run_descriptor) #define MANA_DEFINE_PLUGIN_EVENT_DATA(name) \ - void name(void *context, int eventIndex, event_data_t event_data) + void name(void *context, int eventIndex, mana_event_data_t *event_data) -typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_init_t); +#define MANA_DEFINE_PLUGIN_SYSTEM_EVENT(name) \ + void name(void *context, const uint32_t *data, size_t size) + +typedef MANA_DEFINE_PLUGIN_INIT(mana_init_t); typedef MANA_DEFINE_PLUGIN_SHUTDOWN(mana_shutdown_t); typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_begin_run_t); -typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_end_run_t); -typedef MANA_DEFINE_PLUGIN_EVENT_DATA(mana_event_data_t); +typedef MANA_DEFINE_PLUGIN_END_RUN(mana_end_run_t); +typedef MANA_DEFINE_PLUGIN_EVENT_DATA(mana_process_event_t); +typedef MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_process_system_event_t); + +typedef struct +{ + mana_init_t *init; + mana_shutdown_t *shutdown; + mana_begin_run_t *begin_run; + mana_end_run_t *end_run; + mana_process_event_t *process_event; + mana_process_system_event_t *process_system_event; +} mana_plugin_t; }