From c8be12c1b65481eb646a72725765de96b4c185fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20L=C3=BCke?= Date: Wed, 25 Dec 2024 05:47:38 +0100 Subject: [PATCH] more work on mana, commiting state --- src/CMakeLists.txt | 4 + src/internal/mana_analysis.h | 92 +++++++++++++++++++++++ src/internal/mana_api.h | 79 ++++++++++++++++++++ src/{tools => internal}/mana_lib.hpp | 59 ++++++++++++--- src/mana.test.cc | 3 +- src/tools/mana_auto_replay.cc | 97 ++++++------------------ src/tools/mana_replay_api.h | 108 --------------------------- 7 files changed, 247 insertions(+), 195 deletions(-) create mode 100644 src/internal/mana_analysis.h create mode 100644 src/internal/mana_api.h rename src/{tools => internal}/mana_lib.hpp (56%) delete mode 100644 src/tools/mana_replay_api.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3a2f614..6602053 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,6 +8,9 @@ target_link_libraries(mesytec-mnode PUBLIC nng mnode-proto PRIVATE spdlog) target_compile_features(mesytec-mnode PRIVATE cxx_std_17) target_compile_options(mesytec-mnode PRIVATE ${MVLC_NNG_MNODE_WARN_FLAGS}) +add_library(mana INTERFACE) +target_link_libraries(mana INTERFACE nlohmann_json::nlohmann_json mesytec-mvlc) + add_subdirectory(tools) if (MNODE_BUILD_TESTS) @@ -18,4 +21,5 @@ if (MNODE_BUILD_TESTS) endfunction() add_mnode_gtest(mana) + target_link_libraries(test_mana PRIVATE mana) endif() diff --git a/src/internal/mana_analysis.h b/src/internal/mana_analysis.h new file mode 100644 index 0000000..5972b2c --- /dev/null +++ b/src/internal/mana_analysis.h @@ -0,0 +1,92 @@ +#ifndef B63F110F_BB53_46E7_AA8E_FF6BE10CAB40 +#define B63F110F_BB53_46E7_AA8E_FF6BE10CAB40 + +#include "mana_arena.h" +#include "mana_lib.hpp" +#include + +namespace mesytec::mnode::mana +{ + +struct BitFilterExtractor +{ + mvlc::util::DataFilter filter; + mvlc::util::CacheEntry fAddress; + mvlc::util::CacheEntry fValue; + mana_offset_array_t *dest; +}; + +struct ModuleDataStage +{ + mana::Arena arena; + nlohmann::json runDescriptor; + // eventindex -> list of pointers to output offset arrays + std::vector> eventArrayPointers; + // eventIndex, moduleIndex -> list of filters attached to the module. + std::vector>> dataSources; + mana_sink_t sink = {}; + void *sinkContext = nullptr; +}; + +inline ModuleDataStage make_module_data_stage(const std::string &runName, + const mvlc::CrateConfig &crateConfig, + nlohmann::json moduleDb, mana_sink_t sink, + void *sinkContext) +{ + ModuleDataStage result; + + return result; +} + +inline void module_data_stage_begin_run(ModuleDataStage &ctx) +{ + ctx.sink.begin_run(ctx.sinkContext, ctx.runDescriptor.dump().c_str()); +} + +inline void module_data_stage_end_run(ModuleDataStage &ctx) +{ + ctx.sink.end_run(ctx.sinkContext, ctx.runDescriptor.dump().c_str()); +} + +inline void +module_data_stage_process_module_data(ModuleDataStage &ctx, int eventIndex, + const mvlc::readout_parser::ModuleData *moduleDataList, + unsigned moduleCount) +{ + auto extract_module_data = + [](const mvlc::readout_parser::DataBlock &data, BitFilterExtractor &ex) + { + auto dest = get_span(*ex.dest); + + std::fill(std::begin(dest), std::end(dest), mnode::make_quiet_nan()); + + for (const u32 *word = data.data, *end = data.data + data.size; word < end; ++word) + { + if (mvlc::util::matches(ex.filter, *word)) + { + u32 address = mvlc::util::extract(ex.fAddress, *word); + u32 value = mvlc::util::extract(ex.fValue, *word); + assert(address < dest.size()); + dest[address] = value; + } + } + }; + + auto &eventSources = ctx.dataSources.at(eventIndex); + + for (unsigned mi = 0; mi < moduleCount; ++mi) + { + auto &moduleData = moduleDataList[mi]; + auto dataBlock = moduleData.hasDynamic ? dynamic_span(moduleData) : prefix_span(moduleData); + + for (auto &source: eventSources.at(mi)) + extract_module_data(dataBlock, source); + } + + // auto arrays = ctx.eventArrayPointers.at(eventIndex); + // ctx.sink.process_event(ctx.sinkContext, eventIndex, +} + +} // namespace mesytec::mnode::mana + +#endif /* B63F110F_BB53_46E7_AA8E_FF6BE10CAB40 */ diff --git a/src/internal/mana_api.h b/src/internal/mana_api.h new file mode 100644 index 0000000..ec79b70 --- /dev/null +++ b/src/internal/mana_api.h @@ -0,0 +1,79 @@ +#ifndef A51A04C1_ABD6_4DE9_B16A_49A9DA46C67E +#define A51A04C1_ABD6_4DE9_B16A_49A9DA46C67E + +#include +#include + +#ifdef __cplusplus +extern "C" +#endif +{ + + typedef enum + { + mana_custom, + mana_uint8, + mana_sint8, + mana_uint16, + mana_uint32, + mana_uint64, + mana_float, + mana_double, + } mana_data_type_t; + + typedef struct + { + mana_data_type_t data_type; + ptrdiff_t offset; + } mana_offset_ptr_t; + + typedef struct + { + mana_offset_ptr_t ptr; + size_t size_bytes; + } mana_offset_array_t; + +#define MANA_DEFINE_PLUGIN_INIT(name) \ + void *name() + +#define MANA_DEFINE_PLUGIN_SHUTDOWN(name) \ + void name(void *context) + +#define MANA_DEFINE_PLUGIN_BEGIN_RUN(name) \ + void name(void *context, const char *descriptor_json) + +#define MANA_DEFINE_PLUGIN_END_RUN(name) \ + void name(void *context, const char *descriptor_json) + +#define MANA_DEFINE_PLUGIN_EVENT_DATA(name) \ + void name(void *context, uint16_t eventIndex, const mana_offset_array_t *arrays, size_t arrayCount, size_t totalBytes) + +#define MANA_DEFINE_PLUGIN_SYSTEM_EVENT(name) \ + void name(void *context, const uint32_t *data, size_t size) + +typedef MANA_DEFINE_PLUGIN_INIT(mana_init_t); +typedef MANA_DEFINE_PLUGIN_SHUTDOWN(mana_shutdown_t); +typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_begin_run_t); +typedef MANA_DEFINE_PLUGIN_END_RUN(mana_end_run_t); +typedef MANA_DEFINE_PLUGIN_EVENT_DATA(mana_process_event_t); +typedef MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_process_system_event_t); + +typedef struct +{ + mana_init_t *init; + mana_shutdown_t *shutdown; + mana_begin_run_t *begin_run; + mana_end_run_t *end_run; + mana_process_event_t *process_event; + mana_process_system_event_t *process_system_event; +} mana_sink_t; + + // event data serialization format: + // u32 event count + // u32 total size in bytes + // u16 eventIndex, u16 array descriptor count + // array descriptors + // data arrays +} + +#endif /* A51A04C1_ABD6_4DE9_B16A_49A9DA46C67E */ diff --git a/src/tools/mana_lib.hpp b/src/internal/mana_lib.hpp similarity index 56% rename from src/tools/mana_lib.hpp rename to src/internal/mana_lib.hpp index 0debee6..010317c 100644 --- a/src/tools/mana_lib.hpp +++ b/src/internal/mana_lib.hpp @@ -3,33 +3,38 @@ #include #include -#include "mana_replay_api.h" +#include +#include +#include +#include "mana_api.h" namespace mesytec::mnode::mana { -inline bool is_null(const mana_offset_ptr_t &ptr) { return ptr.offset == 1; } -inline void set_raw(mana_offset_ptr_t &ptr, void *p) +namespace detail +{ +inline void set(mana_offset_ptr_t &ptr, void *p) { assert(p != reinterpret_cast(&ptr) + 1); if (p) { ptr.offset = reinterpret_cast(p) - reinterpret_cast(&ptr); - spdlog::info("set_raw: &ptr={}, p={}, offset={}", fmt::ptr(&ptr), fmt::ptr(p), ptr.offset); + // spdlog::info("detail::set: &ptr={}, p={}, offset={}", fmt::ptr(&ptr), fmt::ptr(p), + // ptr.offset); } else { ptr.offset = 1; } } +} // namespace detail + +inline bool is_null(const mana_offset_ptr_t &ptr) { return ptr.offset == 1; } inline void *get(mana_offset_ptr_t &ptr) { - if (is_null(ptr)) - return nullptr; - - return reinterpret_cast(&ptr) + ptr.offset; + return is_null(ptr) ? nullptr : reinterpret_cast(&ptr) + ptr.offset; } template T *get(mana_offset_ptr_t &ptr); // to catch unsupported types @@ -52,17 +57,51 @@ template <> double *get(mana_offset_ptr_t &ptr) { return get_(ptr, mana_ inline void set(mana_offset_ptr_t &ptr, mana_data_type_t data_type, void *p) { ptr.data_type = data_type; - set_raw(ptr, p); + detail::set(ptr, p); } inline void set(mana_offset_ptr_t &ptr, std::nullptr_t) { set(ptr, mana_uint32, nullptr); } inline void set(mana_offset_ptr_t &ptr, u32 *p) { set(ptr, mana_uint32, p); } inline void set(mana_offset_ptr_t &ptr, u64 *p) { set(ptr, mana_uint64, p); } -inline void set(mana_offset_ptr_t &ptr, s8 *p) { set(ptr, mana_sint8, p); } +inline void set(mana_offset_ptr_t &ptr, s8 *p) { set(ptr, mana_sint8, p); } inline void set(mana_offset_ptr_t &ptr, char *p) { set(ptr, mana_sint8, p); } inline void set(mana_offset_ptr_t &ptr, float *p) { set(ptr, mana_float, p); } inline void set(mana_offset_ptr_t &ptr, double *p) { set(ptr, mana_double, p); } +template mvlc::util::span get_span(mana_offset_array_t &array) +{ + auto ptr = reinterpret_cast(get(array.ptr)); + auto size = array.size_bytes / sizeof(T); + return { ptr, size }; } +inline nlohmann::json make_array_descriptor(const std::string &name, mana_data_type_t data_type, + size_t size, size_t bits = 0) +{ + nlohmann::json j; + j["name"] = name; + j["data_type"] = data_type; + j["size"] = size; + j["bits"] = bits; + return j; +} + +inline nlohmann::json make_array_descriptor(const std::string &name, const std::string &bit_filter) +{ + auto f = mvlc::util::make_filter_with_caches(bit_filter); + size_t size = 1; + + if (auto c = mvlc::util::get_cache_entry(f, 'A')) + size = 1u << c->extractBits; + + size_t bits = 0; + + if (auto c = mvlc::util::get_cache_entry(f, 'D')) + bits = c->extractBits; + + return make_array_descriptor(name, mana_float, size, bits); +} + +} // namespace mesytec::mnode::mana + #endif /* AAB5E4D2_A05B_4F2F_B76A_406A5A569D55 */ diff --git a/src/mana.test.cc b/src/mana.test.cc index 3d67cd6..8489136 100644 --- a/src/mana.test.cc +++ b/src/mana.test.cc @@ -2,8 +2,7 @@ #include #include "internal/mana_arena.h" #include -#include "tools/mana_replay_api.h" -#include "tools/mana_lib.hpp" +#include "internal/mana_lib.hpp" using namespace mesytec::mnode; diff --git a/src/tools/mana_auto_replay.cc b/src/tools/mana_auto_replay.cc index 5eff86a..f4727fa 100644 --- a/src/tools/mana_auto_replay.cc +++ b/src/tools/mana_auto_replay.cc @@ -38,9 +38,8 @@ #include #include #include -#include "mana_replay_api.h" #include "internal/mana_arena.h" -#include "tools/mana_lib.hpp" +#include "internal/mana_lib.hpp" CMRC_DECLARE(mnode::resources); @@ -140,91 +139,32 @@ MANA_DEFINE_PLUGIN_END_RUN(mana_default_end_run) {} MANA_DEFINE_PLUGIN_EVENT_DATA(mana_default_process_event) {} MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_default_system_event) {} -static mana_plugin_t DefaultPlugin = {.init = mana_default_init, - .shutdown = mana_default_shutdown, - .begin_run = mana_default_begin_run, - .end_run = mana_default_end_run, - .process_event = mana_default_process_event}; +static mana_sink_t DefaultPlugin = {.init = mana_default_init, + .shutdown = mana_default_shutdown, + .begin_run = mana_default_begin_run, + .end_run = mana_default_end_run, + .process_event = mana_default_process_event}; struct DataSource { mvlc::util::DataFilter filter; mvlc::util::CacheEntry fAddress; mvlc::util::CacheEntry fValue; - mvlc::util::span dest; + mana_offset_array_t *dest; }; struct AnalysisContext { std::unique_ptr arena; - mana_run_descriptor_t *runDescriptor = nullptr; - mvlc::util::span eventStorages; - std::vector>> dataSources; // eventIndex, moduleIndex, sourceIndex - mana_plugin_t outputPlugin = DefaultPlugin; + nlohmann::json runDescriptor; + std::vector> eventArrays; + std::vector>> + dataSources; // eventIndex, moduleIndex, sourceIndex + mana_sink_t outputPlugin = DefaultPlugin; void *outputPluginContext = nullptr; size_t runDescriptorBytes = 0; size_t eventStoragesBytes = 0; - - void process_module_data(const mvlc::readout_parser::DataBlock &data, DataSource &source) - { - std::fill(std::begin(source.dest), std::end(source.dest), mnode::make_quiet_nan()); - for (const u32 *word = data.data, *end = data.data + data.size; word < end; ++word) - { - if (mvlc::util::matches(source.filter, *word)) - { - u32 address = mvlc::util::extract(source.fAddress, *word); - u32 value = mvlc::util::extract(source.fValue, *word); - assert(address < source.dest.size()); - source.dest[address] = value; - } - } - } - - void process_module_data(const mvlc::readout_parser::DataBlock &data, std::vector &sources) - { - for (auto &source: sources) - { - process_module_data(data, source); - } - } - - void process_module_data(const mvlc::readout_parser::ModuleData &moduleData, - std::vector &sources) - { - mvlc::readout_parser::DataBlock data = moduleData.hasDynamic ? dynamic_span(moduleData) - : prefix_span(moduleData); - process_module_data(data, sources); - - } - - void process_event_data(int crateIndex, int eventIndex, - const mvlc::readout_parser::ModuleData *moduleDataList, - unsigned moduleCount) - { - std::cout << fmt::format( - "ReadoutEvent: ctx={}, crateId={}, eventIndex={}, moduleCount={}\n", fmt::ptr(this), - crateIndex, eventIndex, moduleCount); - - auto &eventSources = dataSources.at(eventIndex); - - for (unsigned mi=0; mi @@ -351,6 +291,7 @@ template mvlc::util::span get_span(mana_offset_array_t &array) return {ptr, size}; } +#if 0 std::pair make_run_descriptor(mana::Arena &arena, const std::string &runName, const std::vector> &eventDsInfo) @@ -429,6 +370,7 @@ void dump(mana_run_descriptor_t *rd) } } } +#endif std::optional make_mana(const std::string runName, const mvlc::CrateConfig &crateConfig, @@ -445,6 +387,7 @@ std::optional make_mana(const std::string runName, // eventIndex -> moduleIndex -> module info json auto moduleDataSources = match_modules(crateConfig, moduleInfoByType); +#if 0 auto eventDsInfo = make_event_ds_info(crateConfig, moduleDataSources); auto arena = ctx.arena.get(); @@ -483,6 +426,7 @@ std::optional make_mana(const std::string runName, ctx.eventStorages = eventStorages; ctx.runDescriptorBytes = runDescriptorSize; ctx.eventStoragesBytes = eventStoragesSize; +#endif return ctx; } @@ -538,12 +482,15 @@ int main(int argc, char *argv[]) const mvlc::readout_parser::ModuleData *moduleDataList, unsigned moduleCount) { - reinterpret_cast(ctx)->process_event_data(crateIndex, eventIndex, - moduleDataList, moduleCount); + // reinterpret_cast(ctx)->process_event_data(crateIndex, eventIndex, + // moduleDataList, + // moduleCount); }; auto system_event = [](void *ctx, int crateIndex, const u32 *header, u32 size) - { reinterpret_cast(ctx)->process_system_event(crateIndex, header, size); }; + { + // reinterpret_cast(ctx)->process_system_event(crateIndex, header, size); + }; auto parserContext = make_parser_context(listfileContext->crateConfig, {event_data, system_event}); diff --git a/src/tools/mana_replay_api.h b/src/tools/mana_replay_api.h deleted file mode 100644 index 5026a71..0000000 --- a/src/tools/mana_replay_api.h +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef DA93DC95_9F70_4B58_9B6C_5E32D3811E48 -#define DA93DC95_9F70_4B58_9B6C_5E32D3811E48 - -#include -#include - -#ifdef __cplusplus -extern "C" -#endif -{ - -// descriptors - daq tree structure -// daq -> events -> modules -> data_sources -// alternative concepts: -// crate -> triggers -> modules -> data_sources -// example data sources: "amplitude", "channel_time", "timestamp" -// data is float, data sources have a size of at least 1 -// todo: add number of valid bits in the data source -// todo: rng handling? - -typedef enum -{ - mana_custom, - mana_uint8, - mana_sint8, - mana_uint16, - mana_uint32, - mana_uint64, - mana_float, - mana_double, -} mana_data_type_t; - -typedef struct -{ - mana_data_type_t data_type; - ptrdiff_t offset; -} mana_offset_ptr_t; - -typedef struct -{ - mana_offset_ptr_t ptr; - size_t size_bytes; -} mana_offset_array_t; - -typedef struct -{ - mana_offset_ptr_t name; // mana_uint8 - mana_data_type_t data_type; - size_t size; - size_t bits; -} mana_array_descriptor_t; - -typedef struct -{ - mana_offset_array_t data_arrays; // mana_custom: mana_array_descriptor_t - size_t data_array_count; -} mana_event_descriptor_t; - -typedef struct -{ - mana_offset_ptr_t name; // mana_uint8 - mana_offset_array_t events; // mana_custom: mana_event_descriptor_t - size_t event_count; -} mana_run_descriptor_t; - -typedef struct -{ - mana_offset_array_t data_arrays; // mana_custom: mana_offset_array_t -} mana_event_data_t; - -#define MANA_DEFINE_PLUGIN_INIT(name) \ - void *name() - -#define MANA_DEFINE_PLUGIN_SHUTDOWN(name) \ - void name(void *context) - -#define MANA_DEFINE_PLUGIN_BEGIN_RUN(name) \ - void name(void *context, mana_run_descriptor_t *run_descriptor) - -#define MANA_DEFINE_PLUGIN_END_RUN(name) \ - void name(void *context, mana_run_descriptor_t *run_descriptor) - -#define MANA_DEFINE_PLUGIN_EVENT_DATA(name) \ - void name(void *context, int eventIndex, mana_event_data_t *event_data) - -#define MANA_DEFINE_PLUGIN_SYSTEM_EVENT(name) \ - void name(void *context, const uint32_t *data, size_t size) - -typedef MANA_DEFINE_PLUGIN_INIT(mana_init_t); -typedef MANA_DEFINE_PLUGIN_SHUTDOWN(mana_shutdown_t); -typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_begin_run_t); -typedef MANA_DEFINE_PLUGIN_END_RUN(mana_end_run_t); -typedef MANA_DEFINE_PLUGIN_EVENT_DATA(mana_process_event_t); -typedef MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_process_system_event_t); - -typedef struct -{ - mana_init_t *init; - mana_shutdown_t *shutdown; - mana_begin_run_t *begin_run; - mana_end_run_t *end_run; - mana_process_event_t *process_event; - mana_process_system_event_t *process_system_event; -} mana_plugin_t; - -} - -#endif /* DA93DC95_9F70_4B58_9B6C_5E32D3811E48 */