florian/mesytec-mnode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

more work on mana, commiting state

Browse source
This commit is contained in:
Florian Lüke 2024-12-25 05:47:38 +01:00
parent dfe5d7eb3b
commit c8be12c1b6
7 changed files with 247 additions and 195 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/CMakeLists.txt
View file

@ -8,6 +8,9 @@ target_link_libraries(mesytec-mnode PUBLIC nng mnode-proto PRIVATE spdlog)
target_compile_features(mesytec-mnode PRIVATE cxx_std_17)
target_compile_options(mesytec-mnode PRIVATE ${MVLC_NNG_MNODE_WARN_FLAGS})
add_library(mana INTERFACE)
target_link_libraries(mana INTERFACE nlohmann_json::nlohmann_json mesytec-mvlc)
add_subdirectory(tools)
if (MNODE_BUILD_TESTS)
@ -18,4 +21,5 @@ if (MNODE_BUILD_TESTS)
endfunction()
add_mnode_gtest(mana)
target_link_libraries(test_mana PRIVATE mana)
endif()

92
src/internal/mana_analysis.h Normal file
View file

@ -0,0 +1,92 @@
#ifndef B63F110F_BB53_46E7_AA8E_FF6BE10CAB40
#define B63F110F_BB53_46E7_AA8E_FF6BE10CAB40
#include "mana_arena.h"
#include "mana_lib.hpp"
#include <mesytec-mvlc/mesytec-mvlc.h>
namespace mesytec::mnode::mana
{
struct BitFilterExtractor
{
mvlc::util::DataFilter filter;
mvlc::util::CacheEntry fAddress;
mvlc::util::CacheEntry fValue;
mana_offset_array_t *dest;
};
struct ModuleDataStage
{
mana::Arena arena;
nlohmann::json runDescriptor;
// eventindex -> list of pointers to output offset arrays
std::vector<std::vector<mana_offset_array_t *>> eventArrayPointers;
// eventIndex, moduleIndex -> list of filters attached to the module.
std::vector<std::vector<std::vector<BitFilterExtractor>>> dataSources;
mana_sink_t sink = {};
void *sinkContext = nullptr;
};
inline ModuleDataStage make_module_data_stage(const std::string &runName,
const mvlc::CrateConfig &crateConfig,
nlohmann::json moduleDb, mana_sink_t sink,
void *sinkContext)
{
ModuleDataStage result;
return result;
}
inline void module_data_stage_begin_run(ModuleDataStage &ctx)
{
ctx.sink.begin_run(ctx.sinkContext, ctx.runDescriptor.dump().c_str());
}
inline void module_data_stage_end_run(ModuleDataStage &ctx)
{
ctx.sink.end_run(ctx.sinkContext, ctx.runDescriptor.dump().c_str());
}
inline void
module_data_stage_process_module_data(ModuleDataStage &ctx, int eventIndex,
const mvlc::readout_parser::ModuleData *moduleDataList,
unsigned moduleCount)
{
auto extract_module_data =
[](const mvlc::readout_parser::DataBlock &data, BitFilterExtractor &ex)
{
auto dest = get_span<float>(*ex.dest);
std::fill(std::begin(dest), std::end(dest), mnode::make_quiet_nan());
for (const u32 *word = data.data, *end = data.data + data.size; word < end; ++word)
{
if (mvlc::util::matches(ex.filter, *word))
{
u32 address = mvlc::util::extract(ex.fAddress, *word);
u32 value = mvlc::util::extract(ex.fValue, *word);
assert(address < dest.size());
dest[address] = value;
}
}
};
auto &eventSources = ctx.dataSources.at(eventIndex);
for (unsigned mi = 0; mi < moduleCount; ++mi)
{
auto &moduleData = moduleDataList[mi];
auto dataBlock = moduleData.hasDynamic ? dynamic_span(moduleData) : prefix_span(moduleData);
for (auto &source: eventSources.at(mi))
extract_module_data(dataBlock, source);
}
// auto arrays = ctx.eventArrayPointers.at(eventIndex);
// ctx.sink.process_event(ctx.sinkContext, eventIndex,
}
} // namespace mesytec::mnode::mana
#endif /* B63F110F_BB53_46E7_AA8E_FF6BE10CAB40 */

79
src/internal/mana_api.h Normal file
View file

@ -0,0 +1,79 @@
#ifndef A51A04C1_ABD6_4DE9_B16A_49A9DA46C67E
#define A51A04C1_ABD6_4DE9_B16A_49A9DA46C67E
#include <stdint.h>
#include <stddef.h>
#ifdef __cplusplus
extern "C"
#endif
{
typedef enum
{
mana_custom,
mana_uint8,
mana_sint8,
mana_uint16,
mana_uint32,
mana_uint64,
mana_float,
mana_double,
} mana_data_type_t;
typedef struct
{
mana_data_type_t data_type;
ptrdiff_t offset;
} mana_offset_ptr_t;
typedef struct
{
mana_offset_ptr_t ptr;
size_t size_bytes;
} mana_offset_array_t;
#define MANA_DEFINE_PLUGIN_INIT(name) \
void *name()
#define MANA_DEFINE_PLUGIN_SHUTDOWN(name) \
void name(void *context)
#define MANA_DEFINE_PLUGIN_BEGIN_RUN(name) \
void name(void *context, const char *descriptor_json)
#define MANA_DEFINE_PLUGIN_END_RUN(name) \
void name(void *context, const char *descriptor_json)
#define MANA_DEFINE_PLUGIN_EVENT_DATA(name) \
void name(void *context, uint16_t eventIndex, const mana_offset_array_t *arrays, size_t arrayCount, size_t totalBytes)
#define MANA_DEFINE_PLUGIN_SYSTEM_EVENT(name) \
void name(void *context, const uint32_t *data, size_t size)
typedef MANA_DEFINE_PLUGIN_INIT(mana_init_t);
typedef MANA_DEFINE_PLUGIN_SHUTDOWN(mana_shutdown_t);
typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_begin_run_t);
typedef MANA_DEFINE_PLUGIN_END_RUN(mana_end_run_t);
typedef MANA_DEFINE_PLUGIN_EVENT_DATA(mana_process_event_t);
typedef MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_process_system_event_t);
typedef struct
{
mana_init_t *init;
mana_shutdown_t *shutdown;
mana_begin_run_t *begin_run;
mana_end_run_t *end_run;
mana_process_event_t *process_event;
mana_process_system_event_t *process_system_event;
} mana_sink_t;
// event data serialization format:
// u32 event count
// u32 total size in bytes
// u16 eventIndex, u16 array descriptor count
// array descriptors
// data arrays
}
#endif /* A51A04C1_ABD6_4DE9_B16A_49A9DA46C67E */

57
src/tools/mana_lib.hpp → src/internal/mana_lib.hpp
View file

@ -3,33 +3,38 @@
#include <cassert>
#include <mesytec-mnode/mnode_cpp_types.h>
#include "mana_replay_api.h"
#include <mesytec-mvlc/cpp_compat.h>
#include <mesytec-mvlc/util/data_filter.h>
#include <nlohmann/json.hpp>
#include "mana_api.h"
namespace mesytec::mnode::mana
{
inline bool is_null(const mana_offset_ptr_t &ptr) { return ptr.offset == 1; }
inline void set_raw(mana_offset_ptr_t &ptr, void *p)
namespace detail
{
inline void set(mana_offset_ptr_t &ptr, void *p)
{
assert(p != reinterpret_cast<u8 *>(&ptr) + 1);
if (p)
{
ptr.offset = reinterpret_cast<u8 *>(p) - reinterpret_cast<u8 *>(&ptr);
spdlog::info("set_raw: &ptr={}, p={}, offset={}", fmt::ptr(&ptr), fmt::ptr(p), ptr.offset);
// spdlog::info("detail::set: &ptr={}, p={}, offset={}", fmt::ptr(&ptr), fmt::ptr(p),
// ptr.offset);
}
else
{
ptr.offset = 1;
}
}
} // namespace detail
inline bool is_null(const mana_offset_ptr_t &ptr) { return ptr.offset == 1; }
inline void *get(mana_offset_ptr_t &ptr)
{
if (is_null(ptr))
return nullptr;
return reinterpret_cast<u8 *>(&ptr) + ptr.offset;
return is_null(ptr) ? nullptr : reinterpret_cast<u8 *>(&ptr) + ptr.offset;
}
template <typename T> T *get(mana_offset_ptr_t &ptr); // to catch unsupported types
@ -52,7 +57,7 @@ template <> double *get(mana_offset_ptr_t &ptr) { return get_<double>(ptr, mana_
inline void set(mana_offset_ptr_t &ptr, mana_data_type_t data_type, void *p)
{
ptr.data_type = data_type;
set_raw(ptr, p);
detail::set(ptr, p);
}
inline void set(mana_offset_ptr_t &ptr, std::nullptr_t) { set(ptr, mana_uint32, nullptr); }
@ -63,6 +68,40 @@ inline void set(mana_offset_ptr_t &ptr, char *p) { set(ptr, mana_sint8, p); }
inline void set(mana_offset_ptr_t &ptr, float *p) { set(ptr, mana_float, p); }
inline void set(mana_offset_ptr_t &ptr, double *p) { set(ptr, mana_double, p); }
template <typename T> mvlc::util::span<T> get_span(mana_offset_array_t &array)
{
auto ptr = reinterpret_cast<T *>(get(array.ptr));
auto size = array.size_bytes / sizeof(T);
return { ptr, size };
}
inline nlohmann::json make_array_descriptor(const std::string &name, mana_data_type_t data_type,
size_t size, size_t bits = 0)
{
nlohmann::json j;
j["name"] = name;
j["data_type"] = data_type;
j["size"] = size;
j["bits"] = bits;
return j;
}
inline nlohmann::json make_array_descriptor(const std::string &name, const std::string &bit_filter)
{
auto f = mvlc::util::make_filter_with_caches(bit_filter);
size_t size = 1;
if (auto c = mvlc::util::get_cache_entry(f, 'A'))
size = 1u << c->extractBits;
size_t bits = 0;
if (auto c = mvlc::util::get_cache_entry(f, 'D'))
bits = c->extractBits;
return make_array_descriptor(name, mana_float, size, bits);
}
} // namespace mesytec::mnode::mana
#endif /* AAB5E4D2_A05B_4F2F_B76A_406A5A569D55 */

3
src/mana.test.cc
View file

@ -2,8 +2,7 @@
#include <mesytec-mnode/mnode_cpp_types.h>
#include "internal/mana_arena.h"
#include <spdlog/spdlog.h>
#include "tools/mana_replay_api.h"
#include "tools/mana_lib.hpp"
#include "internal/mana_lib.hpp"
using namespace mesytec::mnode;

89
src/tools/mana_auto_replay.cc
View file

@ -38,9 +38,8 @@
#include <nlohmann/json.hpp>
#include <list>
#include <sstream>
#include "mana_replay_api.h"
#include "internal/mana_arena.h"
#include "tools/mana_lib.hpp"
#include "internal/mana_lib.hpp"
CMRC_DECLARE(mnode::resources);
@ -140,7 +139,7 @@ MANA_DEFINE_PLUGIN_END_RUN(mana_default_end_run) {}
MANA_DEFINE_PLUGIN_EVENT_DATA(mana_default_process_event) {}
MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_default_system_event) {}
static mana_plugin_t DefaultPlugin = {.init = mana_default_init,
static mana_sink_t DefaultPlugin = {.init = mana_default_init,
.shutdown = mana_default_shutdown,
.begin_run = mana_default_begin_run,
.end_run = mana_default_end_run,
@ -151,80 +150,21 @@ struct DataSource
mvlc::util::DataFilter filter;
mvlc::util::CacheEntry fAddress;
mvlc::util::CacheEntry fValue;
mvlc::util::span<float> dest;
mana_offset_array_t *dest;
};
struct AnalysisContext
{
std::unique_ptr<mana::Arena> arena;
mana_run_descriptor_t *runDescriptor = nullptr;
mvlc::util::span<mana_event_data_t> eventStorages;
std::vector<std::vector<std::vector<DataSource>>> dataSources; // eventIndex, moduleIndex, sourceIndex
mana_plugin_t outputPlugin = DefaultPlugin;
nlohmann::json runDescriptor;
std::vector<mvlc::util::span<mana_offset_array_t *>> eventArrays;
std::vector<std::vector<std::vector<DataSource>>>
dataSources; // eventIndex, moduleIndex, sourceIndex
mana_sink_t outputPlugin = DefaultPlugin;
void *outputPluginContext = nullptr;
size_t runDescriptorBytes = 0;
size_t eventStoragesBytes = 0;
void process_module_data(const mvlc::readout_parser::DataBlock &data, DataSource &source)
{
std::fill(std::begin(source.dest), std::end(source.dest), mnode::make_quiet_nan());
for (const u32 *word = data.data, *end = data.data + data.size; word < end; ++word)
{
if (mvlc::util::matches(source.filter, *word))
{
u32 address = mvlc::util::extract(source.fAddress, *word);
u32 value = mvlc::util::extract(source.fValue, *word);
assert(address < source.dest.size());
source.dest[address] = value;
}
}
}
void process_module_data(const mvlc::readout_parser::DataBlock &data, std::vector<DataSource> &sources)
{
for (auto &source: sources)
{
process_module_data(data, source);
}
}
void process_module_data(const mvlc::readout_parser::ModuleData &moduleData,
std::vector<DataSource> &sources)
{
mvlc::readout_parser::DataBlock data = moduleData.hasDynamic ? dynamic_span(moduleData)
: prefix_span(moduleData);
process_module_data(data, sources);
}
void process_event_data(int crateIndex, int eventIndex,
const mvlc::readout_parser::ModuleData *moduleDataList,
unsigned moduleCount)
{
std::cout << fmt::format(
"ReadoutEvent: ctx={}, crateId={}, eventIndex={}, moduleCount={}\n", fmt::ptr(this),
crateIndex, eventIndex, moduleCount);
auto &eventSources = dataSources.at(eventIndex);
for (unsigned mi=0; mi<moduleCount; ++mi)
{
auto &moduleData = moduleDataList[mi];
auto &sources = eventSources.at(mi);
process_module_data(moduleData, sources);
}
outputPlugin.process_event(outputPluginContext, eventIndex, &eventStorages[eventIndex]);
}
void process_system_event(int crateIndex, const u32 *header, u32 size)
{
std::cout << fmt::format("SystemEvent: ctx={}, crateId={}, size={}\n", fmt::ptr(this),
crateIndex, size);
outputPlugin.process_system_event(outputPluginContext, header, size);
}
};
std::map<std::string, nlohmann::json>
@ -351,6 +291,7 @@ template <typename T> mvlc::util::span<T> get_span(mana_offset_array_t &array)
return {ptr, size};
}
#if 0
std::pair<mana_run_descriptor_t *, size_t>
make_run_descriptor(mana::Arena &arena, const std::string &runName,
const std::vector<std::vector<DataSourceInfo>> &eventDsInfo)
@ -429,6 +370,7 @@ void dump(mana_run_descriptor_t *rd)
}
}
}
#endif
std::optional<AnalysisContext> make_mana(const std::string runName,
const mvlc::CrateConfig &crateConfig,
@ -445,6 +387,7 @@ std::optional<AnalysisContext> make_mana(const std::string runName,
// eventIndex -> moduleIndex -> module info json
auto moduleDataSources = match_modules(crateConfig, moduleInfoByType);
#if 0
auto eventDsInfo = make_event_ds_info(crateConfig, moduleDataSources);
auto arena = ctx.arena.get();
@ -483,6 +426,7 @@ std::optional<AnalysisContext> make_mana(const std::string runName,
ctx.eventStorages = eventStorages;
ctx.runDescriptorBytes = runDescriptorSize;
ctx.eventStoragesBytes = eventStoragesSize;
#endif
return ctx;
}
@ -538,12 +482,15 @@ int main(int argc, char *argv[])
const mvlc::readout_parser::ModuleData *moduleDataList,
unsigned moduleCount)
{
reinterpret_cast<AnalysisContext *>(ctx)->process_event_data(crateIndex, eventIndex,
moduleDataList, moduleCount);
// reinterpret_cast<AnalysisContext *>(ctx)->process_event_data(crateIndex, eventIndex,
// moduleDataList,
// moduleCount);
};
auto system_event = [](void *ctx, int crateIndex, const u32 *header, u32 size)
{ reinterpret_cast<AnalysisContext *>(ctx)->process_system_event(crateIndex, header, size); };
{
// reinterpret_cast<AnalysisContext *>(ctx)->process_system_event(crateIndex, header, size);
};
auto parserContext =
make_parser_context(listfileContext->crateConfig, {event_data, system_event});

108
src/tools/mana_replay_api.h
View file

@ -1,108 +0,0 @@
#ifndef DA93DC95_9F70_4B58_9B6C_5E32D3811E48
#define DA93DC95_9F70_4B58_9B6C_5E32D3811E48
#include <stdint.h>
#include <stddef.h>
#ifdef __cplusplus
extern "C"
#endif
{
// descriptors - daq tree structure
// daq -> events -> modules -> data_sources
// alternative concepts:
// crate -> triggers -> modules -> data_sources
// example data sources: "amplitude", "channel_time", "timestamp"
// data is float, data sources have a size of at least 1
// todo: add number of valid bits in the data source
// todo: rng handling?
typedef enum
{
mana_custom,
mana_uint8,
mana_sint8,
mana_uint16,
mana_uint32,
mana_uint64,
mana_float,
mana_double,
} mana_data_type_t;
typedef struct
{
mana_data_type_t data_type;
ptrdiff_t offset;
} mana_offset_ptr_t;
typedef struct
{
mana_offset_ptr_t ptr;
size_t size_bytes;
} mana_offset_array_t;
typedef struct
{
mana_offset_ptr_t name; // mana_uint8
mana_data_type_t data_type;
size_t size;
size_t bits;
} mana_array_descriptor_t;
typedef struct
{
mana_offset_array_t data_arrays; // mana_custom: mana_array_descriptor_t
size_t data_array_count;
} mana_event_descriptor_t;
typedef struct
{
mana_offset_ptr_t name; // mana_uint8
mana_offset_array_t events; // mana_custom: mana_event_descriptor_t
size_t event_count;
} mana_run_descriptor_t;
typedef struct
{
mana_offset_array_t data_arrays; // mana_custom: mana_offset_array_t
} mana_event_data_t;
#define MANA_DEFINE_PLUGIN_INIT(name) \
void *name()
#define MANA_DEFINE_PLUGIN_SHUTDOWN(name) \
void name(void *context)
#define MANA_DEFINE_PLUGIN_BEGIN_RUN(name) \
void name(void *context, mana_run_descriptor_t *run_descriptor)
#define MANA_DEFINE_PLUGIN_END_RUN(name) \
void name(void *context, mana_run_descriptor_t *run_descriptor)
#define MANA_DEFINE_PLUGIN_EVENT_DATA(name) \
void name(void *context, int eventIndex, mana_event_data_t *event_data)
#define MANA_DEFINE_PLUGIN_SYSTEM_EVENT(name) \
void name(void *context, const uint32_t *data, size_t size)
typedef MANA_DEFINE_PLUGIN_INIT(mana_init_t);
typedef MANA_DEFINE_PLUGIN_SHUTDOWN(mana_shutdown_t);
typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_begin_run_t);
typedef MANA_DEFINE_PLUGIN_END_RUN(mana_end_run_t);
typedef MANA_DEFINE_PLUGIN_EVENT_DATA(mana_process_event_t);
typedef MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_process_system_event_t);
typedef struct
{
mana_init_t *init;
mana_shutdown_t *shutdown;
mana_begin_run_t *begin_run;
mana_end_run_t *end_run;
mana_process_event_t *process_event;
mana_process_system_event_t *process_system_event;
} mana_plugin_t;
}
#endif /* DA93DC95_9F70_4B58_9B6C_5E32D3811E48 */