florian/mesytec-mnode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

mana: unfinished offset ptr stuff

Browse source
This commit is contained in:
Florian Lüke 2024-12-25 03:19:29 +01:00
parent 7751d487ed
commit dfe5d7eb3b
3 changed files with 473 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

390
src/tools/mana_auto_replay.cc
View file

@ -37,8 +37,10 @@
#include <mesytec-mnode/mnode_math.h>
#include <nlohmann/json.hpp>
#include <list>
#include <sstream>
#include "mana_replay_api.h"
#include "internal/mana_arena.h"
#include "tools/mana_lib.hpp"
CMRC_DECLARE(mnode::resources);
@ -131,25 +133,357 @@ std::optional<ParserContext> make_parser_context(const mvlc::CrateConfig &crateC
}
}
MANA_DEFINE_PLUGIN_INIT(mana_default_init) { return nullptr; }
MANA_DEFINE_PLUGIN_SHUTDOWN(mana_default_shutdown) {}
MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_default_begin_run) {}
MANA_DEFINE_PLUGIN_END_RUN(mana_default_end_run) {}
MANA_DEFINE_PLUGIN_EVENT_DATA(mana_default_process_event) {}
MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_default_system_event) {}
static mana_plugin_t DefaultPlugin = {.init = mana_default_init,
.shutdown = mana_default_shutdown,
.begin_run = mana_default_begin_run,
.end_run = mana_default_end_run,
.process_event = mana_default_process_event};
struct DataSource
{
mvlc::util::DataFilter filter;
mvlc::util::CacheEntry fAddress;
mvlc::util::CacheEntry fValue;
mvlc::util::span<float> dest;
};
struct AnalysisContext
{
std::unique_ptr<mana::Arena> arena;
run_descriptor_t *runDescriptor = nullptr;
mana_run_descriptor_t *runDescriptor = nullptr;
mvlc::util::span<mana_event_data_t> eventStorages;
std::vector<std::vector<std::vector<DataSource>>> dataSources; // eventIndex, moduleIndex, sourceIndex
mana_plugin_t outputPlugin = DefaultPlugin;
void *outputPluginContext = nullptr;
size_t runDescriptorBytes = 0;
size_t eventStoragesBytes = 0;
void process_module_data(const mvlc::readout_parser::DataBlock &data, DataSource &source)
{
std::fill(std::begin(source.dest), std::end(source.dest), mnode::make_quiet_nan());
for (const u32 *word = data.data, *end = data.data + data.size; word < end; ++word)
{
if (mvlc::util::matches(source.filter, *word))
{
u32 address = mvlc::util::extract(source.fAddress, *word);
u32 value = mvlc::util::extract(source.fValue, *word);
assert(address < source.dest.size());
source.dest[address] = value;
}
}
}
void process_module_data(const mvlc::readout_parser::DataBlock &data, std::vector<DataSource> &sources)
{
for (auto &source: sources)
{
process_module_data(data, source);
}
}
void process_module_data(const mvlc::readout_parser::ModuleData &moduleData,
std::vector<DataSource> &sources)
{
mvlc::readout_parser::DataBlock data = moduleData.hasDynamic ? dynamic_span(moduleData)
: prefix_span(moduleData);
process_module_data(data, sources);
}
void process_event_data(int crateIndex, int eventIndex,
const mvlc::readout_parser::ModuleData *moduleDataList,
unsigned moduleCount)
{
std::cout << fmt::format(
"ReadoutEvent: ctx={}, crateId={}, eventIndex={}, moduleCount={}\n", fmt::ptr(this),
crateIndex, eventIndex, moduleCount);
auto &eventSources = dataSources.at(eventIndex);
for (unsigned mi=0; mi<moduleCount; ++mi)
{
auto &moduleData = moduleDataList[mi];
auto &sources = eventSources.at(mi);
process_module_data(moduleData, sources);
}
outputPlugin.process_event(outputPluginContext, eventIndex, &eventStorages[eventIndex]);
}
void process_system_event(int crateIndex, const u32 *header, u32 size)
{
std::cout << fmt::format("SystemEvent: ctx={}, crateId={}, size={}\n", fmt::ptr(this),
crateIndex, size);
outputPlugin.process_system_event(outputPluginContext, header, size);
}
};
std::optional<AnalysisContext> make_mana(const mvlc::CrateConfig &crateConfig,
std::map<std::string, nlohmann::json>
make_module_info_by_type(const nlohmann::json &jModuleDataSources)
{
// module type name -> module info json
std::map<std::string, nlohmann::json> moduleInfoByType;
for (const auto &mod_: jModuleDataSources["modules"])
moduleInfoByType[mod_["type_name"]] = mod_;
return moduleInfoByType;
}
std::vector<std::vector<nlohmann::json>>
match_modules(const mvlc::CrateConfig &crateConfig,
const std::map<std::string, nlohmann::json> &moduleInfoByType)
{
std::vector<std::vector<nlohmann::json>> moduleDataSources;
for (const auto &event: crateConfig.stacks)
{
std::vector<nlohmann::json> eventModuleDataSources;
for (const auto &module_: event.getGroups())
{
nlohmann::json jModule;
// match the meta data module type name against the concrete module name
for (const auto &[type_, info]: moduleInfoByType)
{
if (module_.name.find(type_) != std::string::npos)
{
spdlog::info("match: type={}, name={}", type_, module_.name);
jModule = info;
break;
}
}
if (jModule.empty())
{
spdlog::warn("No module info found for module name '{}'", module_.name);
}
eventModuleDataSources.emplace_back(jModule);
}
moduleDataSources.emplace_back(eventModuleDataSources);
}
return moduleDataSources;
}
struct DataSourceInfo
{
const std::string name;
const std::string filterString;
mvlc::util::FilterWithCaches filter;
DataSourceInfo(const std::string &name_, const std::string &filterString_)
: name(name_)
, filterString(filterString_)
, filter(mvlc::util::make_filter_with_caches(filterString))
{
}
};
std::vector<std::vector<DataSourceInfo>>
make_event_ds_info(const mvlc::CrateConfig &crateConfig,
const std::vector<std::vector<nlohmann::json>> &moduleDataSources)
{
std::vector<std::vector<DataSourceInfo>> eventDsInfo;
for (size_t eventIndex = 0; eventIndex < crateConfig.stacks.size(); ++eventIndex)
{
const auto &dataSources = moduleDataSources.at(eventIndex);
const auto &event = crateConfig.stacks.at(eventIndex);
const auto &readouts = event.getGroups();
std::vector<DataSourceInfo> dsInfo;
for (size_t moduleIndex = 0; moduleIndex < event.getGroups().size(); ++moduleIndex)
{
const auto &readout = readouts.at(moduleIndex);
const auto &ds = dataSources.at(moduleIndex);
spdlog::info("readout.name={}, ds={}", readout.name, ds.dump());
if (ds.contains("data_sources"))
{
for (const auto &filter: ds["data_sources"])
{
auto name =
fmt::format("{}.{}.{}", event.getName(), readout.name, filter["name"]);
dsInfo.emplace_back(DataSourceInfo(name, filter["filter"]));
}
}
}
eventDsInfo.emplace_back(dsInfo);
}
return eventDsInfo;
}
template <typename T>
T *push_offset_array(mana::Arena &arena, mana_offset_array_t &dest, size_t size)
{
T *ptr = arena.push_t<T>(size);
mana::set(dest.ptr, mana_custom, ptr);
dest.size_bytes = size * sizeof(T);
return ptr;
}
template <typename T>
T *push_typed_offset_array(mana::Arena &arena, mana_offset_array_t &dest, size_t size)
{
T *ptr = arena.push_t<T>(size);
mana::set(dest.ptr, ptr);
dest.size_bytes = size * sizeof(T);
return ptr;
}
template <typename T> size_t element_count(mana_offset_array_t &array)
{
return array.size_bytes / sizeof(T);
}
template <typename T> mvlc::util::span<T> get_span(mana_offset_array_t &array)
{
auto ptr = reinterpret_cast<T *>(mana::get(array.ptr));
auto size = element_count<T>(array);
return {ptr, size};
}
std::pair<mana_run_descriptor_t *, size_t>
make_run_descriptor(mana::Arena &arena, const std::string &runName,
const std::vector<std::vector<DataSourceInfo>> &eventDsInfo)
{
auto rd = arena.push_t<mana_run_descriptor_t>();
mana::set(rd->name, arena.push_cstr(runName));
auto eds = push_offset_array<mana_event_descriptor_t>(arena, rd->events, eventDsInfo.size());
rd->event_count = eventDsInfo.size();
for (const auto &eventDataSources: eventDsInfo)
{
auto ed = eds++;
auto ads = push_offset_array<mana_array_descriptor_t>(arena, ed->data_arrays,
eventDataSources.size());
ed->data_array_count = eventDataSources.size();
for (const auto &ds: eventDataSources)
{
auto ad = ads++;
mana::set(ad->name, arena.push_cstr(ds.name));
ad->data_type = mana_float;
ad->size = 1;
if (auto c = get_cache_entry(ds.filter, 'A'); c)
ad->size = 1u << c->extractBits;
if (auto c = get_cache_entry(ds.filter, 'D'); c)
ad->bits = c->extractBits;
}
}
return {rd, arena.cur_end() - reinterpret_cast<u8 *>(rd)};
}
std::pair<mvlc::util::span<mana_event_data_t>, size_t>
make_event_storages(mana::Arena &arena, const std::vector<std::vector<DataSourceInfo>> &eventDsInfo)
{
auto eds = arena.push_t<mana_event_data_t>(eventDsInfo.size());
for (size_t eventIndex = 0; eventIndex < eventDsInfo.size(); ++eventIndex)
{
auto &ed = eds[eventIndex];
auto &dsInfo = eventDsInfo[eventIndex];
auto das = push_offset_array<mana_offset_array_t>(arena, ed.data_arrays, dsInfo.size());
for (const auto &ds: dsInfo)
{
auto da = das++;
size_t size = 1;
if (auto c = get_cache_entry(ds.filter, 'A'); c)
size = 1u << c->extractBits;
push_typed_offset_array<float>(arena, *da, size);
}
}
return {{eds, eventDsInfo.size()}, arena.cur_end() - reinterpret_cast<u8 *>(eds)};
}
void dump(mana_run_descriptor_t *rd)
{
auto eds = get_span<mana_event_descriptor_t>(rd->events);
spdlog::info("mana_run_descriptor @ {}, name={}, event_count={}", fmt::ptr(rd),
mana::get<char>(rd->name), eds.size());
for (size_t eventIndex = 0; eventIndex < rd->event_count; ++eventIndex)
{
auto &ed = eds[eventIndex];
spdlog::info(" event[{}]:", eventIndex);
auto ads = get_span<mana_array_descriptor_t>(ed.data_arrays);
for (auto &ad: ads)
{
spdlog::info(" array: name={}, data_type={}, size={}, bits={}",
mana::get<char>(ad.name), static_cast<int>(ad.data_type), ad.size,
ad.bits);
}
}
}
std::optional<AnalysisContext> make_mana(const std::string runName,
const mvlc::CrateConfig &crateConfig,
const nlohmann::json &jModuleDataSources)
{
try
{
std::map<std::string, nlohmann::json> moduleInfoByType;
for (const auto &mod_: jModuleDataSources["modules"])
moduleInfoByType[mod_["type_name"]] = mod_;
// module type name -> module info json
auto moduleInfoByType = make_module_info_by_type(jModuleDataSources);
AnalysisContext ctx;
ctx.arena = std::make_unique<mana::Arena>();
// auto arena = ctx.arena.get();
// eventIndex -> moduleIndex -> module info json
auto moduleDataSources = match_modules(crateConfig, moduleInfoByType);
auto eventDsInfo = make_event_ds_info(crateConfig, moduleDataSources);
auto arena = ctx.arena.get();
auto [runDescriptor, runDescriptorSize] = make_run_descriptor(*arena, runName, eventDsInfo);
spdlog::info(
"runDescriptor @ {}, size={}, allocations={}, capacity={}, pad_waste={}, used_size={}",
fmt::ptr(runDescriptor), runDescriptorSize, arena->allocations(), arena->capacity(),
arena->pad_waste(), arena->used());
if (runDescriptor)
dump(runDescriptor);
auto [eventStorages, eventStoragesSize] = make_event_storages(*arena, eventDsInfo);
spdlog::info(
"eventStorages @ {}, size={}, allocations={}, capacity={}, pad_waste={}, used_size={}",
fmt::ptr(&eventStorages.front()), eventStoragesSize, arena->allocations(),
arena->capacity(), arena->pad_waste(), arena->used());
for (size_t eventIndex = 0; eventIndex < eventStorages.size(); ++eventIndex)
{
auto &es = eventStorages[eventIndex];
auto arrays = get_span<mana_offset_array_t>(es.data_arrays);
spdlog::info("eventStorage: event={}, arrays={}", eventIndex, arrays.size());
for (auto &array: arrays)
{
spdlog::info(" array @ {}, size_bytes={}, size={}",
fmt::ptr(mana::get<float>(array.ptr)), array.size_bytes,
element_count<float>(array));
}
}
ctx.runDescriptor = runDescriptor;
ctx.eventStorages = eventStorages;
ctx.runDescriptorBytes = runDescriptorSize;
ctx.eventStoragesBytes = eventStoragesSize;
return ctx;
}
catch (const std::exception &e)
@ -159,22 +493,6 @@ std::optional<AnalysisContext> make_mana(const mvlc::CrateConfig &crateConfig,
}
}
void event_data_callback(AnalysisContext *ctx, int crateIndex, int eventIndex,
const mvlc::readout_parser::ModuleData *moduleDataList,
unsigned moduleCount)
{
assert(moduleDataList);
std::cout << fmt::format("ReadoutEvent: ctx={}, crateId={}, eventIndex={}, moduleCount={}\n",
fmt::ptr(ctx), crateIndex, eventIndex, moduleCount);
}
void system_event_callback(AnalysisContext *ctx, int crateIndex, const u32 *header, u32 size)
{
assert(header);
std::cout << fmt::format("SystemEvent: ctx={}, crateId={}, size={}\n", fmt::ptr(ctx),
crateIndex, size);
}
size_t process_one_buffer(size_t bufferNumber, ListfileContext &listfileContext,
ParserContext &parserContext, AnalysisContext &analysisContext)
{
@ -211,23 +529,21 @@ int main(int argc, char *argv[])
if (!listfileContext)
return 1;
auto analysisContext = make_mana(listfileContext->crateConfig, jModuleDataSources);
auto analysisContext = make_mana(filename, listfileContext->crateConfig, jModuleDataSources);
if (!analysisContext)
return 1;
return 42;
auto event_data = [](void *ctx, int crateIndex, int eventIndex,
const mvlc::readout_parser::ModuleData *moduleDataList,
unsigned moduleCount)
{
event_data_callback(static_cast<AnalysisContext *>(ctx), crateIndex, eventIndex,
moduleDataList, moduleCount);
reinterpret_cast<AnalysisContext *>(ctx)->process_event_data(crateIndex, eventIndex,
moduleDataList, moduleCount);
};
auto system_event = [](void *ctx, int crateIndex, const u32 *header, u32 size)
{ system_event_callback(static_cast<AnalysisContext *>(ctx), crateIndex, header, size); };
{ reinterpret_cast<AnalysisContext *>(ctx)->process_system_event(crateIndex, header, size); };
auto parserContext =
make_parser_context(listfileContext->crateConfig, {event_data, system_event});
@ -241,14 +557,16 @@ int main(int argc, char *argv[])
const std::chrono::milliseconds ReportInterval(500);
mvlc::util::Stopwatch sw;
auto report =
[](const mvlc::util::Stopwatch sw, size_t bufferNumber, size_t totalBytesProcessed)
auto report = [&]
{
auto s = sw.get_elapsed().count() / (1000.0 * 1000.0);
auto bytesPerSecond = totalBytesProcessed / s;
auto MiBPerSecond = bytesPerSecond / (1u << 20);
std::cout << fmt::format("Processed {} buffers, {} bytes. t={} s, rate={} MiB/s\n",
bufferNumber, totalBytesProcessed, s, MiBPerSecond);
[](const mvlc::util::Stopwatch sw, size_t bufferNumber, size_t totalBytesProcessed)
{
auto s = sw.get_elapsed().count() / (1000.0 * 1000.0);
auto bytesPerSecond = totalBytesProcessed / s;
auto MiBPerSecond = bytesPerSecond / (1u << 20);
std::cout << fmt::format("Processed {} buffers, {} bytes. t={} s, rate={} MiB/s\n",
bufferNumber, totalBytesProcessed, s, MiBPerSecond);
}(sw, bufferNumber, totalBytesProcessed);
};
do
@ -260,13 +578,13 @@ int main(int argc, char *argv[])
if (auto elapsed = sw.get_interval(); elapsed >= ReportInterval)
{
report(sw, bufferNumber, totalBytesProcessed);
report();
sw.interval();
}
}
while (bytesProcessed > 0);
report(sw, bufferNumber, totalBytesProcessed);
report();
return 0;
}

68
src/tools/mana_lib.hpp Normal file
View file

@ -0,0 +1,68 @@
#ifndef AAB5E4D2_A05B_4F2F_B76A_406A5A569D55
#define AAB5E4D2_A05B_4F2F_B76A_406A5A569D55
#include <cassert>
#include <mesytec-mnode/mnode_cpp_types.h>
#include "mana_replay_api.h"
namespace mesytec::mnode::mana
{
inline bool is_null(const mana_offset_ptr_t &ptr) { return ptr.offset == 1; }
inline void set_raw(mana_offset_ptr_t &ptr, void *p)
{
assert(p != reinterpret_cast<u8 *>(&ptr) + 1);
if (p)
{
ptr.offset = reinterpret_cast<u8 *>(p) - reinterpret_cast<u8 *>(&ptr);
spdlog::info("set_raw: &ptr={}, p={}, offset={}", fmt::ptr(&ptr), fmt::ptr(p), ptr.offset);
}
else
{
ptr.offset = 1;
}
}
inline void *get(mana_offset_ptr_t &ptr)
{
if (is_null(ptr))
return nullptr;
return reinterpret_cast<u8 *>(&ptr) + ptr.offset;
}
template <typename T> T *get(mana_offset_ptr_t &ptr); // to catch unsupported types
template <typename T> T *get_(mana_offset_ptr_t &ptr, mana_data_type_t expected)
{
if (ptr.data_type != expected)
return nullptr;
return reinterpret_cast<T *>(get(ptr));
}
template <> u32 *get(mana_offset_ptr_t &ptr) { return get_<u32>(ptr, mana_uint32); }
template <> u64 *get(mana_offset_ptr_t &ptr) { return get_<u64>(ptr, mana_uint64); }
template <> s8 *get(mana_offset_ptr_t &ptr) { return get_<s8>(ptr, mana_sint8); }
template <> char *get(mana_offset_ptr_t &ptr) { return get_<char>(ptr, mana_sint8); }
template <> float *get(mana_offset_ptr_t &ptr) { return get_<float>(ptr, mana_float); }
template <> double *get(mana_offset_ptr_t &ptr) { return get_<double>(ptr, mana_double); }
inline void set(mana_offset_ptr_t &ptr, mana_data_type_t data_type, void *p)
{
ptr.data_type = data_type;
set_raw(ptr, p);
}
inline void set(mana_offset_ptr_t &ptr, std::nullptr_t) { set(ptr, mana_uint32, nullptr); }
inline void set(mana_offset_ptr_t &ptr, u32 *p) { set(ptr, mana_uint32, p); }
inline void set(mana_offset_ptr_t &ptr, u64 *p) { set(ptr, mana_uint64, p); }
inline void set(mana_offset_ptr_t &ptr, s8 *p) { set(ptr, mana_sint8, p); }
inline void set(mana_offset_ptr_t &ptr, char *p) { set(ptr, mana_sint8, p); }
inline void set(mana_offset_ptr_t &ptr, float *p) { set(ptr, mana_float, p); }
inline void set(mana_offset_ptr_t &ptr, double *p) { set(ptr, mana_double, p); }
}
#endif /* AAB5E4D2_A05B_4F2F_B76A_406A5A569D55 */

94
src/tools/mana_replay_api.h
View file

@ -20,59 +20,53 @@ extern "C"
typedef enum
{
dt_uint32,
dt_float,
} data_type_t;
mana_custom,
mana_uint8,
mana_sint8,
mana_uint16,
mana_uint32,
mana_uint64,
mana_float,
mana_double,
} mana_data_type_t;
typedef struct
{
const char *array_name;
data_type_t data_type;
size_t array_size;
size_t valid_bits; // belongs into data_type_t
} array_descriptor_t;
mana_data_type_t data_type;
ptrdiff_t offset;
} mana_offset_ptr_t;
typedef struct
{
const char *module_name;
const char *module_type;
array_descriptor_t *data_sources;
size_t data_source_count;
} module_descriptor_t;
mana_offset_ptr_t ptr;
size_t size_bytes;
} mana_offset_array_t;
typedef struct
{
const char *event_name;
module_descriptor_t *modules;
size_t module_count;
} event_descriptor_t;
typedef struct
{
const char *listfile_name;
event_descriptor_t *events;
size_t event_count;
} run_descriptor_t;
// data views - daq data
typedef struct
{
float *data;
mana_offset_ptr_t name; // mana_uint8
mana_data_type_t data_type;
size_t size;
} array_t;
size_t bits;
} mana_array_descriptor_t;
typedef struct
{
array_t *data_arrays;
size_t data_array_count;
} module_data_t;
mana_offset_array_t data_arrays; // mana_custom: mana_array_descriptor_t
size_t data_array_count;
} mana_event_descriptor_t;
typedef struct
{
module_data_t *modules;
size_t module_count;
} event_data_t;
mana_offset_ptr_t name; // mana_uint8
mana_offset_array_t events; // mana_custom: mana_event_descriptor_t
size_t event_count;
} mana_run_descriptor_t;
typedef struct
{
mana_offset_array_t data_arrays; // mana_custom: mana_offset_array_t
} mana_event_data_t;
#define MANA_DEFINE_PLUGIN_INIT(name) \
void *name()
@ -81,19 +75,33 @@ typedef struct
void name(void *context)
#define MANA_DEFINE_PLUGIN_BEGIN_RUN(name) \
void name(void *context, run_descriptor_t run_descriptor)
void name(void *context, mana_run_descriptor_t *run_descriptor)
#define MANA_DEFINE_PLUGIN_END_RUN(name) \
void name(void *context, run_descriptor_t run_descriptor)
void name(void *context, mana_run_descriptor_t *run_descriptor)
#define MANA_DEFINE_PLUGIN_EVENT_DATA(name) \
void name(void *context, int eventIndex, event_data_t event_data)
void name(void *context, int eventIndex, mana_event_data_t *event_data)
typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_init_t);
#define MANA_DEFINE_PLUGIN_SYSTEM_EVENT(name) \
void name(void *context, const uint32_t *data, size_t size)
typedef MANA_DEFINE_PLUGIN_INIT(mana_init_t);
typedef MANA_DEFINE_PLUGIN_SHUTDOWN(mana_shutdown_t);
typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_begin_run_t);
typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_end_run_t);
typedef MANA_DEFINE_PLUGIN_EVENT_DATA(mana_event_data_t);
typedef MANA_DEFINE_PLUGIN_END_RUN(mana_end_run_t);
typedef MANA_DEFINE_PLUGIN_EVENT_DATA(mana_process_event_t);
typedef MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_process_system_event_t);
typedef struct
{
mana_init_t *init;
mana_shutdown_t *shutdown;
mana_begin_run_t *begin_run;
mana_end_run_t *end_run;
mana_process_event_t *process_event;
mana_process_system_event_t *process_system_event;
} mana_plugin_t;
}