begin work on a mini analysis replay tool

src/tools/CMakeLists.txt

@@ -22,6 +22,12 @@ add_mnode_dev_executable(mesy_nng_push_pull_main)
 add_mnode_dev_executable(mesy_nng_pub_producer)
 add_mnode_dev_executable(mesy_nng_sub_consumer)
 
+find_package(Boost COMPONENTS filesystem system)
+if (Boost_FOUND)
+    add_mnode_dev_executable(mana_auto_replay)
+    target_link_libraries(mana_auto_replay PRIVATE nlohmann_json::nlohmann_json mnode::resources Boost::filesystem Boost::system)
+endif()
+
 add_mnode_proto_dev_executable(mnode_proto_test1)
 add_mnode_proto_dev_executable(mnode_proto_ping_client)
 add_mnode_proto_dev_executable(mnode_proto_ping_server)

src/tools/mana_auto_replay.cc

@@ -0,0 +1,180 @@
+// mana - mnode analysis / mini analysis
+// usage: mana_auto_replay <zipfile>
+//
+// open the file
+// read preamble
+// create crateconfig from preamble data
+//
+// setup:
+// for each event in the config:
+//   for each module:
+//     check for module type in meta data
+//     find module type in vme_module_data_sources.json
+//     for each filter in the modules data_sources:
+//       calculate array size (use float as the storage type for now)
+//       reserve buffer space for the array
+//
+// replay:
+// for each event in the listfile:
+//   for each module in the event:
+//     locate the list of filters for the crate, event and module index triplet
+//     for each filter:
+//       match it against every word in the module data
+//       if a word matches extract address and value
+//       store value in the reserved array buffer
+//
+//    -> extracted u32 data is stored in the buffer space for this event
+//    what now?
+//      - histogram
+//      - make arrays available to python and test numpy
+//      - crrate root histograms and accumulate
+//      - create root trees or the new rntuples(?)
+//    -> want plugins. similar to the mvme listfile_reader but on analysis data
+
+#include <cmrc/cmrc.hpp> // mnode::resources
+#include <mesytec-mvlc/mesytec-mvlc.h>
+#include <mesytec-mnode/mnode_cpp_types.h>
+#include <nlohmann/json.hpp>
+#include <list>
+#include "mana_replay_api.h"
+
+CMRC_DECLARE(mnode::resources);
+
+using namespace mesytec;
+using namespace mesytec::mnode;
+
+namespace arena
+{
+
+// source: https://blog.xoria.org/rounding-up/
+inline int64_t round_up(int64_t n, int64_t p)
+{
+    int64_t mask = p - 1;
+    return (n + mask) & ~mask;
+}
+
+struct Arena
+{
+    struct Segment
+    {
+        std::vector<u8> data;
+        size_t used;
+
+        explicit Segment(size_t size)
+            : data(size)
+            , used(0)
+        {
+        }
+        inline size_t free() const { return data.size() - used; }
+        inline u8 *cur() { return data.data() + used; }
+    };
+
+    std::list<Segment> segments;
+    size_t current = 0;
+
+    inline Segment *current_segment()
+    {
+        if (current < segments.size())
+        {
+            auto it = segments.begin();
+            std::advance(it, current);
+            return &(*it);
+        }
+
+        return nullptr;
+    }
+
+    inline u8 *push_size(size_t required)
+    {
+        required = round_up(required, 16);
+
+        if (auto segment = current_segment(); !segment || segment->free() < required)
+        {
+            // could waste segments if required > default size of 1u << 20 but that's fine for now
+            segments.emplace_back(
+                Arena::Segment(std::max(required, static_cast<size_t>(1u << 20))));
+            current = segments.size() - 1;
+        }
+
+        auto segment = current_segment();
+        assert(segment);
+        assert(segment->free() >= required);
+        u8 *result = segment->cur();
+        std::memset(result, 0, required);
+        segment->used += required;
+        return result;
+    }
+
+    template<typename T> T *push_t(size_t count = 1)
+    {
+        static_assert(std::is_trivial<T>::value, "T must be a trivial type");
+        return reinterpret_cast<T *>(push_size(sizeof(T) * count));
+    }
+};
+
+}; // namespace arena
+
+struct ListfileContext
+{
+    mvlc::listfile::ZipReader zipReader;
+    mvlc::listfile::ReadHandle *readHandle;
+    mvlc::listfile::ListfileReaderHelper readerHelper;
+    mvlc::CrateConfig crateConfig;
+};
+
+std::optional<ListfileContext> make_listfile_context(const std::string &filename)
+{
+    try
+    {
+        ListfileContext ctx{};
+        ctx.zipReader.openArchive(filename);
+        ctx.readHandle = ctx.zipReader.openEntry(ctx.zipReader.firstListfileEntryName());
+        ctx.readerHelper = mvlc::listfile::make_listfile_reader_helper(ctx.readHandle);
+        auto configData = ctx.readerHelper.preamble.findCrateConfig();
+        if (!configData)
+        {
+            std::cerr << fmt::format("No MVLC crate config found in {}\n", filename);
+            return {};
+        }
+
+        ctx.crateConfig = mvlc::crate_config_from_yaml(configData->contentsToString());
+        std::cout << fmt::format("Found MVLC crate config in {}\n", filename);
+        for (size_t i=0; i<ctx.crateConfig.stacks.size(); ++i)
+        {
+            std::cout << fmt::format("event[{}] {}:\n", i, ctx.crateConfig.stacks[i].getName());
+            size_t mi=0;
+            for (const auto &module_: ctx.crateConfig.stacks[i].getGroups())
+            {
+                auto meta = fmt::format("{}", fmt::join(module_.meta, ", "));
+                std::cout << fmt::format("  module[{}]: {}, {}\n", mi++, module_.name, meta);
+            }
+        }
+
+        return ctx;
+    }
+    catch (const std::exception &e)
+    {
+        std::cerr << fmt::format("Error: {}\n", e.what());
+        return {};
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    auto f = cmrc::mnode::resources::get_filesystem().open("data/vme_module_data_sources.json");
+    const auto jModuleDataSources = nlohmann::json::parse(f.begin(), f.end());
+
+    if (argc < 2)
+    {
+        std::cout << fmt::format("Usage: {} <listfile.zip>\n", argv[0]);
+        return 1;
+    }
+
+    std::string filename = argv[1];
+    auto listfileContext = make_listfile_context(filename);
+
+    if (!listfileContext)
+        return 1;
+
+    return 0;
+}