florian/mesytec-mnode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev' of fl:~/var/git/mesytec-mnode into dev

Browse source
This commit is contained in:
Florian Lüke 2024-12-25 06:27:20 +01:00
commit f2c36fec77
34 changed files with 5117 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
CMakeLists.txt
View file

@ -7,6 +7,8 @@ if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
set(MESYTEC_MNODE_MAIN_PROJECT ON) set(MESYTEC_MNODE_MAIN_PROJECT ON)
endif() endif()
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
## Create binaries in the root of the build directory ## Create binaries in the root of the build directory
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
@ -15,6 +17,30 @@ set(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "ON")
set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD_REQUIRED ON)
add_subdirectory(proto) option(MNODE_BUILD_TESTS "Build mnode test binaries" ${MESYTEC_MNODE_MAIN_PROJECT})
if (MNODE_BUILD_TESTS)
find_package(GTest CONFIG)
if (NOT GTest_FOUND)
include(FetchContent)
FetchContent_Declare(googletest URL https://github.com/google/googletest/archive/refs/tags/v1.15.2.zip)
# For Windows: Prevent overriding the parent project's compiler/linker settings
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
FetchContent_MakeAvailable(googletest)
endif()
message("-- mnode: Building tests")
include(CTest)
enable_testing()
endif()
include(CMakeRC)
cmrc_add_resource_library(mnode-resources
ALIAS mnode::resources NAMESPACE mnode::resources
data/vme_base_configs.json
data/vme_module_data_sources.json
)
add_subdirectory(external) add_subdirectory(external)
add_subdirectory(proto)
add_subdirectory(src) add_subdirectory(src)

645
cmake/CMakeRC.cmake Normal file
View file

@ -0,0 +1,645 @@
# Source: https://github.com/vector-of-bool/cmrc
# This block is executed when generating an intermediate resource file, not when
# running in CMake configure mode
if(_CMRC_GENERATE_MODE)
# Read in the digits
file(READ "${INPUT_FILE}" bytes HEX)
# Format each pair into a character literal. Heuristics seem to favor doing
# the conversion in groups of five for fastest conversion
string(REGEX REPLACE "(..)(..)(..)(..)(..)" "'\\\\x\\1','\\\\x\\2','\\\\x\\3','\\\\x\\4','\\\\x\\5'," chars "${bytes}")
# Since we did this in groups, we have some leftovers to clean up
string(LENGTH "${bytes}" n_bytes2)
math(EXPR n_bytes "${n_bytes2} / 2")
math(EXPR remainder "${n_bytes} % 5") # <-- '5' is the grouping count from above
set(cleanup_re "$")
set(cleanup_sub )
while(remainder)
set(cleanup_re "(..)${cleanup_re}")
set(cleanup_sub "'\\\\x\\${remainder}',${cleanup_sub}")
math(EXPR remainder "${remainder} - 1")
endwhile()
if(NOT cleanup_re STREQUAL "$")
string(REGEX REPLACE "${cleanup_re}" "${cleanup_sub}" chars "${chars}")
endif()
string(CONFIGURE [[
namespace { const char file_array[] = { @chars@ 0 }; }
namespace cmrc { namespace @NAMESPACE@ { namespace res_chars {
extern const char* const @SYMBOL@_begin = file_array;
extern const char* const @SYMBOL@_end = file_array + @n_bytes@;
}}}
]] code)
file(WRITE "${OUTPUT_FILE}" "${code}")
# Exit from the script. Nothing else needs to be processed
return()
endif()
set(_version 2.0.0)
cmake_minimum_required(VERSION 3.12)
include(CMakeParseArguments)
if(COMMAND cmrc_add_resource_library)
if(NOT DEFINED _CMRC_VERSION OR NOT (_version STREQUAL _CMRC_VERSION))
message(WARNING "More than one CMakeRC version has been included in this project.")
endif()
# CMakeRC has already been included! Don't do anything
return()
endif()
set(_CMRC_VERSION "${_version}" CACHE INTERNAL "CMakeRC version. Used for checking for conflicts")
set(_CMRC_SCRIPT "${CMAKE_CURRENT_LIST_FILE}" CACHE INTERNAL "Path to CMakeRC script")
function(_cmrc_normalize_path var)
set(path "${${var}}")
file(TO_CMAKE_PATH "${path}" path)
while(path MATCHES "//")
string(REPLACE "//" "/" path "${path}")
endwhile()
string(REGEX REPLACE "/+$" "" path "${path}")
set("${var}" "${path}" PARENT_SCOPE)
endfunction()
get_filename_component(_inc_dir "${CMAKE_BINARY_DIR}/_cmrc/include" ABSOLUTE)
set(CMRC_INCLUDE_DIR "${_inc_dir}" CACHE INTERNAL "Directory for CMakeRC include files")
# Let's generate the primary include file
file(MAKE_DIRECTORY "${CMRC_INCLUDE_DIR}/cmrc")
set(hpp_content [==[
#ifndef CMRC_CMRC_HPP_INCLUDED
#define CMRC_CMRC_HPP_INCLUDED
#include <cassert>
#include <functional>
#include <iterator>
#include <list>
#include <map>
#include <mutex>
#include <string>
#include <system_error>
#include <type_traits>
#if !(defined(__EXCEPTIONS) || defined(__cpp_exceptions) || defined(_CPPUNWIND) || defined(CMRC_NO_EXCEPTIONS))
#define CMRC_NO_EXCEPTIONS 1
#endif
namespace cmrc { namespace detail { struct dummy; } }
#define CMRC_DECLARE(libid) \
namespace cmrc { namespace detail { \
struct dummy; \
static_assert(std::is_same<dummy, ::cmrc::detail::dummy>::value, "CMRC_DECLARE() must only appear at the global namespace"); \
} } \
namespace cmrc { namespace libid { \
cmrc::embedded_filesystem get_filesystem(); \
} } static_assert(true, "")
namespace cmrc {
class file {
const char* _begin = nullptr;
const char* _end = nullptr;
public:
using iterator = const char*;
using const_iterator = iterator;
iterator begin() const noexcept { return _begin; }
iterator cbegin() const noexcept { return _begin; }
iterator end() const noexcept { return _end; }
iterator cend() const noexcept { return _end; }
std::size_t size() const { return static_cast<std::size_t>(std::distance(begin(), end())); }
file() = default;
file(iterator beg, iterator end) noexcept : _begin(beg), _end(end) {}
};
class directory_entry;
namespace detail {
class directory;
class file_data;
class file_or_directory {
union _data_t {
class file_data* file_data;
class directory* directory;
} _data;
bool _is_file = true;
public:
explicit file_or_directory(file_data& f) {
_data.file_data = &f;
}
explicit file_or_directory(directory& d) {
_data.directory = &d;
_is_file = false;
}
bool is_file() const noexcept {
return _is_file;
}
bool is_directory() const noexcept {
return !is_file();
}
const directory& as_directory() const noexcept {
assert(!is_file());
return *_data.directory;
}
const file_data& as_file() const noexcept {
assert(is_file());
return *_data.file_data;
}
};
class file_data {
public:
const char* begin_ptr;
const char* end_ptr;
file_data(const file_data&) = delete;
file_data(const char* b, const char* e) : begin_ptr(b), end_ptr(e) {}
};
inline std::pair<std::string, std::string> split_path(const std::string& path) {
auto first_sep = path.find("/");
if (first_sep == path.npos) {
return std::make_pair(path, "");
} else {
return std::make_pair(path.substr(0, first_sep), path.substr(first_sep + 1));
}
}
struct created_subdirectory {
class directory& directory;
class file_or_directory& index_entry;
};
class directory {
std::list<file_data> _files;
std::list<directory> _dirs;
std::map<std::string, file_or_directory> _index;
using base_iterator = std::map<std::string, file_or_directory>::const_iterator;
public:
directory() = default;
directory(const directory&) = delete;
created_subdirectory add_subdir(std::string name) & {
_dirs.emplace_back();
auto& back = _dirs.back();
auto& fod = _index.emplace(name, file_or_directory{back}).first->second;
return created_subdirectory{back, fod};
}
file_or_directory* add_file(std::string name, const char* begin, const char* end) & {
assert(_index.find(name) == _index.end());
_files.emplace_back(begin, end);
return &_index.emplace(name, file_or_directory{_files.back()}).first->second;
}
const file_or_directory* get(const std::string& path) const {
auto pair = split_path(path);
auto child = _index.find(pair.first);
if (child == _index.end()) {
return nullptr;
}
auto& entry = child->second;
if (pair.second.empty()) {
// We're at the end of the path
return &entry;
}
if (entry.is_file()) {
// We can't traverse into a file. Stop.
return nullptr;
}
// Keep going down
return entry.as_directory().get(pair.second);
}
class iterator {
base_iterator _base_iter;
base_iterator _end_iter;
public:
using value_type = directory_entry;
using difference_type = std::ptrdiff_t;
using pointer = const value_type*;
using reference = const value_type&;
using iterator_category = std::input_iterator_tag;
iterator() = default;
explicit iterator(base_iterator iter, base_iterator end) : _base_iter(iter), _end_iter(end) {}
iterator begin() const noexcept {
return *this;
}
iterator end() const noexcept {
return iterator(_end_iter, _end_iter);
}
inline value_type operator*() const noexcept;
bool operator==(const iterator& rhs) const noexcept {
return _base_iter == rhs._base_iter;
}
bool operator!=(const iterator& rhs) const noexcept {
return !(*this == rhs);
}
iterator& operator++() noexcept {
++_base_iter;
return *this;
}
iterator operator++(int) noexcept {
auto cp = *this;
++_base_iter;
return cp;
}
};
using const_iterator = iterator;
iterator begin() const noexcept {
return iterator(_index.begin(), _index.end());
}
iterator end() const noexcept {
return iterator();
}
};
inline std::string normalize_path(std::string path) {
while (path.find("/") == 0) {
path.erase(path.begin());
}
while (!path.empty() && (path.rfind("/") == path.size() - 1)) {
path.pop_back();
}
auto off = path.npos;
while ((off = path.find("//")) != path.npos) {
path.erase(path.begin() + static_cast<std::string::difference_type>(off));
}
return path;
}
using index_type = std::map<std::string, const cmrc::detail::file_or_directory*>;
} // detail
class directory_entry {
std::string _fname;
const detail::file_or_directory* _item;
public:
directory_entry() = delete;
explicit directory_entry(std::string filename, const detail::file_or_directory& item)
: _fname(filename)
, _item(&item)
{}
const std::string& filename() const & {
return _fname;
}
std::string filename() const && {
return std::move(_fname);
}
bool is_file() const {
return _item->is_file();
}
bool is_directory() const {
return _item->is_directory();
}
};
directory_entry detail::directory::iterator::operator*() const noexcept {
assert(begin() != end());
return directory_entry(_base_iter->first, _base_iter->second);
}
using directory_iterator = detail::directory::iterator;
class embedded_filesystem {
// Never-null:
const cmrc::detail::index_type* _index;
const detail::file_or_directory* _get(std::string path) const {
path = detail::normalize_path(path);
auto found = _index->find(path);
if (found == _index->end()) {
return nullptr;
} else {
return found->second;
}
}
public:
explicit embedded_filesystem(const detail::index_type& index)
: _index(&index)
{}
file open(const std::string& path) const {
auto entry_ptr = _get(path);
if (!entry_ptr || !entry_ptr->is_file()) {
#ifdef CMRC_NO_EXCEPTIONS
fprintf(stderr, "Error no such file or directory: %s\n", path.c_str());
abort();
#else
throw std::system_error(make_error_code(std::errc::no_such_file_or_directory), path);
#endif
}
auto& dat = entry_ptr->as_file();
return file{dat.begin_ptr, dat.end_ptr};
}
bool is_file(const std::string& path) const noexcept {
auto entry_ptr = _get(path);
return entry_ptr && entry_ptr->is_file();
}
bool is_directory(const std::string& path) const noexcept {
auto entry_ptr = _get(path);
return entry_ptr && entry_ptr->is_directory();
}
bool exists(const std::string& path) const noexcept {
return !!_get(path);
}
directory_iterator iterate_directory(const std::string& path) const {
auto entry_ptr = _get(path);
if (!entry_ptr) {
#ifdef CMRC_NO_EXCEPTIONS
fprintf(stderr, "Error no such file or directory: %s\n", path.c_str());
abort();
#else
throw std::system_error(make_error_code(std::errc::no_such_file_or_directory), path);
#endif
}
if (!entry_ptr->is_directory()) {
#ifdef CMRC_NO_EXCEPTIONS
fprintf(stderr, "Error not a directory: %s\n", path.c_str());
abort();
#else
throw std::system_error(make_error_code(std::errc::not_a_directory), path);
#endif
}
return entry_ptr->as_directory().begin();
}
};
}
#endif // CMRC_CMRC_HPP_INCLUDED
]==])
set(cmrc_hpp "${CMRC_INCLUDE_DIR}/cmrc/cmrc.hpp" CACHE INTERNAL "")
set(_generate 1)
if(EXISTS "${cmrc_hpp}")
file(READ "${cmrc_hpp}" _current)
if(_current STREQUAL hpp_content)
set(_generate 0)
endif()
endif()
file(GENERATE OUTPUT "${cmrc_hpp}" CONTENT "${hpp_content}" CONDITION ${_generate})
add_library(cmrc-base INTERFACE)
target_include_directories(cmrc-base INTERFACE $<BUILD_INTERFACE:${CMRC_INCLUDE_DIR}>)
# Signal a basic C++11 feature to require C++11.
target_compile_features(cmrc-base INTERFACE cxx_nullptr)
set_property(TARGET cmrc-base PROPERTY INTERFACE_CXX_EXTENSIONS OFF)
add_library(cmrc::base ALIAS cmrc-base)
function(cmrc_add_resource_library name)
set(args ALIAS NAMESPACE TYPE)
cmake_parse_arguments(ARG "" "${args}" "" "${ARGN}")
# Generate the identifier for the resource library's namespace
set(ns_re "[a-zA-Z_][a-zA-Z0-9_]*")
if(NOT DEFINED ARG_NAMESPACE)
# Check that the library name is also a valid namespace
if(NOT name MATCHES "${ns_re}")
message(SEND_ERROR "Library name is not a valid namespace. Specify the NAMESPACE argument")
endif()
set(ARG_NAMESPACE "${name}")
else()
if(NOT ARG_NAMESPACE MATCHES "${ns_re}")
message(SEND_ERROR "NAMESPACE for ${name} is not a valid C++ namespace identifier (${ARG_NAMESPACE})")
endif()
endif()
set(libname "${name}")
# Check that type is either "STATIC" or "OBJECT", or default to "STATIC" if
# not set
if(NOT DEFINED ARG_TYPE)
set(ARG_TYPE STATIC)
elseif(NOT "${ARG_TYPE}" MATCHES "^(STATIC|OBJECT)$")
message(SEND_ERROR "${ARG_TYPE} is not a valid TYPE (STATIC and OBJECT are acceptable)")
set(ARG_TYPE STATIC)
endif()
# Generate a library with the compiled in character arrays.
string(CONFIGURE [=[
#include <cmrc/cmrc.hpp>
#include <map>
#include <utility>
namespace cmrc {
namespace @ARG_NAMESPACE@ {
namespace res_chars {
// These are the files which are available in this resource library
$<JOIN:$<TARGET_PROPERTY:@libname@,CMRC_EXTERN_DECLS>,
>
}
namespace {
const cmrc::detail::index_type&
get_root_index() {
static cmrc::detail::directory root_directory_;
static cmrc::detail::file_or_directory root_directory_fod{root_directory_};
static cmrc::detail::index_type root_index;
root_index.emplace("", &root_directory_fod);
struct dir_inl {
class cmrc::detail::directory& directory;
};
dir_inl root_directory_dir{root_directory_};
(void)root_directory_dir;
$<JOIN:$<TARGET_PROPERTY:@libname@,CMRC_MAKE_DIRS>,
>
$<JOIN:$<TARGET_PROPERTY:@libname@,CMRC_MAKE_FILES>,
>
return root_index;
}
}
cmrc::embedded_filesystem get_filesystem() {
static auto& index = get_root_index();
return cmrc::embedded_filesystem{index};
}
} // @ARG_NAMESPACE@
} // cmrc
]=] cpp_content @ONLY)
get_filename_component(libdir "${CMAKE_CURRENT_BINARY_DIR}/__cmrc_${name}" ABSOLUTE)
get_filename_component(lib_tmp_cpp "${libdir}/lib_.cpp" ABSOLUTE)
string(REPLACE "\n " "\n" cpp_content "${cpp_content}")
file(GENERATE OUTPUT "${lib_tmp_cpp}" CONTENT "${cpp_content}")
get_filename_component(libcpp "${libdir}/lib.cpp" ABSOLUTE)
add_custom_command(OUTPUT "${libcpp}"
DEPENDS "${lib_tmp_cpp}" "${cmrc_hpp}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different "${lib_tmp_cpp}" "${libcpp}"
COMMENT "Generating ${name} resource loader"
)
# Generate the actual static library. Each source file is just a single file
# with a character array compiled in containing the contents of the
# corresponding resource file.
add_library(${name} ${ARG_TYPE} ${libcpp})
set_property(TARGET ${name} PROPERTY CMRC_LIBDIR "${libdir}")
set_property(TARGET ${name} PROPERTY CMRC_NAMESPACE "${ARG_NAMESPACE}")
target_link_libraries(${name} PUBLIC cmrc::base)
set_property(TARGET ${name} PROPERTY CMRC_IS_RESOURCE_LIBRARY TRUE)
if(ARG_ALIAS)
add_library("${ARG_ALIAS}" ALIAS ${name})
endif()
cmrc_add_resources(${name} ${ARG_UNPARSED_ARGUMENTS})
endfunction()
function(_cmrc_register_dirs name dirpath)
if(dirpath STREQUAL "")
return()
endif()
# Skip this dir if we have already registered it
get_target_property(registered "${name}" _CMRC_REGISTERED_DIRS)
if(dirpath IN_LIST registered)
return()
endif()
# Register the parent directory first
get_filename_component(parent "${dirpath}" DIRECTORY)
if(NOT parent STREQUAL "")
_cmrc_register_dirs("${name}" "${parent}")
endif()
# Now generate the registration
set_property(TARGET "${name}" APPEND PROPERTY _CMRC_REGISTERED_DIRS "${dirpath}")
_cm_encode_fpath(sym "${dirpath}")
if(parent STREQUAL "")
set(parent_sym root_directory)
else()
_cm_encode_fpath(parent_sym "${parent}")
endif()
get_filename_component(leaf "${dirpath}" NAME)
set_property(
TARGET "${name}"
APPEND PROPERTY CMRC_MAKE_DIRS
"static auto ${sym}_dir = ${parent_sym}_dir.directory.add_subdir(\"${leaf}\")\;"
"root_index.emplace(\"${dirpath}\", &${sym}_dir.index_entry)\;"
)
endfunction()
function(cmrc_add_resources name)
get_target_property(is_reslib ${name} CMRC_IS_RESOURCE_LIBRARY)
if(NOT TARGET ${name} OR NOT is_reslib)
message(SEND_ERROR "cmrc_add_resources called on target '${name}' which is not an existing resource library")
return()
endif()
set(options)
set(args WHENCE PREFIX)
set(list_args)
cmake_parse_arguments(ARG "${options}" "${args}" "${list_args}" "${ARGN}")
if(NOT ARG_WHENCE)
set(ARG_WHENCE ${CMAKE_CURRENT_SOURCE_DIR})
endif()
_cmrc_normalize_path(ARG_WHENCE)
get_filename_component(ARG_WHENCE "${ARG_WHENCE}" ABSOLUTE)
# Generate the identifier for the resource library's namespace
get_target_property(lib_ns "${name}" CMRC_NAMESPACE)
get_target_property(libdir ${name} CMRC_LIBDIR)
get_target_property(target_dir ${name} SOURCE_DIR)
file(RELATIVE_PATH reldir "${target_dir}" "${CMAKE_CURRENT_SOURCE_DIR}")
if(reldir MATCHES "^\\.\\.")
message(SEND_ERROR "Cannot call cmrc_add_resources in a parent directory from the resource library target")
return()
endif()
foreach(input IN LISTS ARG_UNPARSED_ARGUMENTS)
_cmrc_normalize_path(input)
get_filename_component(abs_in "${input}" ABSOLUTE)
# Generate a filename based on the input filename that we can put in
# the intermediate directory.
file(RELATIVE_PATH relpath "${ARG_WHENCE}" "${abs_in}")
if(relpath MATCHES "^\\.\\.")
# For now we just error on files that exist outside of the soure dir.
message(SEND_ERROR "Cannot add file '${input}': File must be in a subdirectory of ${ARG_WHENCE}")
continue()
endif()
if(DEFINED ARG_PREFIX)
_cmrc_normalize_path(ARG_PREFIX)
endif()
if(ARG_PREFIX AND NOT ARG_PREFIX MATCHES "/$")
set(ARG_PREFIX "${ARG_PREFIX}/")
endif()
get_filename_component(dirpath "${ARG_PREFIX}${relpath}" DIRECTORY)
_cmrc_register_dirs("${name}" "${dirpath}")
get_filename_component(abs_out "${libdir}/intermediate/${ARG_PREFIX}${relpath}.cpp" ABSOLUTE)
# Generate a symbol name relpath the file's character array
_cm_encode_fpath(sym "${relpath}")
# Get the symbol name for the parent directory
if(dirpath STREQUAL "")
set(parent_sym root_directory)
else()
_cm_encode_fpath(parent_sym "${dirpath}")
endif()
# Generate the rule for the intermediate source file
_cmrc_generate_intermediate_cpp(${lib_ns} ${sym} "${abs_out}" "${abs_in}")
target_sources(${name} PRIVATE "${abs_out}")
set_property(TARGET ${name} APPEND PROPERTY CMRC_EXTERN_DECLS
"// Pointers to ${input}"
"extern const char* const ${sym}_begin\;"
"extern const char* const ${sym}_end\;"
)
get_filename_component(leaf "${relpath}" NAME)
set_property(
TARGET ${name}
APPEND PROPERTY CMRC_MAKE_FILES
"root_index.emplace("
" \"${ARG_PREFIX}${relpath}\","
" ${parent_sym}_dir.directory.add_file("
" \"${leaf}\","
" res_chars::${sym}_begin,"
" res_chars::${sym}_end"
" )"
")\;"
)
endforeach()
endfunction()
function(_cmrc_generate_intermediate_cpp lib_ns symbol outfile infile)
add_custom_command(
# This is the file we will generate
OUTPUT "${outfile}"
# These are the primary files that affect the output
DEPENDS "${infile}" "${_CMRC_SCRIPT}"
COMMAND
"${CMAKE_COMMAND}"
-D_CMRC_GENERATE_MODE=TRUE
-DNAMESPACE=${lib_ns}
-DSYMBOL=${symbol}
"-DINPUT_FILE=${infile}"
"-DOUTPUT_FILE=${outfile}"
-P "${_CMRC_SCRIPT}"
COMMENT "Generating intermediate file for ${infile}"
)
endfunction()
function(_cm_encode_fpath var fpath)
string(MAKE_C_IDENTIFIER "${fpath}" ident)
string(MD5 hash "${fpath}")
string(SUBSTRING "${hash}" 0 4 hash)
set(${var} f_${hash}_${ident} PARENT_SCOPE)
endfunction()

2821
data/vme_base_configs.json Normal file
View file

File diff suppressed because it is too large Load diff

413
data/vme_module_data_sources.json Normal file
View file

@ -0,0 +1,413 @@
{
"modules": [
{
"data_sources": [
{
"filter": "00XXX1XX000AAAAA0O0DDDDDDDDDDDDD",
"name": "amplitude"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
}
],
"display_name": "MADC-32",
"header_filter": "0100XXXXMMMMMMMMXXXXSSSSSSSSSSSS",
"type_name": "madc32",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
},
{
"filter": "000001XXAAAAAAAA0O00DDDDDDDDDDDD",
"name": "bus0_amplitudes"
},
{
"filter": "000001XXAAAAAAAA1O00DDDDDDDDDDDD",
"name": "bus1_amplitudes"
}
],
"display_name": "MDI-2",
"header_filter": "0100XXXXMMMMMMMMXXXXSSSSSSSSSSSS",
"type_name": "mdi2",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "0001XXXXXX010AAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "0001XXXXXX000AAADDDDDDDDDDDDDDDD",
"name": "integration_long"
},
{
"filter": "0001XXXXXX110AAADDDDDDDDDDDDDDDD",
"name": "integration_short"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
}
],
"display_name": "MDLL",
"header_filter": "0100 XXXX MMMM MMMM XXXX XXSS SSSS SSSS",
"type_name": "mdll",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "0001XXXXPO00AAAADDDDDDDDDDDDDDDD",
"name": "integration_long"
},
{
"filter": "0001XXXXXX01AAAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
},
{
"filter": "0001XXXXXX10000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
},
{
"filter": "0001XXXXXX11AAAADDDDDDDDDDDDDDDD",
"name": "integration_short"
}
],
"display_name": "MDPP-16_CSI",
"header_filter": "0100XXXXMMMMMMMMXXXXXXSSSSSSSSSS",
"type_name": "mdpp16_csi",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "0001XXXXPO00AAAADDDDDDDDDDDDDDDD",
"name": "amplitude"
},
{
"filter": "0001XXXXXX01AAAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
},
{
"filter": "0001XXXXXX10000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
}
],
"display_name": "MDPP-16_PADC",
"header_filter": "0100XXXXMMMMMMMMXXXXXXSSSSSSSSSS",
"type_name": "mdpp16_padc",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "0001XXXXXX01AAAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "0001XXXXXX00AAAADDDDDDDDDDDDDDDD",
"name": "integration_long"
},
{
"filter": "0001XXXXXX11AAAADDDDDDDDDDDDDDDD",
"name": "integration_short"
},
{
"filter": "0001XXXXXX10000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
}
],
"display_name": "MDPP-16_QDC",
"header_filter": "0100XXXXMMMMMMMMXXXXXXSSSSSSSSSS",
"type_name": "mdpp16_qdc",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "0001XXXXPO00AAAADDDDDDDDDDDDDDDD",
"name": "amplitude"
},
{
"filter": "0001XXXXXX01AAAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "0001XXXXXX10000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
}
],
"display_name": "MDPP-16_RCP",
"header_filter": "0100XXXXMMMMMMMMXXXXXXSSSSSSSSSS",
"type_name": "mdpp16_rcp",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "0001XXXXPO00AAAADDDDDDDDDDDDDDDD",
"name": "amplitude"
},
{
"filter": "0001XXXXXX01AAAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "0001XXXXXX10000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
}
],
"display_name": "MDPP-16_SCP",
"header_filter": "0100XXXXMMMMMMMMXXXXXXSSSSSSSSSS",
"type_name": "mdpp16_scp",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "0001XXXPO00AAAAADDDDDDDDDDDDDDDD",
"name": "amplitude"
},
{
"filter": "0001XXXPO01AAAAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
},
{
"filter": "0001XXXXX100000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
}
],
"display_name": "MDPP-32_PADC",
"header_filter": "0100 XXXX MMMM MMMM XXXX XXSS SSSS SSSS",
"type_name": "mdpp32_padc",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "0001XXXXX01AAAAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "0001XXXXX00AAAAADDDDDDDDDDDDDDDD",
"name": "integration_long"
},
{
"filter": "0001XXXXX11AAAAADDDDDDDDDDDDDDDD",
"name": "integration_short"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
},
{
"filter": "0001XXXXX100000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
}
],
"display_name": "MDPP-32_QDC",
"header_filter": "0100 XXXX MMMM MMMM XXXX XXSS SSSS SSSS",
"type_name": "mdpp32_qdc",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "0001XXXPO00AAAAADDDDDDDDDDDDDDDD",
"name": "amplitude"
},
{
"filter": "0001XXXPO01AAAAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "0001XXXXX100000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
}
],
"display_name": "MDPP-32_SCP",
"header_filter": "0100 XXXX MMMM MMMM XXXX XXSS SSSS SSSS",
"type_name": "mdpp32_scp",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "00XXX1XX000AAAAAO000DDDDDDDDDDDD",
"name": "amplitude"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
}
],
"display_name": "MQDC-32",
"header_filter": "0100XXXXMMMMMMMMXXXXSSSSSSSSSSSS",
"type_name": "mqdc32",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "00XXX1XX000AAAAADDDDDDDDDDDDDDDD",
"name": "channel_time"
},
{
"filter": "00XXX1XX0010000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
}
],
"display_name": "MTDC-32",
"header_filter": "0100XXXXMMMMMMMMXXXXSSSSSSSSSSSS",
"type_name": "mtdc32",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
}
],
"display_name": "MVLC Event Stamper",
"header_filter": "0100XXXXMMMMMMMMXXXXXXSSSSSSSSSS",
"type_name": "mvlc_event_stamper",
"vendor_name": "mesytec",
"vme_address": "0xffff0000"
},
{
"data_sources": [
{
"filter": "0010AAAA00000000DDDDDDDDDDDDDDDD",
"name": "bus_time"
},
{
"filter": "00010000XXXXXAAAAAAADDDDDDDDDDDD",
"name": "amplitude_bus0"
},
{
"filter": "11DDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "module_timestamp"
},
{
"filter": "0001XXXXXX10000ADDDDDDDDDDDDDDDD",
"name": "trigger_time"
}
],
"display_name": "VMMR-8/16",
"header_filter": "0100 XXXX MMMM MMMM XXXX SSSS SSSS SSSS",
"type_name": "vmmr",
"vendor_name": "mesytec",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "000000AAAAAAADDDDDDDDDDDDDDDDDDD",
"name": "tdc_channels"
},
{
"filter": "10001DDDDDDDDDDDDDDDDDDDDDDDDDDD",
"name": "ext_trig_time_tag"
},
{
"filter": "01000DDDDDDDDDDDDDDDDDDDDDDXXXXX",
"name": "event_counter"
}
],
"display_name": "CAEN V1190A",
"header_filter": "1000 0LOE XXXS SSSS SSSS SSSS SSSG GGGG",
"type_name": "caen_v1190a",
"vendor_name": "CAEN",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "XXXXX000XXXAAAAAXXUODDDDDDDDDDDD",
"name": "amplitude"
},
{
"filter": "XXXXX100DDDDDDDDDDDDDDDDDDDDDDDD",
"name": "event_counter"
}
],
"display_name": "CAEN V775",
"header_filter": "XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX",
"type_name": "caen_v775",
"vendor_name": "CAEN",
"vme_address": "0x00000000"
},
{
"data_sources": [
{
"filter": "XXXXX000XXXAAAAAXXUODDDDDDDDDDDD",
"name": "amplitude"
},
{
"filter": "XXXXX100DDDDDDDDDDDDDDDDDDDDDDDD",
"name": "event_counter"
}
],
"display_name": "CAEN V785",
"header_filter": "XXXX X010 XXXX XXXX 00XX XXXX XXXX XXXX",
"type_name": "caen_v785",
"vendor_name": "CAEN",
"vme_address": "0x00000000"
}
],
"mvme_commit": "1c9e1f1b99defea1c849269386ad41f73f6aa1cc",
"mvme_version": "1.15.1.22"
}

2
external/mesytec-mvlc vendored

@ -1 +1 @@
Subproject commit cab2dd297af6fb0b91837cdec704570101519e82 Subproject commit 1c1f32dbc587747219a942aabe2a2668fab1ae82

26
include/mesytec-mnode/mnode_math.h Normal file
View file

@ -0,0 +1,26 @@
#ifndef C536E080_25D7_476D_B8E3_25912B9CFC3B
#define C536E080_25D7_476D_B8E3_25912B9CFC3B
#include <cstdint>
#include <limits>
namespace mesytec::mnode
{
// round n up to the nearest multiple of p
// source: https://blog.xoria.org/rounding-up/
inline int64_t round_up(int64_t n, int64_t p)
{
int64_t mask = p - 1;
return (n + mask) & ~mask;
}
inline double make_quiet_nan()
{
double result = std::numeric_limits<double>::quiet_NaN();
assert(((uintptr_t)(result) & 0xffffffff) == 0);
return result;
}
}
#endif /* C536E080_25D7_476D_B8E3_25912B9CFC3B */

8
include/mesytec-mnode/mnode_nng_async.h
View file

@ -12,18 +12,18 @@ struct IWork
virtual void work() = 0; virtual void work() = 0;
}; };
struct IAsyncReqWork: public IWork struct IReqWork: public IWork
{ {
virtual nng::unique_msg make_request() = 0; virtual nng::unique_msg make_request() = 0;
virtual void handle_reply(nng::unique_msg &&request, nng::unique_msg &&reply) = 0; virtual void handle_reply(nng::unique_msg &&request, nng::unique_msg &&reply) = 0;
}; };
struct IAsyncRepWork: public IWork struct IRepWork: public IWork
{ {
virtual nng::unique_msg handle_request(nng::unique_msg &&request) = 0; virtual nng::unique_msg handle_request(nng::unique_msg &&request) = 0;
}; };
class AsyncReqWork: public IAsyncReqWork class AsyncReqWork: public IReqWork
{ {
public: public:
explicit AsyncReqWork(nng_socket socket); explicit AsyncReqWork(nng_socket socket);
@ -39,7 +39,7 @@ class AsyncReqWork: public IAsyncReqWork
nng::unique_msg request_; nng::unique_msg request_;
}; };
class AsyncRepWork: public IAsyncRepWork class AsyncRepWork: public IRepWork
{ {
public: public:
explicit AsyncRepWork(nng_socket socket); explicit AsyncRepWork(nng_socket socket);

64
proto/mvlc.proto
View file

@ -29,8 +29,72 @@ message ReadRegisterResponse
optional google.rpc.Status status = 2; optional google.rpc.Status status = 2;
} }
message Buffer
{
repeated uint32 data = 1;
}
message LowLevelRequest
{
Buffer contents = 1;
}
message LowLevelResponse
{
Buffer contents = 1;
}
message CommandRequest
{
Buffer contents = 1;
};
message CommandResponse
{
Buffer contents = 1;
};
enum Pipe
{
Command = 0;
Data = 1;
}
message WriteRequest
{
Pipe pipe = 1;
bytes data = 2;
}
message WriteResponse
{
google.rpc.Status status = 1;
uint32 bytes_written = 2;
}
message ReadRequest
{
Pipe pipe = 1;
uint32 bytes_to_read = 2;
}
message ReadResponse
{
google.rpc.Status status = 1;
bytes data = 2;
}
service MvlcCoreService
{
rpc Write(WriteRequest) returns (WriteResponse);
rpc Read(ReadRequest) returns (ReadResponse);
}
service MVLCService service MVLCService
{ {
rpc LowLevelTransaction(LowLevelRequest) returns (LowLevelResponse);
rpc CommandTransaction(CommandRequest) returns (CommandResponse);
rpc ReadRegister(ReadRegisterRequest) returns (ReadRegisterResponse); rpc ReadRegister(ReadRegisterRequest) returns (ReadRegisterResponse);
rpc WriteRegister(WriteRegisterRequest) returns (WriteRegisterResponse); rpc WriteRegister(WriteRegisterRequest) returns (WriteRegisterResponse);
rpc VMERead(mesytec.mnode.proto.vme.ReadRequest) returns (mesytec.mnode.proto.vme.ReadResponse); rpc VMERead(mesytec.mnode.proto.vme.ReadRequest) returns (mesytec.mnode.proto.vme.ReadResponse);

2
proto/vme.proto
View file

@ -60,7 +60,7 @@ enum ResponseFlags
message ReadResponse message ReadResponse
{ {
repeated uint32 value = 1; repeated uint32 values = 1;
optional ResponseFlags flags = 2; optional ResponseFlags flags = 2;
optional google.rpc.Status status = 3; optional google.rpc.Status status = 3;
} }

51
src/CMakeLists.txt
View file

@ -1,10 +1,4 @@
set(MVLC_NNG_MNODE_WARN_FLAGS -Wall -Wextra -Wpedantic) set(MVLC_NNG_MNODE_WARN_FLAGS -Wall -Wextra -Wpedantic -Werror=return-type)
#find_program(CLANG_TIDY_EXECUTABLE clang-tidy)
#if (CLANG_TIDY_EXECUTABLE)
# set(CMAKE_C_CLANG_TIDY clang-tidy -p ${CMAKE_BINARY_DIR} --extra-arg=-std=c11)
# set(CMAKE_CXX_CLANG_TIDY clang-tidy -p ${CMAKE_BINARY_DIR} --extra-arg=-std=c++17)
#endif()
add_library(mesytec-mnode mnode_nng.cc mnode_nng_async.cc mnode_nng_proto.cc) add_library(mesytec-mnode mnode_nng.cc mnode_nng_async.cc mnode_nng_proto.cc)
target_include_directories(mesytec-mnode target_include_directories(mesytec-mnode
@ -12,37 +6,20 @@ target_include_directories(mesytec-mnode
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/mesytec-mnode) PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/mesytec-mnode)
target_link_libraries(mesytec-mnode PUBLIC nng mnode-proto PRIVATE spdlog) target_link_libraries(mesytec-mnode PUBLIC nng mnode-proto PRIVATE spdlog)
target_compile_features(mesytec-mnode PRIVATE cxx_std_17) target_compile_features(mesytec-mnode PRIVATE cxx_std_17)
target_compile_options(mesytec-mnode PRIVATE ${MVLC_NNG_MNODE_WARN_FLAGS})
add_library(mesytec-mnode-dev INTERFACE) add_library(mana INTERFACE)
target_link_libraries(mesytec-mnode-dev INTERFACE mesytec-mnode mesytec-mvlc) target_link_libraries(mana INTERFACE nlohmann_json::nlohmann_json mesytec-mvlc)
function(add_mnode_dev_executable name) add_subdirectory(tools)
add_executable(${name} ${name}.cc)
target_link_libraries(${name} PRIVATE mesytec-mnode-dev) if (MNODE_BUILD_TESTS)
target_compile_options(${name} PRIVATE ${MVLC_NNG_MNODE_WARN_FLAGS}) function (add_mnode_gtest name)
add_executable(test_${name} ${name}.test.cc)
target_link_libraries(test_${name} PRIVATE mesytec-mnode GTest::gtest_main spdlog)
add_test(NAME name COMMAND $<TARGET_FILE:test_${name}>)
endfunction() endfunction()
function(add_mnode_proto_dev_executable name) add_mnode_gtest(mana)
add_mnode_dev_executable(${name}) target_link_libraries(test_mana PRIVATE mana)
target_link_libraries(${name} PRIVATE mnode-proto mesytec-mvlc) endif()
endfunction()
add_mnode_dev_executable(pair_producer)
add_mnode_dev_executable(pair_consumer)
add_mnode_dev_executable(pair_inproc)
add_mnode_dev_executable(mvlc_nng_replay)
add_mnode_dev_executable(mesy_nng_pipeline_main)
add_mnode_dev_executable(mesy_nng_push_pull_main)
add_mnode_dev_executable(mesy_nng_pub_producer)
add_mnode_dev_executable(mesy_nng_sub_consumer)
add_mnode_proto_dev_executable(mnode_proto_test1)
add_mnode_proto_dev_executable(mnode_proto_ping_client)
add_mnode_proto_dev_executable(mnode_proto_ping_server)
add_mnode_proto_dev_executable(mnode_proto_rpc_ping_server)
add_mnode_proto_dev_executable(mnode_proto_rpc_ping_client)
#add_subdirectory(qt)
#unset(CMAKE_C_CLANG_TIDY)
#unset(CMAKE_CXX_CLANG_TIDY)

92
src/internal/mana_analysis.h Normal file
View file

@ -0,0 +1,92 @@
#ifndef B63F110F_BB53_46E7_AA8E_FF6BE10CAB40
#define B63F110F_BB53_46E7_AA8E_FF6BE10CAB40
#include "mana_arena.h"
#include "mana_lib.hpp"
#include <mesytec-mvlc/mesytec-mvlc.h>
namespace mesytec::mnode::mana
{
struct BitFilterExtractor
{
mvlc::util::DataFilter filter;
mvlc::util::CacheEntry fAddress;
mvlc::util::CacheEntry fValue;
mana_offset_array_t *dest;
};
struct ModuleDataStage
{
mana::Arena arena;
nlohmann::json runDescriptor;
// eventindex -> list of pointers to output offset arrays
std::vector<std::vector<mana_offset_array_t *>> eventArrayPointers;
// eventIndex, moduleIndex -> list of filters attached to the module.
std::vector<std::vector<std::vector<BitFilterExtractor>>> dataSources;
mana_sink_t sink = {};
void *sinkContext = nullptr;
};
inline ModuleDataStage make_module_data_stage(const std::string &runName,
const mvlc::CrateConfig &crateConfig,
nlohmann::json moduleDb, mana_sink_t sink,
void *sinkContext)
{
ModuleDataStage result;
return result;
}
inline void module_data_stage_begin_run(ModuleDataStage &ctx)
{
ctx.sink.begin_run(ctx.sinkContext, ctx.runDescriptor.dump().c_str());
}
inline void module_data_stage_end_run(ModuleDataStage &ctx)
{
ctx.sink.end_run(ctx.sinkContext, ctx.runDescriptor.dump().c_str());
}
inline void
module_data_stage_process_module_data(ModuleDataStage &ctx, int eventIndex,
const mvlc::readout_parser::ModuleData *moduleDataList,
unsigned moduleCount)
{
auto extract_module_data =
[](const mvlc::readout_parser::DataBlock &data, BitFilterExtractor &ex)
{
auto dest = get_span<float>(*ex.dest);
std::fill(std::begin(dest), std::end(dest), mnode::make_quiet_nan());
for (const u32 *word = data.data, *end = data.data + data.size; word < end; ++word)
{
if (mvlc::util::matches(ex.filter, *word))
{
u32 address = mvlc::util::extract(ex.fAddress, *word);
u32 value = mvlc::util::extract(ex.fValue, *word);
assert(address < dest.size());
dest[address] = value;
}
}
};
auto &eventSources = ctx.dataSources.at(eventIndex);
for (unsigned mi = 0; mi < moduleCount; ++mi)
{
auto &moduleData = moduleDataList[mi];
auto dataBlock = moduleData.hasDynamic ? dynamic_span(moduleData) : prefix_span(moduleData);
for (auto &source: eventSources.at(mi))
extract_module_data(dataBlock, source);
}
// auto arrays = ctx.eventArrayPointers.at(eventIndex);
// ctx.sink.process_event(ctx.sinkContext, eventIndex,
}
} // namespace mesytec::mnode::mana
#endif /* B63F110F_BB53_46E7_AA8E_FF6BE10CAB40 */

79
src/internal/mana_api.h Normal file
View file

@ -0,0 +1,79 @@
#ifndef A51A04C1_ABD6_4DE9_B16A_49A9DA46C67E
#define A51A04C1_ABD6_4DE9_B16A_49A9DA46C67E
#include <stdint.h>
#include <stddef.h>
#ifdef __cplusplus
extern "C"
#endif
{
typedef enum
{
mana_custom,
mana_uint8,
mana_sint8,
mana_uint16,
mana_uint32,
mana_uint64,
mana_float,
mana_double,
} mana_data_type_t;
typedef struct
{
mana_data_type_t data_type;
ptrdiff_t offset;
} mana_offset_ptr_t;
typedef struct
{
mana_offset_ptr_t ptr;
size_t size_bytes;
} mana_offset_array_t;
#define MANA_DEFINE_PLUGIN_INIT(name) \
void *name()
#define MANA_DEFINE_PLUGIN_SHUTDOWN(name) \
void name(void *context)
#define MANA_DEFINE_PLUGIN_BEGIN_RUN(name) \
void name(void *context, const char *descriptor_json)
#define MANA_DEFINE_PLUGIN_END_RUN(name) \
void name(void *context, const char *descriptor_json)
#define MANA_DEFINE_PLUGIN_EVENT_DATA(name) \
void name(void *context, uint16_t eventIndex, const mana_offset_array_t *arrays, size_t arrayCount, size_t totalBytes)
#define MANA_DEFINE_PLUGIN_SYSTEM_EVENT(name) \
void name(void *context, const uint32_t *data, size_t size)
typedef MANA_DEFINE_PLUGIN_INIT(mana_init_t);
typedef MANA_DEFINE_PLUGIN_SHUTDOWN(mana_shutdown_t);
typedef MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_begin_run_t);
typedef MANA_DEFINE_PLUGIN_END_RUN(mana_end_run_t);
typedef MANA_DEFINE_PLUGIN_EVENT_DATA(mana_process_event_t);
typedef MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_process_system_event_t);
typedef struct
{
mana_init_t *init;
mana_shutdown_t *shutdown;
mana_begin_run_t *begin_run;
mana_end_run_t *end_run;
mana_process_event_t *process_event;
mana_process_system_event_t *process_system_event;
} mana_sink_t;
// event data serialization format:
// u32 event count
// u32 total size in bytes
// u16 eventIndex, u16 array descriptor count
// array descriptors
// data arrays
}
#endif /* A51A04C1_ABD6_4DE9_B16A_49A9DA46C67E */

131
src/internal/mana_arena.h Normal file
View file

@ -0,0 +1,131 @@
#ifndef F20CF38F_7327_4608_8307_6AE058041CD5
#define F20CF38F_7327_4608_8307_6AE058041CD5
#include <cstring>
#include <list>
#include <mesytec-mnode/mnode_math.h>
#include <numeric>
namespace mesytec::mnode::mana
{
class Arena
{
public:
static constexpr size_t initial_segment_size = 1u << 20;
static constexpr size_t default_pad = 8;
static constexpr size_t default_max_segments = 1;
Arena() = default;
~Arena() = default;
Arena(Arena &&other);
Arena &operator=(Arena &&other);
// pushes at least the required amount of memory onto the arena.
// 'required' is rounded up to the nearest multiple of 'default_pad' to make
// subsequent allocations aligned.
u8 *push_size(size_t required)
{
size_t padded = round_up(required, default_pad);
size_t pad_waste = padded - required;
required = padded;
if (!segment_ || segment_->free() < required)
{
if (segment_count() < max_segments())
{
// could waste complete segments if required > default size of 1u << 20 but that's
// fine for now
segment_ = &segments_.emplace_back(
Arena::Segment(std::max(required, initial_segment_size)));
}
else
{
throw std::bad_alloc();
}
}
assert(segment_ && segment_->free() >= required);
auto result = segment_->cur();
std::memset(result, 0, required);
segment_->used += required;
++allocations_;
pad_waste_ += pad_waste;
return result;
}
template <typename T> T *push_t(size_t count = 1)
{
static_assert(std::is_trivial<T>::value, "T must be a trivial type");
return reinterpret_cast<T *>(push_size(sizeof(T) * count));
}
char *push_cstr(const std::string &str)
{
auto mem = push_size(str.size() + 1);
std::memcpy(mem, str.c_str(), str.size());
return reinterpret_cast<char *>(mem);
}
void reset()
{
std::for_each(std::begin(segments_), std::end(segments_),
[](Segment &seg) { seg.reset(); });
segment_ = segments_.empty() ? nullptr : &segments_.front();
allocations_ = 0;
pad_waste_ = 0;
}
size_t max_segments() const { return max_segments_; }
size_t allocations() const { return allocations_; }
size_t pad_waste() const { return pad_waste_; }
size_t segment_count() const { return segments_.size(); }
size_t capacity() const
{
auto accu = [](size_t sum, const Segment &seg) { return sum + seg.data.size(); };
return std::accumulate(std::begin(segments_), std::end(segments_), static_cast<size_t>(0u),
accu);
}
size_t used() const
{
auto accu = [](size_t sum, const Segment &seg) { return sum + seg.used; };
return std::accumulate(std::begin(segments_), std::end(segments_), static_cast<size_t>(0u),
accu) -
pad_waste_;
}
const u8 *cur_begin() const { return segment_ ? segment_->data.data() : nullptr; }
const u8 *cur_end() const { return segment_ ? segment_->cur() : nullptr; }
private:
Arena(const Arena &) = delete;
Arena &operator=(const Arena &) = delete;
struct Segment
{
std::vector<u8> data;
size_t used;
explicit Segment(size_t size)
: data(size)
, used(0)
{
}
size_t free() const { return data.size() - used; }
u8 *cur() { return data.data() + used; }
void reset() { used = 0; }
};
size_t max_segments_ = default_max_segments;
std::list<Segment> segments_;
Segment *segment_ = nullptr;
size_t allocations_ = 0;
size_t pad_waste_ = 0;
};
} // namespace mesytec::mnode::mana
#endif /* F20CF38F_7327_4608_8307_6AE058041CD5 */

107
src/internal/mana_lib.hpp Normal file
View file

@ -0,0 +1,107 @@
#ifndef AAB5E4D2_A05B_4F2F_B76A_406A5A569D55
#define AAB5E4D2_A05B_4F2F_B76A_406A5A569D55
#include <cassert>
#include <mesytec-mnode/mnode_cpp_types.h>
#include <mesytec-mvlc/cpp_compat.h>
#include <mesytec-mvlc/util/data_filter.h>
#include <nlohmann/json.hpp>
#include "mana_api.h"
namespace mesytec::mnode::mana
{
namespace detail
{
inline void set(mana_offset_ptr_t &ptr, void *p)
{
assert(p != reinterpret_cast<u8 *>(&ptr) + 1);
if (p)
{
ptr.offset = reinterpret_cast<u8 *>(p) - reinterpret_cast<u8 *>(&ptr);
// spdlog::info("detail::set: &ptr={}, p={}, offset={}", fmt::ptr(&ptr), fmt::ptr(p),
// ptr.offset);
}
else
{
ptr.offset = 1;
}
}
} // namespace detail
inline bool is_null(const mana_offset_ptr_t &ptr) { return ptr.offset == 1; }
inline void *get(mana_offset_ptr_t &ptr)
{
return is_null(ptr) ? nullptr : reinterpret_cast<u8 *>(&ptr) + ptr.offset;
}
template <typename T> T *get(mana_offset_ptr_t &ptr); // to catch unsupported types
template <typename T> T *get_(mana_offset_ptr_t &ptr, mana_data_type_t expected)
{
if (ptr.data_type != expected)
return nullptr;
return reinterpret_cast<T *>(get(ptr));
}
template <> u32 *get(mana_offset_ptr_t &ptr) { return get_<u32>(ptr, mana_uint32); }
template <> u64 *get(mana_offset_ptr_t &ptr) { return get_<u64>(ptr, mana_uint64); }
template <> s8 *get(mana_offset_ptr_t &ptr) { return get_<s8>(ptr, mana_sint8); }
template <> char *get(mana_offset_ptr_t &ptr) { return get_<char>(ptr, mana_sint8); }
template <> float *get(mana_offset_ptr_t &ptr) { return get_<float>(ptr, mana_float); }
template <> double *get(mana_offset_ptr_t &ptr) { return get_<double>(ptr, mana_double); }
inline void set(mana_offset_ptr_t &ptr, mana_data_type_t data_type, void *p)
{
ptr.data_type = data_type;
detail::set(ptr, p);
}
inline void set(mana_offset_ptr_t &ptr, std::nullptr_t) { set(ptr, mana_uint32, nullptr); }
inline void set(mana_offset_ptr_t &ptr, u32 *p) { set(ptr, mana_uint32, p); }
inline void set(mana_offset_ptr_t &ptr, u64 *p) { set(ptr, mana_uint64, p); }
inline void set(mana_offset_ptr_t &ptr, s8 *p) { set(ptr, mana_sint8, p); }
inline void set(mana_offset_ptr_t &ptr, char *p) { set(ptr, mana_sint8, p); }
inline void set(mana_offset_ptr_t &ptr, float *p) { set(ptr, mana_float, p); }
inline void set(mana_offset_ptr_t &ptr, double *p) { set(ptr, mana_double, p); }
template <typename T> mvlc::util::span<T> get_span(mana_offset_array_t &array)
{
auto ptr = reinterpret_cast<T *>(get(array.ptr));
auto size = array.size_bytes / sizeof(T);
return { ptr, size };
}
inline nlohmann::json make_array_descriptor(const std::string &name, mana_data_type_t data_type,
size_t size, size_t bits = 0)
{
nlohmann::json j;
j["name"] = name;
j["data_type"] = data_type;
j["size"] = size;
j["bits"] = bits;
return j;
}
inline nlohmann::json make_array_descriptor(const std::string &name, const std::string &bit_filter)
{
auto f = mvlc::util::make_filter_with_caches(bit_filter);
size_t size = 1;
if (auto c = mvlc::util::get_cache_entry(f, 'A'))
size = 1u << c->extractBits;
size_t bits = 0;
if (auto c = mvlc::util::get_cache_entry(f, 'D'))
bits = c->extractBits;
return make_array_descriptor(name, mana_float, size, bits);
}
} // namespace mesytec::mnode::mana
#endif /* AAB5E4D2_A05B_4F2F_B76A_406A5A569D55 */

110
src/mana.test.cc Normal file
View file

@ -0,0 +1,110 @@
#include <gtest/gtest.h>
#include <mesytec-mnode/mnode_cpp_types.h>
#include "internal/mana_arena.h"
#include <spdlog/spdlog.h>
#include "internal/mana_lib.hpp"
using namespace mesytec::mnode;
TEST(mnode_mana_arena, push_reset_push)
{
mana::Arena arena;
ASSERT_EQ(arena.allocations(), 0);
ASSERT_EQ(arena.pad_waste(), 0);
ASSERT_EQ(arena.segment_count(), 0);
ASSERT_EQ(arena.capacity(), 0);
ASSERT_EQ(arena.used(), 0);
for (size_t i = 0; i < 10; ++i)
{
auto mem = arena.push_size(mana::Arena::default_pad + 2);
ASSERT_NE(mem, nullptr);
}
ASSERT_EQ(arena.allocations(), 10);
ASSERT_GE(arena.pad_waste(), 0);
ASSERT_GE(arena.segment_count(), 1);
ASSERT_GE(arena.capacity(), 10 * (mana::Arena::default_pad + 2));
ASSERT_EQ(arena.used(), 10 * (mana::Arena::default_pad + 2));
arena.reset();
ASSERT_EQ(arena.allocations(), 0);
ASSERT_EQ(arena.pad_waste(), 0);
ASSERT_GE(arena.segment_count(), 1);
ASSERT_GE(arena.capacity(), 10 * (mana::Arena::default_pad + 2));
ASSERT_EQ(arena.used(), 0);
for (size_t i = 0; i < 10; ++i)
{
auto mem = arena.push_size(mana::Arena::default_pad + 2);
ASSERT_NE(mem, nullptr);
}
ASSERT_EQ(arena.allocations(), 10);
ASSERT_GE(arena.pad_waste(), 0);
ASSERT_GE(arena.segment_count(), 1);
ASSERT_GE(arena.capacity(), 10 * (mana::Arena::default_pad + 2));
ASSERT_EQ(arena.used(), 10 * (mana::Arena::default_pad + 2));
}
TEST(mnode_mana_offset_ptr, basic)
{
using namespace mana;
struct OffsetPtrTest
{
u32 a;
mana_offset_ptr_t ptr0; // &a
mana_offset_ptr_t ptr1; // &b
u32 b;
};
mana::Arena arena;
auto t = arena.push_t<OffsetPtrTest>();
t->a = 42;
t->b = 43;
set(t->ptr0, &t->a);
set(t->ptr1, &t->b);
ASSERT_EQ(get<u32>(t->ptr0), &t->a);
ASSERT_EQ(get<u32>(t->ptr1), &t->b);
ASSERT_EQ(get<double>(t->ptr0), nullptr);
ASSERT_EQ(get<double>(t->ptr1), nullptr);
}
TEST(mnode_mana_offset_ptr, strings)
{
using namespace mana;
struct OffsetPtrTest
{
mana_offset_ptr_t ptr0;
mana_offset_ptr_t ptr1;
};
mana::Arena arena;
auto t = arena.push_t<OffsetPtrTest>();
auto s0 = arena.push_cstr("hello");
auto s1 = arena.push_cstr("world");
spdlog::info("&t->ptr0={}, &t->ptr1={}", fmt::ptr(&t->ptr0), fmt::ptr(&t->ptr1));
spdlog::info("s0={} @ {}, s1={} @ {}", s0, fmt::ptr(s0), s1, fmt::ptr(s1));
set(t->ptr0, s0);
set(t->ptr1, s1);
ASSERT_EQ(get<char>(t->ptr0), std::string("hello"));
ASSERT_EQ(get<char>(t->ptr1), std::string("world"));
ASSERT_EQ(get<double>(t->ptr0), nullptr);
ASSERT_EQ(get<double>(t->ptr1), nullptr);
set(t->ptr0, nullptr);
ASSERT_EQ(get<char>(t->ptr0), nullptr);
ASSERT_EQ(get<char>(t->ptr1), std::string("world"));
}

14
src/pipeline.h
View file

@ -1,14 +0,0 @@
#ifndef CF3DAF23_18D2_419E_9DCC_3855C5A63226
#define CF3DAF23_18D2_419E_9DCC_3855C5A63226
#include <mesytec-node/mesytec_node_nng.h>
namespace mesytec::mnode::pipeline
{
}
#endif /* CF3DAF23_18D2_419E_9DCC_3855C5A63226 */

22
src/thread_name.cc
View file

@ -1,22 +0,0 @@
#include "thread_name.h"
#ifdef __linux__
#include <sys/prctl.h>
#endif
namespace mesytec::util
{
#ifdef __linux__
void set_thread_name(const char *name)
{
prctl(PR_SET_NAME,name,0,0,0);
}
#else
void set_thread_name(const char *)
{
}
#endif
}

9
src/thread_name.h
View file

@ -1,9 +0,0 @@
#ifndef CAF8B988_F0C9_475A_8E38_8789949859DB
#define CAF8B988_F0C9_475A_8E38_8789949859DB
namespace mesytec::util
{
void set_thread_name(const char *name);
}
#endif /* CAF8B988_F0C9_475A_8E38_8789949859DB */

35
src/tools/CMakeLists.txt Normal file
View file

@ -0,0 +1,35 @@
add_library(mesytec-mnode-dev INTERFACE)
target_link_libraries(mesytec-mnode-dev INTERFACE mesytec-mnode mesytec-mvlc)
target_include_directories(mesytec-mnode-dev INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
function(add_mnode_dev_executable name)
add_executable(${name} ${name}.cc)
target_link_libraries(${name} PRIVATE mesytec-mnode-dev)
target_compile_options(${name} PRIVATE ${MVLC_NNG_MNODE_WARN_FLAGS})
endfunction()
function(add_mnode_proto_dev_executable name)
add_mnode_dev_executable(${name})
target_link_libraries(${name} PRIVATE mnode-proto mesytec-mvlc)
endfunction()
add_mnode_dev_executable(pair_producer)
add_mnode_dev_executable(pair_consumer)
add_mnode_dev_executable(pair_inproc)
add_mnode_dev_executable(mvlc_nng_replay)
add_mnode_dev_executable(mesy_nng_pipeline_main)
add_mnode_dev_executable(mesy_nng_push_pull_main)
add_mnode_dev_executable(mesy_nng_pub_producer)
add_mnode_dev_executable(mesy_nng_sub_consumer)
find_package(Boost COMPONENTS filesystem system)
if (Boost_FOUND)
add_mnode_dev_executable(mana_auto_replay)
target_link_libraries(mana_auto_replay PRIVATE nlohmann_json::nlohmann_json mnode::resources Boost::filesystem Boost::system)
endif()
add_mnode_proto_dev_executable(mnode_proto_test1)
add_mnode_proto_dev_executable(mnode_proto_ping_client)
add_mnode_proto_dev_executable(mnode_proto_ping_server)
add_mnode_proto_dev_executable(mnode_proto_rpc_ping_server)
add_mnode_proto_dev_executable(mnode_proto_rpc_ping_client)

537
src/tools/mana_auto_replay.cc Normal file
View file

@ -0,0 +1,537 @@
// mana - mnode analysis / mini analysis
// usage: mana_auto_replay <zipfile>
//
// open the file
// read preamble
// create crateconfig from preamble data
//
// setup:
// for each event in the config:
// for each module:
// check for module type in meta data
// find module type in vme_module_data_sources.json
// for each filter in the modules data_sources:
// calculate array size (use float as the storage type for now)
// reserve buffer space for the array
//
// replay:
// for each event in the listfile:
// for each module in the event:
// locate the list of filters for the crate, event and module index triplet
// for each filter:
// match it against every word in the module data
// if a word matches extract address and value
// store value in the reserved array buffer
//
// -> extracted u32 data is stored in the buffer space for this event
// what now?
// - histogram
// - make arrays available to python and test numpy
// - crrate root histograms and accumulate
// - create root trees or the new rntuples(?)
// -> want plugins. similar to the mvme listfile_reader but on analysis data
#include <cmrc/cmrc.hpp> // mnode::resources
#include <mesytec-mvlc/mesytec-mvlc.h>
#include <mesytec-mnode/mnode_cpp_types.h>
#include <mesytec-mnode/mnode_math.h>
#include <nlohmann/json.hpp>
#include <list>
#include <sstream>
#include "internal/mana_arena.h"
#include "internal/mana_lib.hpp"
CMRC_DECLARE(mnode::resources);
using namespace mesytec;
using namespace mesytec::mnode;
struct ListfileContext
{
std::unique_ptr<mvlc::listfile::ZipReader> zipReader;
mvlc::listfile::ReadHandle *readHandle;
mvlc::listfile::ListfileReaderHelper readerHelper;
mvlc::CrateConfig crateConfig;
ListfileContext() = default;
ListfileContext(const ListfileContext &) = delete;
ListfileContext &operator=(const ListfileContext &) = delete;
ListfileContext(ListfileContext &&) = default;
ListfileContext &operator=(ListfileContext &&) = default;
};
std::optional<ListfileContext> make_listfile_context(const std::string &filename)
{
try
{
ListfileContext ctx;
ctx.zipReader = std::make_unique<mvlc::listfile::ZipReader>();
ctx.zipReader->openArchive(filename);
ctx.readHandle = ctx.zipReader->openEntry(ctx.zipReader->firstListfileEntryName());
ctx.readerHelper = mvlc::listfile::make_listfile_reader_helper(ctx.readHandle);
auto configData = ctx.readerHelper.preamble.findCrateConfig();
if (!configData)
{
std::cerr << fmt::format("No MVLC crate config found in {}\n", filename);
return {};
}
ctx.crateConfig = mvlc::crate_config_from_yaml(configData->contentsToString());
std::cout << fmt::format("Found MVLC crate config in {}\n", filename);
for (size_t i = 0; i < ctx.crateConfig.stacks.size(); ++i)
{
std::cout << fmt::format("event[{}] {}:\n", i, ctx.crateConfig.stacks[i].getName());
size_t mi = 0;
for (const auto &module_: ctx.crateConfig.stacks[i].getGroups())
{
auto meta = fmt::format("{}", fmt::join(module_.meta, ", "));
std::cout << fmt::format(" module[{}]: name={}, meta={{{}}}", mi++, module_.name,
meta);
if (!mvlc::produces_output(module_))
std::cout << ", (no output)";
std::cout << "\n";
}
}
return ctx;
}
catch (const std::exception &e)
{
std::cerr << fmt::format("Error: {}\n", e.what());
return {};
}
}
struct ParserContext
{
using Parser = mvlc::readout_parser::ReadoutParserState;
using Counters = mvlc::readout_parser::ReadoutParserCounters;
using Callbacks = mvlc::readout_parser::ReadoutParserCallbacks;
Parser parser;
Counters counters;
Callbacks callbacks;
mvlc::CrateConfig crateConfig;
};
std::optional<ParserContext> make_parser_context(const mvlc::CrateConfig &crateConfig,
ParserContext::Callbacks callbacks)
{
try
{
ParserContext ctx{};
ctx.parser = mvlc::readout_parser::make_readout_parser(crateConfig.stacks);
ctx.crateConfig = crateConfig;
ctx.callbacks = callbacks;
return ctx;
}
catch (const std::exception &e)
{
std::cerr << fmt::format("Error: {}\n", e.what());
return {};
}
}
MANA_DEFINE_PLUGIN_INIT(mana_default_init) { return nullptr; }
MANA_DEFINE_PLUGIN_SHUTDOWN(mana_default_shutdown) {}
MANA_DEFINE_PLUGIN_BEGIN_RUN(mana_default_begin_run) {}
MANA_DEFINE_PLUGIN_END_RUN(mana_default_end_run) {}
MANA_DEFINE_PLUGIN_EVENT_DATA(mana_default_process_event) {}
MANA_DEFINE_PLUGIN_SYSTEM_EVENT(mana_default_system_event) {}
static mana_sink_t DefaultPlugin = {.init = mana_default_init,
.shutdown = mana_default_shutdown,
.begin_run = mana_default_begin_run,
.end_run = mana_default_end_run,
.process_event = mana_default_process_event};
struct DataSource
{
mvlc::util::DataFilter filter;
mvlc::util::CacheEntry fAddress;
mvlc::util::CacheEntry fValue;
mana_offset_array_t *dest;
};
struct AnalysisContext
{
std::unique_ptr<mana::Arena> arena;
nlohmann::json runDescriptor;
std::vector<mvlc::util::span<mana_offset_array_t *>> eventArrays;
std::vector<std::vector<std::vector<DataSource>>>
dataSources; // eventIndex, moduleIndex, sourceIndex
mana_sink_t outputPlugin = DefaultPlugin;
void *outputPluginContext = nullptr;
size_t runDescriptorBytes = 0;
size_t eventStoragesBytes = 0;
};
std::map<std::string, nlohmann::json>
make_module_info_by_type(const nlohmann::json &jModuleDataSources)
{
// module type name -> module info json
std::map<std::string, nlohmann::json> moduleInfoByType;
for (const auto &mod_: jModuleDataSources["modules"])
moduleInfoByType[mod_["type_name"]] = mod_;
return moduleInfoByType;
}
std::vector<std::vector<nlohmann::json>>
match_modules(const mvlc::CrateConfig &crateConfig,
const std::map<std::string, nlohmann::json> &moduleInfoByType)
{
std::vector<std::vector<nlohmann::json>> moduleDataSources;
for (const auto &event: crateConfig.stacks)
{
std::vector<nlohmann::json> eventModuleDataSources;
for (const auto &module_: event.getGroups())
{
nlohmann::json jModule;
// match the meta data module type name against the concrete module name
for (const auto &[type_, info]: moduleInfoByType)
{
if (module_.name.find(type_) != std::string::npos)
{
spdlog::info("match: type={}, name={}", type_, module_.name);
jModule = info;
break;
}
}
if (jModule.empty())
{
spdlog::warn("No module info found for module name '{}'", module_.name);
}
eventModuleDataSources.emplace_back(jModule);
}
moduleDataSources.emplace_back(eventModuleDataSources);
}
return moduleDataSources;
}
struct DataSourceInfo
{
const std::string name;
const std::string filterString;
mvlc::util::FilterWithCaches filter;
DataSourceInfo(const std::string &name_, const std::string &filterString_)
: name(name_)
, filterString(filterString_)
, filter(mvlc::util::make_filter_with_caches(filterString))
{
}
};
std::vector<std::vector<DataSourceInfo>>
make_event_ds_info(const mvlc::CrateConfig &crateConfig,
const std::vector<std::vector<nlohmann::json>> &moduleDataSources)
{
std::vector<std::vector<DataSourceInfo>> eventDsInfo;
for (size_t eventIndex = 0; eventIndex < crateConfig.stacks.size(); ++eventIndex)
{
const auto &dataSources = moduleDataSources.at(eventIndex);
const auto &event = crateConfig.stacks.at(eventIndex);
const auto &readouts = event.getGroups();
std::vector<DataSourceInfo> dsInfo;
for (size_t moduleIndex = 0; moduleIndex < event.getGroups().size(); ++moduleIndex)
{
const auto &readout = readouts.at(moduleIndex);
const auto &ds = dataSources.at(moduleIndex);
spdlog::info("readout.name={}, ds={}", readout.name, ds.dump());
if (ds.contains("data_sources"))
{
for (const auto &filter: ds["data_sources"])
{
auto name =
fmt::format("{}.{}.{}", event.getName(), readout.name, filter["name"]);
dsInfo.emplace_back(DataSourceInfo(name, filter["filter"]));
}
}
}
eventDsInfo.emplace_back(dsInfo);
}
return eventDsInfo;
}
template <typename T>
T *push_offset_array(mana::Arena &arena, mana_offset_array_t &dest, size_t size)
{
T *ptr = arena.push_t<T>(size);
mana::set(dest.ptr, mana_custom, ptr);
dest.size_bytes = size * sizeof(T);
return ptr;
}
template <typename T>
T *push_typed_offset_array(mana::Arena &arena, mana_offset_array_t &dest, size_t size)
{
T *ptr = arena.push_t<T>(size);
mana::set(dest.ptr, ptr);
dest.size_bytes = size * sizeof(T);
return ptr;
}
template <typename T> size_t element_count(mana_offset_array_t &array)
{
return array.size_bytes / sizeof(T);
}
template <typename T> mvlc::util::span<T> get_span(mana_offset_array_t &array)
{
auto ptr = reinterpret_cast<T *>(mana::get(array.ptr));
auto size = element_count<T>(array);
return {ptr, size};
}
#if 0
std::pair<mana_run_descriptor_t *, size_t>
make_run_descriptor(mana::Arena &arena, const std::string &runName,
const std::vector<std::vector<DataSourceInfo>> &eventDsInfo)
{
auto rd = arena.push_t<mana_run_descriptor_t>();
mana::set(rd->name, arena.push_cstr(runName));
auto eds = push_offset_array<mana_event_descriptor_t>(arena, rd->events, eventDsInfo.size());
rd->event_count = eventDsInfo.size();
for (const auto &eventDataSources: eventDsInfo)
{
auto ed = eds++;
auto ads = push_offset_array<mana_array_descriptor_t>(arena, ed->data_arrays,
eventDataSources.size());
ed->data_array_count = eventDataSources.size();
for (const auto &ds: eventDataSources)
{
auto ad = ads++;
mana::set(ad->name, arena.push_cstr(ds.name));
ad->data_type = mana_float;
ad->size = 1;
if (auto c = get_cache_entry(ds.filter, 'A'); c)
ad->size = 1u << c->extractBits;
if (auto c = get_cache_entry(ds.filter, 'D'); c)
ad->bits = c->extractBits;
}
}
return {rd, arena.cur_end() - reinterpret_cast<u8 *>(rd)};
}
std::pair<mvlc::util::span<mana_event_data_t>, size_t>
make_event_storages(mana::Arena &arena, const std::vector<std::vector<DataSourceInfo>> &eventDsInfo)
{
auto eds = arena.push_t<mana_event_data_t>(eventDsInfo.size());
for (size_t eventIndex = 0; eventIndex < eventDsInfo.size(); ++eventIndex)
{
auto &ed = eds[eventIndex];
auto &dsInfo = eventDsInfo[eventIndex];
auto das = push_offset_array<mana_offset_array_t>(arena, ed.data_arrays, dsInfo.size());
for (const auto &ds: dsInfo)
{
auto da = das++;
size_t size = 1;
if (auto c = get_cache_entry(ds.filter, 'A'); c)
size = 1u << c->extractBits;
push_typed_offset_array<float>(arena, *da, size);
}
}
return {{eds, eventDsInfo.size()}, arena.cur_end() - reinterpret_cast<u8 *>(eds)};
}
void dump(mana_run_descriptor_t *rd)
{
auto eds = get_span<mana_event_descriptor_t>(rd->events);
spdlog::info("mana_run_descriptor @ {}, name={}, event_count={}", fmt::ptr(rd),
mana::get<char>(rd->name), eds.size());
for (size_t eventIndex = 0; eventIndex < rd->event_count; ++eventIndex)
{
auto &ed = eds[eventIndex];
spdlog::info(" event[{}]:", eventIndex);
auto ads = get_span<mana_array_descriptor_t>(ed.data_arrays);
for (auto &ad: ads)
{
spdlog::info(" array: name={}, data_type={}, size={}, bits={}",
mana::get<char>(ad.name), static_cast<int>(ad.data_type), ad.size,
ad.bits);
}
}
}
#endif
std::optional<AnalysisContext> make_mana(const std::string runName,
const mvlc::CrateConfig &crateConfig,
const nlohmann::json &jModuleDataSources)
{
try
{
// module type name -> module info json
auto moduleInfoByType = make_module_info_by_type(jModuleDataSources);
AnalysisContext ctx;
ctx.arena = std::make_unique<mana::Arena>();
// auto arena = ctx.arena.get();
// eventIndex -> moduleIndex -> module info json
auto moduleDataSources = match_modules(crateConfig, moduleInfoByType);
#if 0
auto eventDsInfo = make_event_ds_info(crateConfig, moduleDataSources);
auto arena = ctx.arena.get();
auto [runDescriptor, runDescriptorSize] = make_run_descriptor(*arena, runName, eventDsInfo);
spdlog::info(
"runDescriptor @ {}, size={}, allocations={}, capacity={}, pad_waste={}, used_size={}",
fmt::ptr(runDescriptor), runDescriptorSize, arena->allocations(), arena->capacity(),
arena->pad_waste(), arena->used());
if (runDescriptor)
dump(runDescriptor);
auto [eventStorages, eventStoragesSize] = make_event_storages(*arena, eventDsInfo);
spdlog::info(
"eventStorages @ {}, size={}, allocations={}, capacity={}, pad_waste={}, used_size={}",
fmt::ptr(&eventStorages.front()), eventStoragesSize, arena->allocations(),
arena->capacity(), arena->pad_waste(), arena->used());
for (size_t eventIndex = 0; eventIndex < eventStorages.size(); ++eventIndex)
{
auto &es = eventStorages[eventIndex];
auto arrays = get_span<mana_offset_array_t>(es.data_arrays);
spdlog::info("eventStorage: event={}, arrays={}", eventIndex, arrays.size());
for (auto &array: arrays)
{
spdlog::info(" array @ {}, size_bytes={}, size={}",
fmt::ptr(mana::get<float>(array.ptr)), array.size_bytes,
element_count<float>(array));
}
}
ctx.runDescriptor = runDescriptor;
ctx.eventStorages = eventStorages;
ctx.runDescriptorBytes = runDescriptorSize;
ctx.eventStoragesBytes = eventStoragesSize;
#endif
return ctx;
}
catch (const std::exception &e)
{
std::cerr << fmt::format("Error: {}\n", e.what());
return {};
}
}
size_t process_one_buffer(size_t bufferNumber, ListfileContext &listfileContext,
ParserContext &parserContext, AnalysisContext &analysisContext)
{
listfileContext.readerHelper.destBuf().clear();
auto buffer = mvlc::listfile::read_next_buffer(listfileContext.readerHelper);
if (!buffer->used())
return 0;
auto bufferView = buffer->viewU32();
parserContext.parser.userContext = &analysisContext;
mvlc::readout_parser::parse_readout_buffer(
listfileContext.readerHelper.bufferFormat, parserContext.parser, parserContext.callbacks,
parserContext.counters, bufferNumber, bufferView.data(), bufferView.size());
return buffer->used();
}
int main(int argc, char *argv[])
{
auto f = cmrc::mnode::resources::get_filesystem().open("data/vme_module_data_sources.json");
const auto jModuleDataSources = nlohmann::json::parse(f.begin(), f.end());
if (argc < 2)
{
std::cout << fmt::format("Usage: {} <listfile.zip>\n", argv[0]);
return 1;
}
std::string filename = argv[1];
auto listfileContext = make_listfile_context(filename);
if (!listfileContext)
return 1;
auto analysisContext = make_mana(filename, listfileContext->crateConfig, jModuleDataSources);
if (!analysisContext)
return 1;
auto event_data = [](void *ctx, int crateIndex, int eventIndex,
const mvlc::readout_parser::ModuleData *moduleDataList,
unsigned moduleCount)
{
// reinterpret_cast<AnalysisContext *>(ctx)->process_event_data(crateIndex, eventIndex,
// moduleDataList,
// moduleCount);
};
auto system_event = [](void *ctx, int crateIndex, const u32 *header, u32 size)
{
// reinterpret_cast<AnalysisContext *>(ctx)->process_system_event(crateIndex, header, size);
};
auto parserContext =
make_parser_context(listfileContext->crateConfig, {event_data, system_event});
if (!parserContext)
return 1;
size_t bufferNumber = 0;
size_t totalBytesProcessed = 0;
size_t bytesProcessed = 0;
const std::chrono::milliseconds ReportInterval(500);
mvlc::util::Stopwatch sw;
auto report = [&]
{
[](const mvlc::util::Stopwatch sw, size_t bufferNumber, size_t totalBytesProcessed)
{
auto s = sw.get_elapsed().count() / (1000.0 * 1000.0);
auto bytesPerSecond = totalBytesProcessed / s;
auto MiBPerSecond = bytesPerSecond / (1u << 20);
std::cout << fmt::format("Processed {} buffers, {} bytes. t={} s, rate={} MiB/s\n",
bufferNumber, totalBytesProcessed, s, MiBPerSecond);
}(sw, bufferNumber, totalBytesProcessed);
};
do
{
bytesProcessed =
process_one_buffer(bufferNumber, *listfileContext, *parserContext, *analysisContext);
totalBytesProcessed += bytesProcessed;
++bufferNumber;
if (auto elapsed = sw.get_interval(); elapsed >= ReportInterval)
{
report();
sw.interval();
}
}
while (bytesProcessed > 0);
report();
return 0;
}

0
src/mesy_nng_pipeline_main.cc → src/tools/mesy_nng_pipeline_main.cc
View file

0
src/mesy_nng_pub_producer.cc → src/tools/mesy_nng_pub_producer.cc
View file

0
src/mesy_nng_push_pull_main.cc → src/tools/mesy_nng_push_pull_main.cc
View file

0
src/mesy_nng_sub_consumer.cc → src/tools/mesy_nng_sub_consumer.cc
View file

0
src/mnode_proto_ping_client.cc → src/tools/mnode_proto_ping_client.cc
View file

0
src/mnode_proto_ping_server.cc → src/tools/mnode_proto_ping_server.cc
View file

0
src/mnode_proto_rpc_ping_client.cc → src/tools/mnode_proto_rpc_ping_client.cc
View file

63
src/mnode_proto_rpc_ping_server.cc → src/tools/mnode_proto_rpc_ping_server.cc
View file

@ -27,13 +27,13 @@ class MVLCService: public mnode::proto::mvlc::MVLCService
mnode::proto::mvlc::ReadRegisterResponse *response, mnode::proto::mvlc::ReadRegisterResponse *response,
::google::protobuf::Closure *done) override ::google::protobuf::Closure *done) override
{ {
(void)controller;
spdlog::info("MVLCService::ReadRegister"); spdlog::info("MVLCService::ReadRegister");
u32 value; u32 value;
if (auto ec = mvlc_.readRegister(request->address(), value)) if (auto ec = mvlc_.readRegister(request->address(), value))
{ {
response->mutable_status()->set_code(ec.value()); response->mutable_status()->set_code(ec.value());
response->mutable_status()->set_message(ec.message()); response->mutable_status()->set_message(ec.message());
controller->SetFailed(ec.message());
} }
else else
{ {
@ -62,14 +62,14 @@ class MVLCService: public mnode::proto::mvlc::MVLCService
ec = mvlc_.vmeBlockRead(request->address(), amod, maxTransfers, dest, fifo); ec = mvlc_.vmeBlockRead(request->address(), amod, maxTransfers, dest, fifo);
for (auto d: dest) for (auto d: dest)
{ {
response->add_value(d); response->add_values(d);
} }
} }
else else
{ {
u32 dest = 0; u32 dest = 0;
ec = mvlc_.vmeRead(request->address(), dest, amod, data_width); ec = mvlc_.vmeRead(request->address(), dest, amod, data_width);
response->add_value(dest); response->add_values(dest);
} }
done->Run(); done->Run();
@ -102,64 +102,19 @@ struct RpcClosure: public google::protobuf::Closure
struct RpcController: public google::protobuf::RpcController struct RpcController: public google::protobuf::RpcController
{ {
void Reset() override { spdlog::debug("RpcController::Reset"); } // Client-side methods ---------------------------------------------
void Reset() override {}
bool Failed() const override { return false; } bool Failed() const override { return false; }
std::string ErrorText() const override { return ""; } std::string ErrorText() const override { return {}; }
void StartCancel() override {} void StartCancel() override {}
void SetFailed(const std::string &reason) override
{ // Server-side methods ---------------------------------------------
spdlog::error("RpcController::SetFailed: reason={}", reason); void SetFailed(const std::string &) override { assert(!"SetFailed not implemented"); }
}
bool IsCanceled() const override { return false; } bool IsCanceled() const override { return false; }
void NotifyOnCancel(google::protobuf::Closure * /*callback*/) override {} void NotifyOnCancel(google::protobuf::Closure * /*callback*/) override {}
}; };
struct RpcChannel: public google::protobuf::RpcChannel
{
void CallMethod(const google::protobuf::MethodDescriptor *method,
google::protobuf::RpcController *controller,
const google::protobuf::Message *request, google::protobuf::Message *response,
google::protobuf::Closure *done) override
{
spdlog::info("RpcChannel::CallMethod: method={}, request={}, response={}",
method->full_name(), request->ShortDebugString(), response->GetTypeName());
}
};
bool serialize_proto_to_nng(const google::protobuf::MessageLite &message, nng_msg *msg)
{
auto messageSize = message.ByteSizeLong();
if (auto res = nng_msg_realloc(msg, nng_msg_len(msg) + sizeof(u32) + messageSize); res != 0)
{
spdlog::error("nng_msg_realloc: {}", nng_strerror(res));
return false;
}
*reinterpret_cast<u32 *>(nng_msg_body(msg)) = messageSize;
return message.SerializeToArray(reinterpret_cast<u8 *>(nng_msg_body(msg)) + sizeof(u32),
messageSize);
}
bool deserialize_proto_from_nng(google::protobuf::MessageLite &message, nng_msg *msg)
{
auto messageSize = mnode::nng::msg_trim_read<u32>(msg);
if (!messageSize)
return false;
if (nng_msg_len(msg) < *messageSize)
{
spdlog::error("message too short");
return false;
}
bool ret = message.ParseFromArray(reinterpret_cast<u8 *>(nng_msg_body(msg)), *messageSize);
nng_msg_trim(msg, *messageSize);
return ret;
}
struct RPCServer struct RPCServer
{ {
explicit RPCServer(nng_socket socket) explicit RPCServer(nng_socket socket)

0
src/mnode_proto_test1.cc → src/tools/mnode_proto_test1.cc
View file

0
src/mvlc_nng_replay.cc → src/tools/mvlc_nng_replay.cc
View file

0
src/pair_consumer.cc → src/tools/pair_consumer.cc
View file

2
src/pair_inproc.cc → src/tools/pair_inproc.cc
View file

@ -2,7 +2,7 @@
#include <mesytec-mvlc/mesytec-mvlc.h> #include <mesytec-mvlc/mesytec-mvlc.h>
int main(int argc, char *argv[]) int main()
{ {
spdlog::set_level(spdlog::level::info); spdlog::set_level(spdlog::level::info);
auto producerSocket = make_pair_socket(); auto producerSocket = make_pair_socket();

0
src/pair_producer.cc → src/tools/pair_producer.cc
View file

0
src/test_producer_consumer.h → src/tools/test_producer_consumer.h
View file