Reco.cxx

Go to the documentation of this file.

/* Copyright (C) 2023-2025 FIAS Frankfurt Institute for Advanced Studies, Frankfurt / Main
   SPDX-License-Identifier: GPL-3.0-only
   Authors: Felix Weiglhofer [committer], P.-A. Loizeau, Sergei Zharko */
#include "algo/global/Reco.h"
 
#include "AlgoFairloggerCompat.h"
#include "CaTrack.h"
#include "CbmDigiEvent.h"
#include "CbmYaml.h"
#include "algo/base/AuxDigiData.h"
#include "algo/base/BuildInfo.h"
#include "algo/base/Exceptions.h"
#include "algo/base/HistogramSender.h"
#include "algo/base/compat/OpenMP.h"
#include "algo/base/util/TimingsFormat.h"
#include "algo/base/util/TsUtils.h"
#include "algo/ca/TrackingChain.h"
#include "algo/detectors/bmon/Calibrate.h"
#include "algo/detectors/bmon/Hitfind.h"
#include "algo/detectors/bmon/ReadoutConfig.h"
#include "algo/detectors/bmon/Unpack.h"
#include "algo/detectors/fsd/Unpack.h"
#include "algo/detectors/much/Unpack.h"
// #include "algo/detectors/mvd/ChannelMaskSet.h" // TODO
// #include "algo/detectors/mvd/HitfinderChain.h" // TODO
#include "algo/detectors/mvd/Unpack.h"
#include "algo/detectors/rich/Unpack.h"
#include "algo/detectors/sts/ChannelMaskSet.h"
#include "algo/detectors/sts/HitfinderChain.h"
#include "algo/detectors/sts/Unpack.h"
#include "algo/detectors/tof/Calibrate.h"
#include "algo/detectors/tof/Hitfind.h"
#include "algo/detectors/tof/Unpack.h"
#include "algo/detectors/trd/Hitfind.h"
#include "algo/detectors/trd/Unpack.h"
#include "algo/detectors/trd2d/Unpack.h"
#include "algo/evbuild/Config.h"
#include "algo/evbuild/EventbuildChain.h"
#include "algo/global/ParFiles.h"
#include "algo/global/RecoParFiles.h"
#include "algo/kfp/KfpV0FinderChain.h"
#include "algo/kfp/KfpV0FinderConfig.h"
#include "algo/qa/QaManager.h"
#include "algo/qa/RecoGeneralQa.h"
#include "algo/qa/hitfind/BmonHitfindQa.h"
#include "algo/qa/hitfind/TofHitfindQa.h"
// #include "algo/qa/unpack/MvdDigiQa.h" // TODO
#include "algo/qa/unpack/StsDigiQa.h"
#include "algo/setup/RecoSetup.h"
 
#include <Monitor.hpp>
#include <System.hpp>
 
#include <xpu/host.h>
 
using namespace cbm::algo;
using fles::Subsystem;
 
namespace chron = std::chrono;
 
Reco::Reco() {}
Reco::~Reco() {}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::Validate(const Options& opts) const
{
  if (!fs::exists(opts.ParamsDir())) throw FatalError("ParamsDir does not exist: {}", opts.ParamsDir().string());
 
  bool hasOutputFile = !opts.OutputFile().empty();
  bool hasOutputType = !opts.OutputTypes().empty();
 
  if (!hasOutputFile && hasOutputType) {
    throw FatalError("Output types specified, but no output file given: -o <file> missing");
  }
 
  if (hasOutputFile && !hasOutputType) {
    throw FatalError("Output file specified, but no output types given: -O <types> missing");
  }
 
  if (!BuildInfo::WITH_ZSTD && opts.CompressArchive()) {
    throw FatalError("Archive compression enabled but compiled without Zstd: Remove --archive-compression flag");
  }
 
  if (opts.Has(Step::LocalReco) && !opts.Has(Step::Unpack)) {
    throw FatalError("Local reco can't run without unpacking: Add 'Unpack' to the reco steps");
  }
 
  if (opts.Has(Step::Tracking) && !opts.Has(Step::LocalReco)) {
    throw FatalError("Tracking can't run without local reco: Add 'LocalReco' to the reco steps");
  }
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::Validate(const ParFiles& parFiles) const
{
  // Unpack
  if (Opts().Has(Step::Unpack)) {
    const auto& readout = parFiles.readout;
    if (readout == std::nullopt) {
      throw FatalError("Unpacking was required, but cannot run without readout parameters. Please specify the "
                       "\"readout\" node in the main config \""
                       + Opts().MainConfigPath().string() + "\"");
    }
 
    // Individual readout nodes for each detector subsystem
    auto Check = [&](Subsystem subsystem, const auto& parNode) -> bool {
      if (Opts().Has(subsystem) && parNode == std::nullopt) {
        L_(error) << "Unpacking was required for " << subsystem << ", but the parameters are not found in readout node";
        return false;
      }
      return true;
    };
 
    bool valid{true};
    // FIXME: uncomment, when the parameters will be available!
    valid = Check(Subsystem::MVD, readout->mvd) && valid;
    valid = Check(Subsystem::STS, readout->sts) && valid;
    //valid = Check(Subsystem::RICH, readout->rich) && valid;
    valid = Check(Subsystem::MUCH, readout->much) && valid;
    valid = Check(Subsystem::TRD, readout->trd) && valid;
    valid = Check(Subsystem::TOF, readout->tof) && valid;
    valid = Check(Subsystem::BMON, readout->bmon) && valid;
    valid = Check(Subsystem::TRD2D, readout->trd2d) && valid;
    //valid = Check(Subsystem::FSD, readout->fsd) && valid;
 
    if (!valid) {
      throw FatalError("Some of the readout parameters were not provided for required unpacking steps (see errors "
                       "above). Please specify the parameters in "
                       + Opts().MainConfigPath().string() + ". Stop execution.");
    }
  }
 
  // Digi-timeslice reconstruction
  if (Opts().Has(Step::LocalReco)) {
    const auto& reco = parFiles.timesliceReco;
    if (reco == std::nullopt) {
      throw FatalError("Reconstruction of digi-timeslices was required, but cannot run without readout parameters. "
                       "Please specify the \"timesliceReco\" node in the main config "
                       + Opts().MainConfigPath().string() + ". Stop execution.");
    }
 
    // Individual readout nodes for each detector subsystem
    auto Check = [&](Subsystem subsystem, const auto& parNode) -> bool {
      if (Opts().Has(subsystem) && parNode == std::nullopt) {
        L_(error) << "Local reconstruction of digi-timeslice was required for " << subsystem
                  << ", but the parameters"
                     " are not found in timesliceReco node";
        return false;
      }
      return true;
    };
 
    bool valid{true};
    valid = Check(Subsystem::STS, reco->sts) && valid;
    valid = Check(Subsystem::TRD, reco->trd) && valid;
    valid = Check(Subsystem::TRD, reco->trd2d) && valid;
    valid = Check(Subsystem::TOF, reco->tof) && valid;
    if (!valid) {
      throw FatalError(
        "Some of the parameters for local reconstruction of digi-timeslices were not provided (see errors "
        "above). Please specify the parameters in "
        + Opts().MainConfigPath().string() + ". Stop execution.");
    }
 
    if (Opts().Has(Step::Tracking) && reco->tracking == std::nullopt) {
      throw FatalError(
        "Parameters for tracking in timeslice were not provided (see errors above). Please specify the parameters in "
        + Opts().MainConfigPath().string() + ". Stop execution.");
    }
  }
 
  // NOTE: Temporary, must be redefined in https://git.cbm.gsi.de/computing/cbmroot/-/merge_requests/2094
 
  // Digi-event reconstruction: local reconstruction
  if (Opts().ReconstructDigiEvents()) {
    const auto& reco = parFiles.eventReco;
    if (reco == std::nullopt) {
      throw FatalError("Reconstruction of digi-events was required, but cannot run without readout parameters. "
                       "Please specify the \"eventReco\" node in the main config "
                       + Opts().MainConfigPath().string() + ". Stop execution.");
    }
 
    // Individual readout nodes for each detector subsystem
    auto Check = [&](Subsystem subsystem, const auto& parNode) -> bool {
      if (Opts().Has(subsystem) && parNode == std::nullopt) {
        L_(error) << "Local reconstruction of digi-events was required for " << subsystem
                  << ", but the parameters"
                     " are not found in eventReco node";
        return false;
      }
      return true;
    };
 
    bool valid{true};
    valid = Check(Subsystem::STS, reco->sts) && valid;
    valid = Check(Subsystem::TRD, reco->trd) && valid;
    valid = Check(Subsystem::TRD, reco->trd2d) && valid;
    valid = Check(Subsystem::TOF, reco->tof) && valid;
    if (reco->tracking == std::nullopt) {
      LOG(error) << "Parameters for tracking in digi-events were not found in eventReco node";
    }
    if (!valid) {
      throw FatalError("Some of the parameters for local reconstruction of digi-events were not provided (see errors "
                       "above). Please specify the parameters in "
                       + Opts().MainConfigPath().string() + ". Stop execution.");
    }
  }
 
  // Reco-setup
  // NOTE: at the moment is used in:
  //  - digi-event selector
  //  - tracking
  //  - digi-event reconstruction
  if ((Opts().Has(Step::DigiTrigger) || Opts().Has(Step::Tracking) || Opts().ReconstructDigiEvents())
      && parFiles.recoSetupPath == std::nullopt) {
    throw FatalError("digi-trigger, tracking or digi-event reconstruction were required, but the recoSetupPath node"
                     "is not provided with the config "
                     + Opts().MainConfigPath().string()
                     + ". Please specify it. "
                       "Stop execution.");
  }
}
 
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::Init(const Options& opts)
{
  if (fInitialized) throw std::runtime_error("Chain already initialized");
 
  Validate(opts);
 
  fContext.opts = opts;
  SetContext(&fContext);
 
  if (Opts().HistogramUri() != "") {
    fSender =
      std::make_shared<HistogramSender>(Opts().HistogramUri(), Opts().HistogramHwm(), Opts().CompressHistograms());
    // fContext.sender = fSender;
 
    fRunStartTimeNs = Opts().RunStart();
    if (0 == fRunStartTimeNs) {
      fRunStartTimeNs = chron::duration_cast<chron::nanoseconds>(chron::system_clock::now().time_since_epoch()).count();
      L_(info) << "Start time for run " << opts.RunId() << " not provided => using current time";
    }
  }
 
  bool bProcessQa = (fSender != nullptr) || Opts().DumpBatchQaHistograms();
  if ((fSender != nullptr) && Opts().DumpBatchQaHistograms()) {
    L_(warn) << "Storing of the histograms in the batch mode was required with option --batch-qa-output=<file>, but "
                "the histogram server is available. Since the priority of sending the histograms on the histogram "
                "server is higher, the option --batch-qa-output=<file> will be ignored.";
  }
 
  xpu::device_prop props{xpu::device::active()};
  L_(info) << "Running CBM Reco on Device '" << props.name() << "' (Using " << openmp::GetMaxThreads()
           << " OpenMP threads)";
 
  if (!opts.MonitorUri().empty()) {
    fContext.monitor = std::make_unique<cbm::Monitor>(opts.MonitorUri());
    L_(info) << "Monitoring enabled, sending to " << opts.MonitorUri();
  }
 
  //* Parameters parsing
  // TODO: wrap it into a class
  auto mainConfigNode = YAML::LoadFile(Opts().MainConfigPath().string());
 
  // Reco Params (at the moment only STS, it is to be reorganized in the next merge request)
  // FIXME: RecoParams -> HitfindingParams/LocalRecoParams/....... needs to be renamed
  fContext.recoParams.sts = yaml::Read<RecoParams::STS>(mainConfigNode["sts"]);
 
  auto parFiles = yaml::Read<ParFiles>(mainConfigNode["parFiles"]);
  Validate(parFiles);
 
  // QA instantiation
  if (bProcessQa) {
    // QA manager
    fQaManager = std::make_unique<qa::Manager>(fSender);
    fQaManager->SetContext(&fContext);
 
    // General QA
    fGeneralQa     = std::make_unique<qa::RecoGeneralQa>(fRunStartTimeNs, fSender);
    fGeneralQaData = std::make_unique<qa::GeneralQaData>();
  }
 
  // Unpackers
  if (Opts().Has(Subsystem::BMON) && Opts().Has(Step::Unpack)) {
    bmon::ReadoutSetup readoutSetup =
      yaml::ReadFromFile<bmon::ReadoutSetup>(Opts().ParamsDir() / parFiles.readout->bmon->readout);
    bmon::ReadoutConfig cfg{readoutSetup};
    fBmonUnpack = std::make_unique<bmon::Unpack>(cfg);
  }
 
  if (Opts().Has(Subsystem::MUCH) && Opts().Has(Step::Unpack)) {
    much::ReadoutSetup readoutSetup;
    bool bLoadingOk             = false;
    std::string sDefaultMessage = "MUCH params file not defined for this setup => using hardcoded values";
    if (std::nullopt != parFiles.readout->much && "" != parFiles.readout->much->readout) {
      try {
        readoutSetup = yaml::ReadFromFile<much::ReadoutSetup>(Opts().ParamsDir() / parFiles.readout->much->readout);
        bLoadingOk   = true;
      }
      catch (YAML::BadFile&) {
        bLoadingOk = false;
        // Ignore error as if file is bad or missing
        sDefaultMessage =
          "MUCH param file not found at " + parFiles.readout->much->readout + " => using hardcoded values";
      }
      catch (YAML::ParserException& err) {
        bLoadingOk = false;
        // Ignore error as if file is bad or missing
        sDefaultMessage = "Bad MUCH param file at " + parFiles.readout->much->readout + ": " + err.what()
                          + " => using hardcoded values";
      }
    }
 
    if (bLoadingOk) {
      much::ReadoutConfig cfg(readoutSetup);
      fMuchUnpack = std::make_unique<much::Unpack>(cfg);
    }
    else {
      L_(info) << sDefaultMessage;
      much::ReadoutConfig cfg{};
      fMuchUnpack = std::make_unique<much::Unpack>(cfg);
    }
  }
 
  if (Opts().Has(Subsystem::RICH) && Opts().Has(Step::Unpack)) {
    rich::ReadoutConfig cfg{};
    fRichUnpack = std::make_unique<rich::Unpack>(cfg);
  }
 
  if (Opts().Has(Subsystem::STS) && Opts().Has(Step::Unpack)) {
    sts::ReadoutSetup readoutSetup =
      yaml::ReadFromFile<sts::ReadoutSetup>(Opts().ParamsDir() / parFiles.readout->sts->readout);
    auto chanMask    = yaml::ReadFromFile<sts::ChannelMaskSet>(Opts().ParamsDir() / parFiles.readout->sts->channelMask);
    auto walkMap     = yaml::ReadFromFile<sts::WalkMap>(Opts().ParamsDir() / parFiles.readout->sts->walkMap);
    bool bCollectAux = (fSender != nullptr && Opts().CollectAuxData());
    sts::ReadoutConfig readout{readoutSetup, chanMask};
    sts::Unpack::Config cfg{.readout = readout, .walkMap = walkMap, .bCollectAuxData = bCollectAux};
    fStsUnpack = std::make_unique<sts::Unpack>(cfg);
    if (bProcessQa && Opts().Has(QaStep::UnpackSts)) {
      fStsDigiQa = std::make_unique<sts::DigiQa>(fSender);
      fStsDigiQa->SetUseAuxData(bCollectAux);
      fStsDigiQa->RegisterReadoutSetup(readoutSetup);
      fStsDigiQa->Init();
    }
  }
 
  if (Opts().Has(Subsystem::MVD) && Opts().Has(Step::Unpack)) {
    mvd::ReadoutSetup readoutSetup;
    bool bLoadingOk             = false;
    std::string sDefaultMessage = "MVD params file not defined for this setup => using hardcoded values";
    if (std::nullopt != parFiles.readout->mvd && "" != parFiles.readout->mvd->readout) {
      try {
        readoutSetup = yaml::ReadFromFile<mvd::ReadoutSetup>(Opts().ParamsDir() / parFiles.readout->mvd->readout);
        bLoadingOk   = true;
      }
      catch (YAML::BadFile&) {
        bLoadingOk = false;
        // Ignore error as if file is bad or missing
        sDefaultMessage =
          "MVD param file not found at " + parFiles.readout->mvd->readout + " => using hardcoded values";
      }
      catch (YAML::ParserException& err) {
        bLoadingOk = false;
        // Ignore error as if file is bad or missing
        sDefaultMessage =
          "Bad MVD param file at " + parFiles.readout->mvd->readout + ": " + err.what() + " => using hardcoded values";
      }
    }
 
    if (bLoadingOk) {
      // auto pixMask = yaml::ReadFromFile<mvd::ChannelMaskSet>(Opts().ParamsDir() / parFiles.mvd.chanMask);
      // mvd::ReadoutConfig readout{readoutSetup, pixMask};
      mvd::ReadoutConfig cfg(readoutSetup);
      fMvdUnpack = std::make_unique<mvd::Unpack>(cfg);
    }
    else {
      L_(info) << sDefaultMessage;
      mvd::ReadoutConfig cfg{};
      fMvdUnpack = std::make_unique<mvd::Unpack>(cfg);
    }
    // if (bProcessQa && Opts().Has(QaStep::UnpackMvd)) {
    //   fMvdDigiQa = std::make_unique<mvd::DigiQa>(fSender);
    //   fMvdDigiQa->SetUseAuxData(bCollectAux);
    //   fMvdDigiQa->RegisterReadoutSetup(readoutSetup);
    //   fMvdDigiQa->Init();
    // }
  }
 
  if (Opts().Has(Subsystem::TOF) && Opts().Has(Step::Unpack)) {
    tof::ReadoutSetup readoutSetup =
      yaml::ReadFromFile<tof::ReadoutSetup>(Opts().ParamsDir() / parFiles.readout->tof->readout);
    tof::ReadoutConfig cfg{readoutSetup};
    fTofUnpack = std::make_unique<tof::Unpack>(cfg);
  }
 
  if (Opts().Has(Subsystem::TRD) && Opts().Has(Step::Unpack)) {
    auto cfg   = yaml::ReadFromFile<trd::ReadoutConfig>(Opts().ParamsDir() / parFiles.readout->trd->readout);
    fTrdUnpack = std::make_unique<trd::Unpack>(cfg);
  }
 
  if (Opts().Has(Subsystem::TRD2D) && Opts().Has(Step::Unpack)) {
    auto setup = yaml::ReadFromFile<trd2d::ReadoutSetup>(Opts().ParamsDir() / parFiles.readout->trd2d->readout);
    auto calib = yaml::ReadFromFile<trd2d::ReadoutCalib>(Opts().ParamsDir() / parFiles.readout->trd2d->fee);
    trd2d::Unpack::Config cfg{.roSetup = setup, .roCalib = calib};
    fTrd2dUnpack = std::make_unique<trd2d::Unpack>(cfg);
  }
 
  if (Opts().Has(Subsystem::FSD) && Opts().Has(Step::Unpack)) {
    fFsdUnpack = std::make_unique<fsd::Unpack>();
  }
 
  // --- Reco setup
  std::shared_ptr<RecoSetup> pRecoSetup{nullptr};
  if (Opts().Has(Step::DigiTrigger) || Opts().Has(Step::Tracking) || Opts().ReconstructDigiEvents()) {
    pRecoSetup = std::make_shared<RecoSetup>((Opts().ParamsDir() / parFiles.recoSetupPath.value()).string());
  }
 
  // TODO: Provide a uniform validation mechanism (check, if the setup unit exists)
 
  // --- Event building
  if (Opts().Has(Step::DigiTrigger)) {
    evbuild::Config config(mainConfigNode);  // NOTE: At the moment reads only selected nodes
    fEventBuild =
      std::make_unique<evbuild::EventbuildChain>(config, (Opts().Has(QaStep::EventBuilding) ? fSender : nullptr));
    if (fQaManager != nullptr && Opts().Has(QaStep::V0Trigger)) {
      // FIXME: Replace with a common function SetTriggerQa(fQaManager)
      auto pTriggerQa = std::make_shared<evbuild::V0TriggerQa>(fQaManager);
      pTriggerQa->Init();
      fEventBuild->SetV0TriggerQa(pTriggerQa);
    }
    fEventBuild->RegisterSetup(pRecoSetup);
  }
 
  // STS Hitfinder
  if (Opts().Has(fles::Subsystem::STS) && Opts().Has(Step::LocalReco)) {
    sts::HitfinderPars hitFinderSetup =
      yaml::ReadFromFile<sts::HitfinderPars>(opts.ParamsDir() / parFiles.timesliceReco->sts->hitfinder);
    hitFinderSetup.landauTable = sts::LandauTable::FromFile(opts.ParamsDir() / "LandauWidthTable.txt");
    // FIXME (?): Does the landau width table depend on the setup? Maybe, we should store it in algo/detectors/sts?
    sts::HitfinderChainPars hitFinderPars;
    hitFinderPars.setup  = std::move(hitFinderSetup);
    hitFinderPars.memory = Params().sts.memory;
    fStsHitFinder        = std::make_unique<sts::HitfinderChain>();
    fStsHitFinder->SetContext(&fContext);
    fStsHitFinder->SetParameters(hitFinderPars);
  }
 
  // TOF Hitfinder
  if (Opts().Has(fles::Subsystem::TOF) && Opts().Has(Step::LocalReco)) {
    auto calibSetup =
      yaml::ReadFromFile<tof::CalibrateSetup>(opts.ParamsDir() / parFiles.timesliceReco->tof->calibrate);
    fTofCalibrator  = std::make_unique<tof::Calibrate>(calibSetup);
 
    auto hitfindSetup =
      yaml::ReadFromFile<tof::HitfindSetup>(opts.ParamsDir() / parFiles.timesliceReco->tof->hitfinder);
    fTofHitFinder     = std::make_unique<tof::Hitfind>(hitfindSetup);
 
    if (fQaManager != nullptr && Opts().Has(QaStep::RecoTof)) {
      fTofHitFinderQa = std::make_unique<tof::HitfindQa>(fQaManager, "TofHitfind");
      fTofHitFinderQa->InitParameters(hitfindSetup);
      fTofHitFinderQa->Init();
    }
  }
 
  if (Opts().Has(fles::Subsystem::TRD) && Opts().Has(Step::LocalReco)) {
    auto setup   = yaml::ReadFromFile<trd::HitfindSetup>(opts.ParamsDir() / parFiles.timesliceReco->trd->hitfinder);
    auto setup2d = yaml::ReadFromFile<trd::Hitfind2DSetup>(opts.ParamsDir() / parFiles.timesliceReco->trd2d->hitfinder);
    fTrdHitfind  = std::make_unique<trd::Hitfind>(setup, setup2d);
  }
 
  // Digi event reconstruction:
  {
    fbReconstructDigiEvents = Opts().ReconstructDigiEvents();
    // It makes no sence to reconstruct an event, if there is no STS, TRD or TOF
    fbReconstructDigiEvents &= Opts().Has(fles::Subsystem::STS);
    fbReconstructDigiEvents &= Opts().Has(fles::Subsystem::TRD);
    fbReconstructDigiEvents &= Opts().Has(fles::Subsystem::TOF);
    fbReconstructDigiEvents &= Opts().Has(fles::Subsystem::BMON);
  }
 
  // Tracking
  if (Opts().Has(Step::Tracking)) {
    if (fQaManager != nullptr && Opts().Has(QaStep::Tracking)) {
      fTracking = std::make_unique<TrackingChain>(ECbmRecoMode::Timeslice, fQaManager, "CaTimeslice");
    }
    else {
      fTracking = std::make_unique<TrackingChain>(ECbmRecoMode::Timeslice);
    }
    fTracking->RegisterSetup(pRecoSetup);
    fTracking->SetContext(&fContext);
    std::string caParam = (Opts().ParamsDir() / parFiles.timesliceReco->tracking->capar).string();
    fTracking->Init(caParam);
  }
 
  if (fbReconstructDigiEvents) {
    fEvSelectingMonitor.Reset();
 
    // BMON hit finding in event reconstruction
    auto bmonCalSetup =
      yaml::ReadFromFile<bmon::CalibrateSetup>(opts.ParamsDir() / parFiles.eventReco->bmon->calibrate);
    auto bmonHitSetup = yaml::ReadFromFile<bmon::HitfindSetup>(opts.ParamsDir() / parFiles.eventReco->bmon->hitfinder);
    fBmonCalibrator   = std::make_unique<bmon::Calibrate>(bmonCalSetup);
    fBmonHitFinder    = std::make_unique<bmon::Hitfind>(bmonHitSetup);
    if (fQaManager != nullptr && Opts().Has(QaStep::RecoBmon)) {
      fBmonHitFinderQa = std::make_unique<bmon::HitfindQa>(fQaManager, "BmonHitfindEvent");
      fBmonHitFinderQa->InitParameters(bmonCalSetup, bmonHitSetup);
      fBmonHitFinderQa->Init();
    }
 
    // Tracking in event reconstruction
    if (fQaManager != nullptr && Opts().Has(QaStep::Tracking)) {
      fTrackingEvent = std::make_unique<TrackingChain>(ECbmRecoMode::EventByEvent, fQaManager, "CaEvent");
    }
    else {
      fTrackingEvent = std::make_unique<TrackingChain>(ECbmRecoMode::EventByEvent);
    }
    fTrackingEvent->RegisterSetup(pRecoSetup);
    fTrackingEvent->SetContext(&fContext);
    std::string caParam = (Opts().ParamsDir() / parFiles.eventReco->tracking->capar).string();
    fTrackingEvent->Init(caParam);
 
    if (fQaManager != nullptr && Opts().Has(QaStep::V0Finder)) {
      fV0Finder = std::make_unique<V0FinderChain>(fQaManager);
    }
    else {
      fV0Finder = std::make_unique<V0FinderChain>();
    }
    fV0Finder->SetContext(&fContext);
    fV0Finder->SetBmonDefinedAddresses(fBmonHitFinder->GetDiamondAddresses());
 
    auto config = yaml::Read<kfp::V0FinderConfig>(mainConfigNode["v0finder"]);
    fV0Finder->Init(config);
  }
 
  // Initialize the QA manager
  if (fQaManager != nullptr) {
    fQaManager->Init();
  }
 
  fInitialized = true;
 
  L_(debug) << "CBM Reco finished initialization";
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
RecoResults Reco::Run(const fles::Timeslice& ts)
{
  if (!fInitialized) {
    throw std::runtime_error("Chain not initialized");
  }
 
  ProcessingMonitor procMon;
  evbuild::EventbuildChainMonitorData evbuildMonitor;
 
  RecoResults recoData;  
  RecoResults results;   
  {
    xpu::scoped_timer t_(fmt::format("TS {}", ts.index()), &procMon.time);
    xpu::t_add_bytes(ts_utils::SizeBytes(ts));
 
    L_(info) << ">>> Processing TS " << ts.index();
    xpu::set<cbm::algo::Params>(Params());
 
    DigiData digis;
    AuxDigiData auxDigis;
 
    if (Opts().Has(Step::Unpack)) {
      xpu::scoped_timer timerU("Unpack", &procMon.timeUnpack);
      xpu::t_add_bytes(ts_utils::SizeBytes(ts));
 
      std::tie(digis.fBmon, auxDigis.fBmon)   = RunUnpacker(fBmonUnpack, ts);
      std::tie(digis.fMuch, auxDigis.fMuch)   = RunUnpacker(fMuchUnpack, ts);
      std::tie(digis.fMvd, auxDigis.fMvd)     = RunUnpacker(fMvdUnpack, ts);
      std::tie(digis.fRich, auxDigis.fRich)   = RunUnpacker(fRichUnpack, ts);
      std::tie(digis.fSts, auxDigis.fSts)     = RunUnpacker(fStsUnpack, ts);
      std::tie(digis.fTof, auxDigis.fTof)     = RunUnpacker(fTofUnpack, ts);
      std::tie(digis.fTrd, auxDigis.fTrd)     = RunUnpacker(fTrdUnpack, ts);
      std::tie(digis.fTrd2d, auxDigis.fTrd2d) = RunUnpacker(fTrd2dUnpack, ts);
      std::tie(digis.fFsd, auxDigis.fFsd)     = RunUnpacker(fFsdUnpack, ts);
 
      L_(info) << "TS contains Digis: STS=" << digis.fSts.size() << " MUCH=" << digis.fMuch.size()
               << " TOF=" << digis.fTof.size() << " BMON=" << digis.fBmon.size() << " TRD=" << digis.fTrd.size()
               << " TRD2D=" << digis.fTrd2d.size() << " RICH=" << digis.fRich.size() << " MVD=" << digis.fMvd.size()
               << " FSD=" << digis.fFsd.size();
 
 
      if (fGeneralQaData) {
        fGeneralQaData->digis_nb["BMON"] = digis.fBmon.size();
        // fGeneralQaData->digis_nb["MVD"]   = digis.fMvd.size();
        fGeneralQaData->digis_nb["STS"]   = digis.fSts.size();
        fGeneralQaData->digis_nb["RICH"]  = digis.fRich.size();
        fGeneralQaData->digis_nb["MUCH"]  = digis.fMuch.size();
        fGeneralQaData->digis_nb["TRD"]   = digis.fTrd.size();
        fGeneralQaData->digis_nb["TRD2D"] = digis.fTrd2d.size();
        fGeneralQaData->digis_nb["TOF"]   = digis.fTof.size();
        fGeneralQaData->digis_nb["FSD"]   = digis.fFsd.size();
      }
 
      // --- Raw digi QAs
      if (fStsDigiQa != nullptr) {
        fStsDigiQa->RegisterDigiData(&digis.fSts);
        fStsDigiQa->RegisterAuxDigiData(&auxDigis.fSts);
        fStsDigiQa->SetTimesliceIndex(ts.index());
        fStsDigiQa->Exec();
      }
    }
 
 
    sts::HitfinderMon stsHitfinderMonitor;
    if (fStsHitFinder) {
      xpu::scoped_timer timerSTS("STS Reco", &procMon.timeSTS);
      xpu::t_add_bytes(digis.fSts.size() * sizeof(CbmStsDigi));
      bool storeClusters   = Opts().HasOutput(RecoData::Cluster);
      auto stsResults      = (*fStsHitFinder)(digis.fSts, storeClusters);
      stsHitfinderMonitor  = std::move(stsResults.monitor);
      recoData.stsHits     = stsResults.hits;
      recoData.stsClusters = std::move(stsResults.clusters);
      QueueStsRecoMetrics(stsHitfinderMonitor);
    }
 
    PartitionedVector<tof::Hit> tofHits;
    if (Opts().Has(Step::LocalReco) && Opts().Has(fles::Subsystem::TOF)) {
      xpu::scoped_timer timerTOF("TOF Reco", &procMon.timeTOF);
      xpu::t_add_bytes(digis.fTof.size() * sizeof(CbmTofDigi));
      auto [caldigis, calmonitor] = (*fTofCalibrator)(digis.fTof);
      auto nUnknownRPC            = calmonitor.fDigiCalibUnknownRPC;
      if (nUnknownRPC > 0) {
        L_(error) << "TOF Digis with unknown RPCs: " << nUnknownRPC;
      }
      auto [hits, hitmonitor, digiindices] = (*fTofHitFinder)(caldigis);
      if (fTofHitFinderQa != nullptr) {
        fTofHitFinderQa->RegisterHits(&hits);
        fTofHitFinderQa->Exec();
      }
      recoData.tofHits = std::move(hits);
      QueueTofCalibMetrics(calmonitor);
      QueueTofRecoMetrics(hitmonitor);
    }
 
    PartitionedVector<trd::Hit> trdHits;
    if (fTrdHitfind) {
      xpu::scoped_timer timerTRD("TRD Reco", &procMon.timeTRD);
      xpu::t_add_bytes(digis.fTrd.size() * sizeof(CbmTrdDigi));
      // FIXME: additional copy of digis, figure out how to pass 1d + 2d digis at once to hitfinder
      const auto& digis1d = digis.fTrd;
      const auto& digis2d = digis.fTrd2d;
      PODVector<CbmTrdDigi> allDigis{};
      allDigis.reserve(digis1d.size() + digis2d.size());
      std::copy(digis1d.begin(), digis1d.end(), std::back_inserter(allDigis));
      std::copy(digis2d.begin(), digis2d.end(), std::back_inserter(allDigis));
      auto trdResults  = (*fTrdHitfind)(allDigis);
      recoData.trdHits = std::move(std::get<0>(trdResults));
      QueueTrdRecoMetrics(std::get<1>(trdResults));
    }
 
    L_(info) << "TS contains Hits: STS=" << recoData.stsHits.NElements() << " TOF=" << recoData.tofHits.NElements()
             << " TRD=" << recoData.trdHits.NElements();
 
 
    // --- Tracking
    TrackingChain::Output_t trackingOutput{};
    if (Opts().Has(Step::Tracking)) {
      xpu::scoped_timer timerCA("CA", &procMon.timeCA);
      xpu::t_add_bytes(recoData.stsHits.NElements() * sizeof(sts::Hit));
      xpu::t_add_bytes(recoData.tofHits.NElements() * sizeof(tof::Hit));
      xpu::t_add_bytes(recoData.trdHits.NElements() * sizeof(trd::Hit));
      TrackingChain::Input_t input{
        .stsHits = recoData.stsHits,
        .tofHits = recoData.tofHits,
        .trdHits = recoData.trdHits,
      };
      trackingOutput  = fTracking->Run(input);
      recoData.tracks = std::move(trackingOutput.tracks);
      // FIXME: SZh: the hit-matching objects has to be soreted alongside with tracks
      std::sort(recoData.tracks.begin(), recoData.tracks.end(),
                [](const cbm::algo::ca::Track& track1, const cbm::algo::ca::Track& track2) {
                  return track1.fParPV.Time() < track2.fParPV.Time();
                });
      QueueTrackingMetrics(trackingOutput.monitorData);
    }
 
    // --- Event building
    std::vector<DigiEvent> events;
    if (Opts().Has(Step::DigiTrigger)) {
      auto [ev, mon] = fEventBuild->Run(digis, recoData);
      events         = std::move(ev);
      evbuildMonitor = mon;
      QueueEvbuildMetrics(evbuildMonitor);
    }
 
    // --- Reconstruct and select digi events
    if (fbReconstructDigiEvents) {
      fEvSelectingMonitor.IncrementCounter(evselect::ECounter::Timeslices);
      fEvSelectingMonitor.IncrementCounter(evselect::ECounter::EventsTotal, events.size());
      for (auto& event : events) {
        fEvSelectingMonitor.StartTimer(evselect::ETimer::EventReconstruction);
        event.fSelectionTriggers = ReconstructEvent(event);
        fEvSelectingMonitor.StopTimer(evselect::ETimer::EventReconstruction);
      }
      auto v0FinderMonitor = fV0Finder->GetMonitor();
      fEvSelectingMonitor.IncrementCounter(evselect::ECounter::LambdaCandidates,
                                           v0FinderMonitor.GetCounterValue(kfp::ECounter::KfpLambdaCandidates));
    }
 
    // --- Filter data for output
    if (Opts().HasOutput(RecoData::DigiTimeslice)) {
      results.bmonDigis  = std::move(digis.fBmon);
      results.stsDigis   = std::move(digis.fSts);
      results.muchDigis  = std::move(digis.fMuch);
      results.mvdDigis   = std::move(digis.fMvd);
      results.trd2dDigis = std::move(digis.fTrd2d);
      results.trdDigis   = std::move(digis.fTrd);
      results.tofDigis   = std::move(digis.fTof);
      results.richDigis  = std::move(digis.fRich);
      results.fsdDigis   = std::move(digis.fFsd);
    }
    if (Opts().HasOutput(RecoData::Track)) {
      results.tracks             = std::move(recoData.tracks);
      results.trackStsHitIndices = std::move(trackingOutput.stsHitIndices);
      results.trackTofHitIndices = std::move(trackingOutput.tofHitIndices);
      results.trackTrdHitIndices = std::move(trackingOutput.trdHitIndices);
    }
    if (Opts().HasOutput(RecoData::DigiEvent)) results.events = std::move(events);
    if (Opts().HasOutput(RecoData::Cluster)) results.stsClusters = std::move(recoData.stsClusters);
    if (Opts().HasOutput(RecoData::Hit)) {
      results.stsHits = std::move(recoData.stsHits);
      results.tofHits = std::move(recoData.tofHits);
      results.trdHits = std::move(recoData.trdHits);
    }
 
    // QA
    if (fSender != nullptr) {
      // Send all the histograms, collected through the timeslice
      fQaManager->SetTimesliceId(ts.index());
      fQaManager->SetTimesliceTimeInRun((ts.start_time() - fRunStartTimeNs) * 1e-9);
      fQaManager->SendHistograms();
    }
  }
  PrintTimings(procMon.time);
  if (prevTsId) {
    procMon.tsDelta = ts.index() - *prevTsId;
  }
  prevTsId = ts.index();
  QueueProcessingMetrics(procMon);
 
  // QA
  if (fSender != nullptr) {
    // Moved out of main emission as otherwise the times are not yet updated
    (*fGeneralQa)(ts, (*fGeneralQaData));
    fGeneralQaData->Reset();
 
    // Send all the histograms, collected after the timeslice
    // (timeslice index and start in tun time are identical to block 30-50 lines above)
    fQaManager->SendHistograms();
  }
 
  return results;
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::Finalize()
{
  if (fStsHitFinder) {
    fStsHitFinder->Finalize();
  }
  if (fTracking) {
    L_(info) << "Track finding in a timeslice:";
    fTracking->Finalize();
  }
  if (fTrackingEvent) {
    L_(info) << "Track finding in digi events:";
    fTrackingEvent->Finalize();
  }
  if (fbReconstructDigiEvents) {
    fV0Finder->Finalize();
    L_(info) << fEvSelectingMonitor.ToString();
  }
 
 
  if (Opts().Profiling() >= ProfilingSummary) {
    L_(info) << MakeReportSubtimers("Run Summary", fTimesliceTimesAcc) << "\n"
             << MakeReportSummary("Total", fTimesliceTimesAcc);
 
    if (Opts().TimingsFile() != "") {
      std::ofstream file(Opts().TimingsFile().string());
      file << MakeReportYaml(fTimesliceTimesAcc);
    }
  }
 
  if (fSender == nullptr && Opts().DumpBatchQaHistograms()) {
    fQaManager->WriteHistograms(Opts().BatchQaOutput().string(), Opts().CompressArchive());
  }
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::PrintTimings(xpu::timings& timings)
{
  if (Opts().CollectKernelTimes()) {
    fTimesliceTimesAcc.merge(timings);
  }
 
  if (Opts().Profiling() >= ProfilingPerTS) {
    L_(info) << MakeReportSubtimers("TS timings", timings) << "\n" << MakeReportSummary("Total", timings);
  }
  else {
    L_(info) << "TS Processing time (Wall): " << timings.wall() << " ms";
  }
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
CbmEventTriggers Reco::ReconstructEvent(const DigiEvent& digiEvent)
{
  CbmEventTriggers triggers(0);
  RecoResults recoEvent;
  //* BMON hit reconstruction
  {
    fEvSelectingMonitor.StartTimer(evselect::ETimer::BmonHitFinder);
    auto [calDigis, calMonitor]          = (*fBmonCalibrator)(digiEvent.fBmon);
    auto [hits, hitMonitor, digiIndices] = (*fBmonHitFinder)(calDigis);
    fEvSelectingMonitor.StopTimer(evselect::ETimer::BmonHitFinder);
    if (fBmonHitFinderQa != nullptr) {
      fBmonHitFinderQa->RegisterDigis(&calDigis);
      fBmonHitFinderQa->RegisterHits(&hits);
      fBmonHitFinderQa->RegisterDigiIndices(&digiIndices);
      fBmonHitFinderQa->Exec();
    }
    recoEvent.bmonHits = std::move(hits);
  }
 
  //* STS hit reconstruction
  {
    fEvSelectingMonitor.StartTimer(evselect::ETimer::StsHitFinder);
    auto stsResults = (*fStsHitFinder)(digiEvent.fSts);
    fEvSelectingMonitor.StopTimer(evselect::ETimer::StsHitFinder);
    if (stsResults.hits.NElements() < 4) {  // TODO: Provide a config for cuts (testing mode for now)
      fEvSelectingMonitor.IncrementCounter(evselect::ECounter::EventsNeStsHits);
      return triggers;
    }
    recoEvent.stsHits = stsResults.hits;
  }
 
  //* TOF hit reconstruction
  {
    fEvSelectingMonitor.StartTimer(evselect::ETimer::TofHitFinder);
    auto [caldigis, calmonitor]          = (*fTofCalibrator)(digiEvent.fTof);
    auto [hits, hitmonitor, digiindices] = (*fTofHitFinder)(caldigis);
    fEvSelectingMonitor.StopTimer(evselect::ETimer::TofHitFinder);
    if (hits.NElements() < 2) {  // TODO: Provide a config for cuts (testing mode for now)
      fEvSelectingMonitor.IncrementCounter(evselect::ECounter::EventsNeTofHits);
      return triggers;
    }
    recoEvent.tofHits = std::move(hits);
  }
 
  //* TRD hit reconstruction
  {
    // FIXME: additional copy of digis, figure out how to pass 1d + 2d digis at once to hitfinder
    fEvSelectingMonitor.StartTimer(evselect::ETimer::TrdHitFinder);
    const auto& digis1d = digiEvent.fTrd;
    const auto& digis2d = digiEvent.fTrd2d;
    PODVector<CbmTrdDigi> allDigis{};
    allDigis.reserve(digis1d.size() + digis2d.size());
    std::copy(digis1d.begin(), digis1d.end(), std::back_inserter(allDigis));
    std::copy(digis2d.begin(), digis2d.end(), std::back_inserter(allDigis));
    auto trdResults = (*fTrdHitfind)(allDigis);
    fEvSelectingMonitor.StopTimer(evselect::ETimer::TrdHitFinder);
    recoEvent.trdHits = std::move(std::get<0>(trdResults));
  }
 
  //* Tracking
  {
    fEvSelectingMonitor.StartTimer(evselect::ETimer::TrackFinder);
    TrackingChain::Input_t input{.stsHits = recoEvent.stsHits,
                                 .tofHits = recoEvent.tofHits,
                                 .trdHits = recoEvent.trdHits};
    TrackingChain::Output_t output = fTrackingEvent->Run(input);
    recoEvent.tracks               = std::move(output.tracks);
    recoEvent.trackStsHitIndices   = std::move(output.stsHitIndices);
    recoEvent.trackTofHitIndices   = std::move(output.tofHitIndices);
    recoEvent.trackTrdHitIndices   = std::move(output.trdHitIndices);
    fEvSelectingMonitor.StopTimer(evselect::ETimer::TrackFinder);
    if (recoEvent.tracks.size() < 2) {  // Reject all events with less then two tracks
      fEvSelectingMonitor.IncrementCounter(evselect::ECounter::EventsNeTracks);
      return triggers;
    }
  }
 
  //* V0-selector
  fEvSelectingMonitor.StartTimer(evselect::ETimer::V0Finder);
  triggers = fV0Finder->ProcessEvent(recoEvent);
  if (triggers.Test(CbmEventTriggers::ETrigger::Lambda)) {
    L_(info) << "!!! Found event with potential lambda candidates";
    fEvSelectingMonitor.IncrementCounter(evselect::ECounter::EventsSelected);
  }
  fEvSelectingMonitor.StopTimer(evselect::ETimer::V0Finder);
  return triggers;
}
 
 
// ---------------------------------------------------------------------------------------------------------------------
//
template<class Unpacker>
auto Reco::RunUnpacker(const std::unique_ptr<Unpacker>& unpacker, const fles::Timeslice& ts) -> UnpackResult_t<Unpacker>
{
  if (!unpacker) {
    return {};
  }
  auto [digis, monitor, aux] = (*unpacker)(ts);
  QueueUnpackerMetricsDet(monitor);
  return std::make_tuple(digis, aux);
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
template<class MSMonitor>
void Reco::QueueUnpackerMetricsDet(const UnpackMonitor<MSMonitor>& monitor)
{
 
  L_(debug) << Capitalize(util::ToString(monitor.system)) << " errors: " << monitor.print();
 
  if (!HasMonitor()) {
    return;
  }
 
  std::string_view det = util::ToString(monitor.system);
 
  auto MkKey = [&](std::string_view key) { return fmt::format("{}{}", key, Capitalize(det)); };
 
  GetMonitor().QueueMetric("cbmreco", {{"hostname", fles::system::current_hostname()}, {"child", Opts().ChildId()}},
                           {
                             {MkKey("unpackBytesIn"), monitor.sizeBytesIn},
                             {MkKey("unpackBytesOut"), monitor.sizeBytesOut},
                             {MkKey("unpackExpansionFactor"), monitor.ExpansionFactor()},
                             {MkKey("unpackNumMs"), monitor.numMs},
                             {MkKey("unpackNumErrInvalidSysVer"), monitor.errInvalidSysVer},
                             {MkKey("unpackNumErrInvalidEqId"), monitor.errInvalidEqId},
                           });
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::QueueStsRecoMetrics(const sts::HitfinderMon& monitor)
{
  if (!HasMonitor()) return;
 
  GetMonitor().QueueMetric("cbmreco", {{"hostname", fles::system::current_hostname()}, {"child", Opts().ChildId()}},
                           {
                             {"stsRecoNumClusters", (unsigned long) monitor.nClusterTotal},
                             {"stsRecoNumHits", (unsigned long) monitor.nHitsTotal},
                             {"stsRecoNumClusterBucketOverflow", monitor.nClusterBucketOverflow},
                             {"stsRecoNumHitBucketOverflow", monitor.nHitBucketOverflow},
                           });
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::QueueTofRecoMetrics(const tof::HitfindMonitorData& mon)
{
  if (!HasMonitor()) return;
 
  GetMonitor().QueueMetric("cbmreco", {{"hostname", fles::system::current_hostname()}, {"child", Opts().ChildId()}},
                           {
                             {"tofRecoNumDigisIn", mon.fNumDigis},
                             {"tofRecoNumHits", mon.fNumHits},
                           });
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::QueueTrdRecoMetrics(const trd::HitfindMonitorData& mon)
{
  if (!HasMonitor()) {
    return;
  }
 
  GetMonitor().QueueMetric("cbmreco", {{"hostname", fles::system::current_hostname()}, {"child", Opts().ChildId()}},
                           {
                             {"trdRecoNumDigisIn", mon.numDigis},
                             {"trdRecoNumHits", mon.numHits},
                           });
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::QueueTofCalibMetrics(const tof::CalibrateMonitorData& mon)
{
  if (!HasMonitor()) return;
 
  GetMonitor().QueueMetric("cbmreco", {{"hostname", fles::system::current_hostname()}, {"child", Opts().ChildId()}},
                           {
                             {"tofCalibTimeTotal", mon.fTime.wall()},
                             {"tofCalibThroughput", FilterNan(mon.fTime.throughput())},
                             {"tofCalibNumDigisIn", mon.fNumDigis},
                             {"tofCalibUnknownRPC", mon.fDigiCalibUnknownRPC},
                           });
}
 
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::QueueEvbuildMetrics(const evbuild::EventbuildChainMonitorData& mon)
{
  if (fGeneralQaData) {
    fGeneralQaData->digimultrig_time_ms = mon.digiMultTrigger.time.wall();
    fGeneralQaData->hitmultrig_time_ms  = mon.hitMultTrigger.time.wall();
    fGeneralQaData->v0trig_time_ms      = mon.v0Trigger.time.wall();
    fGeneralQaData->evtbuild_time_ms    = mon.evbuild.time.wall();
  }
 
  if (!HasMonitor()) return;
 
  const MetricTagSet tags = {{"hostname", fles::system::current_hostname()}, {"child", Opts().ChildId()}};
 
  size_t nDigisTotal         = 0;
  size_t nDigisInEventsTotal = 0;
 
  auto queueDetMetrics = [&](std::string_view det, auto& detMon) {
    size_t nDigis         = detMon.nDigis;
    size_t nDigisInEvents = detMon.nDigisInEvents;
    double selectionRatio = nDigis > 0 ? double(nDigisInEvents) / nDigis : 0;
 
    nDigisTotal += nDigis;
    nDigisInEventsTotal += nDigisInEvents;
 
    GetMonitor().QueueMetric("cbmreco", tags,
                             {{fmt::format("{}NumDigisTotal", det), nDigis},
                              {fmt::format("{}NumDigisInEvents", det), nDigisInEvents},
                              {fmt::format("{}EvSelectionRatio", det), selectionRatio}});
  };
 
  queueDetMetrics("mvd", mon.evbuild.mvd);
  queueDetMetrics("sts", mon.evbuild.sts);
  queueDetMetrics("much", mon.evbuild.much);
  queueDetMetrics("mvd", mon.evbuild.mvd);
  queueDetMetrics("tof", mon.evbuild.tof);
  queueDetMetrics("bmon", mon.evbuild.bmon);
  queueDetMetrics("trd", mon.evbuild.trd);
  queueDetMetrics("trd2d", mon.evbuild.trd2d);
  queueDetMetrics("rich", mon.evbuild.rich);
 
  double totalSelectionRatio = nDigisTotal > 0 ? double(nDigisInEventsTotal) / nDigisTotal : 0;
  GetMonitor().QueueMetric("cbmreco", tags,
                           {{"digiTriggerTimeTotal", mon.digiMultTrigger.time.wall()},
                            {"digiTriggerThroughput", FilterNan(mon.digiMultTrigger.time.throughput())},
                            {"hitTriggerTimeTotal", mon.hitMultTrigger.time.wall()},
                            {"hitTriggerThroughput", FilterNan(mon.hitMultTrigger.time.throughput())},
                            {"v0TriggerNumTrackPairs", mon.v0Trigger.numTrackPairs},
                            {"v0TriggerNumTrackPairsCoinc", mon.v0Trigger.numTrackPairsAfterTimeCut},
                            {"v0TriggerErrTracksUnsorted", mon.v0Trigger.errTracksUnsorted},
                            {"v0TriggerTimeTotal", mon.v0Trigger.time.wall()},
                            {"v0TriggerThroughput", FilterNan(mon.v0Trigger.time.throughput())},
                            {"eventbuildTimeTotal", mon.evbuild.time.wall()},
                            {"eventbuildThroughput", FilterNan(mon.evbuild.time.throughput())},
                            {"numTrigger", mon.evbuild.numTriggers},
                            {"numEvents", mon.evbuild.numEvents},
                            {"totalEvSelectionRatio", totalSelectionRatio}});
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::QueueTrackingMetrics(const ca::TrackingMonitorData& monitor)
{
  if (!HasMonitor()) {
    return;
  }
 
  GetMonitor().QueueMetric("cbmreco", {{"hostname", fles::system::current_hostname()}, {"child", Opts().ChildId()}},
                           {{"caTrackFinderTime", monitor.GetTimer(ca::ETimer::FindTracks).GetTotalMs()},
                            {"caTrackFitterTime", monitor.GetTimer(ca::ETimer::FitTracks).GetTotalMs()},
                            {"caNofRecoTracks", monitor.GetCounterValue(ca::ECounter::RecoTrack)},
                            {"caNofRecoHitsTotal", monitor.GetCounterValue(ca::ECounter::RecoHit)},
                            {"caNofRecoHitsUsed", monitor.GetCounterValue(ca::ECounter::RecoHitUsed)},
                            {"caNofWindows", monitor.GetCounterValue(ca::ECounter::SubTS)}});
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::QueueProcessingMetrics(const ProcessingMonitor& mon)
{
  if (fGeneralQaData) {
    fGeneralQaData->total_time_ms    = mon.time.wall();
    fGeneralQaData->unpack_time_ms   = mon.timeUnpack.wall();
    fGeneralQaData->sts_reco_time_ms = mon.timeSTS.wall();
    fGeneralQaData->tof_reco_time_ms = mon.timeTOF.wall();
    fGeneralQaData->trd_reco_time_ms = mon.timeTRD.wall();
    fGeneralQaData->ca_time_ms       = mon.timeCA.wall();
  }
 
  if (!HasMonitor()) {
    return;
  }
 
  MetricFieldSet fields = {
    {"processingTimeTotal", mon.time.wall()},   {"processingThroughput", FilterNan(mon.time.throughput())},
    {"caRecoTimeTotal", mon.timeCA.wall()},     {"caRecoThroughput", FilterNan(mon.timeCA.throughput())},
    {"trdRecoTimeTotal", mon.timeTRD.wall()},   {"trdRecoThroughput", FilterNan(mon.timeTRD.throughput())},
    {"tofRecoTimeTotal", mon.timeTOF.wall()},   {"tofRecoThroughput", FilterNan(mon.timeTOF.throughput())},
    {"stsRecoTimeTotal", mon.timeSTS.wall()},   {"stsRecoThroughput", FilterNan(mon.timeSTS.throughput())},
    {"unpackTimeTotal", mon.timeUnpack.wall()}, {"unpackThroughput", FilterNan(mon.timeUnpack.throughput())}};
 
  if (mon.tsDelta) {
    fields.emplace_back("tsDelta", *mon.tsDelta);
  }
 
  GetMonitor().QueueMetric("cbmreco", {{"hostname", fles::system::current_hostname()}, {"child", Opts().ChildId()}},
                           std::move(fields));
}
 
// ---------------------------------------------------------------------------------------------------------------------
//
void Reco::QueueProcessingExtraMetrics(const ProcessingExtraMonitor& mon)
{
  if (!HasMonitor()) {
    return;
  }
 
  MetricFieldSet fields = {{"processingTimeIdle", FilterNan(mon.timeIdle)},
                           {"processingTimeWriteArchive", mon.timeWriteArchive.wall()},
                           {"processingThroughputWriteArchive", FilterNan(mon.timeWriteArchive.throughput())},
                           {"processingBytesWritten", FilterNan(mon.bytesWritten)}};
 
  GetMonitor().QueueMetric("cbmreco", {{"hostname", fles::system::current_hostname()}, {"child", Opts().ChildId()}},
                           std::move(fields));
}