diff --git a/tools/cabana/streams/replaystream.cc b/tools/cabana/streams/replaystream.cc index 293b363532f29c..18b0317d4637f9 100644 --- a/tools/cabana/streams/replaystream.cc +++ b/tools/cabana/streams/replaystream.cc @@ -49,8 +49,8 @@ void ReplayStream::mergeSegments() { } bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags) { - replay.reset(new Replay(route, {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"}, - {}, nullptr, replay_flags, data_dir, this)); + replay.reset(new Replay(route.toStdString(), {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"}, + {}, nullptr, replay_flags, data_dir.toStdString(), this)); replay->setSegmentCacheLimit(settings.max_cached_minutes); replay->installEventFilter(event_filter, this); QObject::connect(replay.get(), &Replay::seeking, this, &AbstractStream::seeking); @@ -153,7 +153,7 @@ AbstractStream *OpenReplayWidget::open() { route = route.mid(idx + 1); } - bool is_valid_format = Route::parseRoute(route).str.size() > 0; + bool is_valid_format = Route::parseRoute(route.toStdString()).str.size() > 0; if (!is_valid_format) { QMessageBox::warning(nullptr, tr("Warning"), tr("Invalid route format: '%1'").arg(route)); } else { diff --git a/tools/cabana/streams/replaystream.h b/tools/cabana/streams/replaystream.h index 25d6ef4a72eda1..2a9f343645d255 100644 --- a/tools/cabana/streams/replaystream.h +++ b/tools/cabana/streams/replaystream.h @@ -20,7 +20,7 @@ class ReplayStream : public AbstractStream { bool eventFilter(const Event *event); void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); } bool liveStreaming() const override { return false; } - inline QString routeName() const override { return replay->route()->name(); } + inline QString routeName() const override { return QString::fromStdString(replay->route()->name()); } inline QString carFingerprint() const override { return replay->carFingerprint().c_str(); } double minSeconds() const override { return replay->minSeconds(); } double maxSeconds() const { return replay->maxSeconds(); } diff --git a/tools/replay/consoleui.cc b/tools/replay/consoleui.cc index 21ca0ad74b8468..5d2cf409d9c8ad 100644 --- a/tools/replay/consoleui.cc +++ b/tools/replay/consoleui.cc @@ -247,7 +247,7 @@ void ConsoleUI::updateProgressBar(uint64_t cur, uint64_t total, bool success) { void ConsoleUI::updateSummary() { const auto &route = replay->route(); - mvwprintw(w[Win::Stats], 0, 0, "Route: %s, %lu segments", qPrintable(route->name()), route->segments().size()); + mvwprintw(w[Win::Stats], 0, 0, "Route: %s, %lu segments", route->name().c_str(), route->segments().size()); mvwprintw(w[Win::Stats], 1, 0, "Car Fingerprint: %s", replay->carFingerprint().c_str()); wrefresh(w[Win::Stats]); } diff --git a/tools/replay/main.cc b/tools/replay/main.cc index 0bf7cc34230d46..4e11ddb58333f6 100644 --- a/tools/replay/main.cc +++ b/tools/replay/main.cc @@ -4,10 +4,12 @@ #include #include #include +#include #include "common/prefix.h" #include "tools/replay/consoleui.h" #include "tools/replay/replay.h" +#include "tools/replay/util.h" const std::string helpText = R"(Usage: replay [options] @@ -32,10 +34,10 @@ R"(Usage: replay [options] )"; struct ReplayConfig { - QString route; - QStringList allow; - QStringList block; - QString data_dir; + std::string route; + std::vector allow; + std::vector block; + std::string data_dir; std::string prefix; uint32_t flags = REPLAY_FLAG_NONE; int start_seconds = 0; @@ -84,8 +86,8 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { int opt, option_index = 0; while ((opt = getopt_long(argc, argv, "a:b:c:s:x:d:p:h", cli_options, &option_index)) != -1) { switch (opt) { - case 'a': config.allow = QString(optarg).split(","); break; - case 'b': config.block = QString(optarg).split(","); break; + case 'a': config.allow = split(optarg, ','); break; + case 'b': config.block = split(optarg, ','); break; case 'c': config.cache_segments = std::atoi(optarg); break; case 's': config.start_seconds = std::atoi(optarg); break; case 'x': config.playback_speed = std::atof(optarg); break; @@ -106,11 +108,11 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { } // Check for a route name (first positional argument) - if (config.route.isEmpty() && optind < argc) { + if (config.route.empty() && optind < argc) { config.route = argv[optind]; } - if (config.route.isEmpty()) { + if (config.route.empty()) { std::cerr << "No route provided. Use --help for usage information.\n"; return false; } diff --git a/tools/replay/replay.cc b/tools/replay/replay.cc index 34dd33bc1a477d..a0ae90c07928c1 100644 --- a/tools/replay/replay.cc +++ b/tools/replay/replay.cc @@ -11,36 +11,43 @@ static void interrupt_sleep_handler(int signal) {} -Replay::Replay(QString route, QStringList allow, QStringList block, SubMaster *sm_, - uint32_t flags, QString data_dir, QObject *parent) : sm(sm_), flags_(flags), QObject(parent) { +Replay::Replay(const std::string &route, std::vector allow, std::vector block, SubMaster *sm_, + uint32_t flags, const std::string &data_dir, QObject *parent) : sm(sm_), flags_(flags), QObject(parent) { // Register signal handler for SIGUSR1 std::signal(SIGUSR1, interrupt_sleep_handler); if (!(flags_ & REPLAY_FLAG_ALL_SERVICES)) { - block << "uiDebug" << "userFlag"; + block.insert(block.end(), {"uiDebug", "userFlag"}); } - auto event_struct = capnp::Schema::from().asStruct(); - sockets_.resize(event_struct.getUnionFields().size()); + + auto event_schema = capnp::Schema::from().asStruct(); + sockets_.resize(event_schema.getUnionFields().size()); + std::vector active_services; + for (const auto &[name, _] : services) { - if (!block.contains(name.c_str()) && (allow.empty() || allow.contains(name.c_str()))) { - uint16_t which = event_struct.getFieldByName(name).getProto().getDiscriminantValue(); + bool in_block = std::find(block.begin(), block.end(), name) != block.end(); + bool in_allow = std::find(allow.begin(), allow.end(), name) != allow.end(); + if (!in_block && (allow.empty() || in_allow)) { + uint16_t which = event_schema.getFieldByName(name).getProto().getDiscriminantValue(); sockets_[which] = name.c_str(); + active_services.push_back(name); } } - if (!allow.isEmpty()) { + + if (!allow.empty()) { for (int i = 0; i < sockets_.size(); ++i) { filters_.push_back(i == cereal::Event::Which::INIT_DATA || i == cereal::Event::Which::CAR_PARAMS || sockets_[i]); } } - std::vector s; - std::copy_if(sockets_.begin(), sockets_.end(), std::back_inserter(s), - [](const char *name) { return name != nullptr; }); - qDebug() << "services " << s; - qDebug() << "loading route " << route; + rInfo("active services: %s", join(active_services, ',').c_str()); + rInfo("loading route %s", route.c_str()); if (sm == nullptr) { - pm = std::make_unique(s); + std::vector socket_names; + std::copy_if(sockets_.begin(), sockets_.end(), std::back_inserter(socket_names), + [](const char *name) { return name != nullptr; }); + pm = std::make_unique(socket_names); } route_ = std::make_unique(route, data_dir); } @@ -68,23 +75,23 @@ void Replay::stop() { bool Replay::load() { if (!route_->load()) { - qCritical() << "failed to load route" << route_->name() - << "from" << (route_->dir().isEmpty() ? "server" : route_->dir()); + rError("failed to load route %s from %s", route_->name().c_str(), + route_->dir().empty() ? "server" : route_->dir().c_str()); return false; } for (auto &[n, f] : route_->segments()) { - bool has_log = !f.rlog.isEmpty() || !f.qlog.isEmpty(); - bool has_video = !f.road_cam.isEmpty() || !f.qcamera.isEmpty(); + bool has_log = !f.rlog.empty() || !f.qlog.empty(); + bool has_video = !f.road_cam.empty() || !f.qcamera.empty(); if (has_log && (has_video || hasFlag(REPLAY_FLAG_NO_VIPC))) { segments_.insert({n, nullptr}); } } if (segments_.empty()) { - qCritical() << "no valid segments in route" << route_->name(); + rInfo("no valid segments in route: %s", route_->name().c_str()); return false; } - rInfo("load route %s with %zu valid segments", qPrintable(route_->name()), segments_.size()); + rInfo("load route %s with %zu valid segments", route_->name().c_str(), segments_.size()); max_seconds_ = (segments_.rbegin()->first + 1) * 60; return true; } @@ -167,7 +174,7 @@ void Replay::buildTimeline() { const auto &route_segments = route_->segments(); for (auto it = route_segments.cbegin(); it != route_segments.cend() && !exit_; ++it) { std::shared_ptr log(new LogReader()); - if (!log->load(it->second.qlog.toStdString(), &exit_, !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3) || log->events.empty()) continue; + if (!log->load(it->second.qlog, &exit_, !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3) || log->events.empty()) continue; std::vector> timeline; for (const Event &e : log->events) { diff --git a/tools/replay/replay.h b/tools/replay/replay.h index ac829ffb71b752..8d2e07ba5b6033 100644 --- a/tools/replay/replay.h +++ b/tools/replay/replay.h @@ -15,7 +15,7 @@ #include "tools/replay/camera.h" #include "tools/replay/route.h" -const QString DEMO_ROUTE = "a2a0ccea32023010|2023-07-27--13-01-19"; +#define DEMO_ROUTE "a2a0ccea32023010|2023-07-27--13-01-19" // one segment uses about 100M of memory constexpr int MIN_SEGMENTS_CACHE = 5; @@ -49,8 +49,8 @@ class Replay : public QObject { Q_OBJECT public: - Replay(QString route, QStringList allow, QStringList block, SubMaster *sm = nullptr, - uint32_t flags = REPLAY_FLAG_NONE, QString data_dir = "", QObject *parent = 0); + Replay(const std::string &route, std::vector allow, std::vector block, SubMaster *sm = nullptr, + uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = "", QObject *parent = 0); ~Replay(); bool load(); RouteLoadError lastRouteError() const { return route_->lastError(); } diff --git a/tools/replay/route.cc b/tools/replay/route.cc index c1e65526cf7b5e..e298b88b79a821 100644 --- a/tools/replay/route.cc +++ b/tools/replay/route.cc @@ -4,35 +4,44 @@ #include #include #include -#include #include #include +#include +#include #include "selfdrive/ui/qt/api.h" #include "system/hardware/hw.h" #include "tools/replay/replay.h" #include "tools/replay/util.h" -Route::Route(const QString &route, const QString &data_dir) : data_dir_(data_dir) { +Route::Route(const std::string &route, const std::string &data_dir) : data_dir_(data_dir) { route_ = parseRoute(route); } -RouteIdentifier Route::parseRoute(const QString &str) { +RouteIdentifier Route::parseRoute(const std::string &str) { RouteIdentifier identifier = {}; - QRegularExpression rx(R"(^((?[a-z0-9]{16})[|_/])?(?.{20})((?--|/)(?((-?\d+(:(-?\d+)?)?)|(:-?\d+))))?$)"); - if (auto match = rx.match(str); match.hasMatch()) { - identifier.dongle_id = match.captured("dongle_id"); - identifier.timestamp = match.captured("timestamp"); + static const std::regex pattern(R"(^(([a-z0-9]{16})[|_/])?(.{20})((--|/)((-?\d+(:(-?\d+)?)?)|(:-?\d+)))?$)"); + std::smatch match; + + if (std::regex_match(str, match, pattern)) { + identifier.dongle_id = match[2].str(); + identifier.timestamp = match[3].str(); identifier.str = identifier.dongle_id + "|" + identifier.timestamp; - auto range_str = match.captured("range"); - if (auto separator = match.captured("separator"); separator == "/" && !range_str.isEmpty()) { - auto range = range_str.split(":"); - identifier.begin_segment = identifier.end_segment = range[0].toInt(); - if (range.size() == 2) { - identifier.end_segment = range[1].isEmpty() ? -1 : range[1].toInt(); + + const auto separator = match[5].str(); + const auto range_str = match[6].str(); + if (!range_str.empty()) { + if (separator == "/") { + int pos = range_str.find(':'); + int begin_seg = std::stoi(range_str.substr(0, pos)); + identifier.begin_segment = identifier.end_segment = begin_seg; + if (pos != std::string::npos) { + auto end_seg_str = range_str.substr(pos + 1); + identifier.end_segment = end_seg_str.empty() ? -1 : std::stoi(end_seg_str); + } + } else if (separator == "--") { + identifier.begin_segment = std::atoi(range_str.c_str()); } - } else if (separator == "--") { - identifier.begin_segment = range_str.toInt(); } } return identifier; @@ -40,12 +49,12 @@ RouteIdentifier Route::parseRoute(const QString &str) { bool Route::load() { err_ = RouteLoadError::None; - if (route_.str.isEmpty() || (data_dir_.isEmpty() && route_.dongle_id.isEmpty())) { + if (route_.str.empty() || (data_dir_.empty() && route_.dongle_id.empty())) { rInfo("invalid route format"); return false; } - date_time_ = QDateTime::fromString(route_.timestamp, "yyyy-MM-dd--HH-mm-ss"); - bool ret = data_dir_.isEmpty() ? loadFromServer() : loadFromLocal(); + date_time_ = QDateTime::fromString(route_.timestamp.c_str(), "yyyy-MM-dd--HH-mm-ss"); + bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal(); if (ret) { if (route_.begin_segment == -1) route_.begin_segment = segments_.rbegin()->first; if (route_.end_segment == -1) route_.end_segment = segments_.rbegin()->first; @@ -69,7 +78,7 @@ bool Route::loadFromServer(int retries) { result = json; loop.exit((int)err); }); - http.sendRequest(CommaApi::BASE_URL + "/v1/route/" + route_.str + "/files"); + http.sendRequest(CommaApi::BASE_URL + "/v1/route/" + QString::fromStdString(route_.str) + "/files"); auto err = (QNetworkReply::NetworkError)loop.exec(); if (err == QNetworkReply::NoError) { return loadFromJson(result); @@ -96,7 +105,7 @@ bool Route::loadFromJson(const QString &json) { for (const auto &url : value.toArray()) { QString url_str = url.toString(); if (rx.indexIn(url_str) != -1) { - addFileToSegment(rx.cap(1).toInt(), url_str); + addFileToSegment(rx.cap(1).toInt(), url_str.toStdString()); } } } @@ -104,23 +113,26 @@ bool Route::loadFromJson(const QString &json) { } bool Route::loadFromLocal() { - QDirIterator it(data_dir_, {QString("%1--*").arg(route_.timestamp)}, QDir::Dirs | QDir::NoDotAndDotDot); - while (it.hasNext()) { - QString segment = it.next(); - const int seg_num = segment.mid(segment.lastIndexOf("--") + 2).toInt(); - QDir segment_dir(segment); - for (const auto &f : segment_dir.entryList(QDir::Files)) { - addFileToSegment(seg_num, segment_dir.absoluteFilePath(f)); + std::string pattern = route_.timestamp + "--"; + for (const auto &entry : std::filesystem::directory_iterator(data_dir_)) { + if (entry.is_directory() && entry.path().filename().string().find(pattern) == 0) { + std::string segment = entry.path().string(); + int seg_num = std::atoi(segment.substr(segment.rfind("--") + 2).c_str()); + + for (const auto &file : std::filesystem::directory_iterator(segment)) { + if (file.is_regular_file()) { + addFileToSegment(seg_num, file.path().string()); + } + } } } return !segments_.empty(); } -void Route::addFileToSegment(int n, const QString &file) { - QString name = QUrl(file).fileName(); - - const int pos = name.lastIndexOf("--"); - name = pos != -1 ? name.mid(pos + 2) : name; +void Route::addFileToSegment(int n, const std::string &file) { + std::string name = extractFileName(file); + auto pos = name.find_last_of("--"); + name = pos != std::string::npos ? name.substr(pos + 2) : name; if (name == "rlog.bz2" || name == "rlog.zst" || name == "rlog") { segments_[n].rlog = file; @@ -143,15 +155,15 @@ Segment::Segment(int n, const SegmentFile &files, uint32_t flags, const std::vec : seg_num(n), flags(flags), filters_(filters) { // [RoadCam, DriverCam, WideRoadCam, log]. fallback to qcamera/qlog const std::array file_list = { - (flags & REPLAY_FLAG_QCAMERA) || files.road_cam.isEmpty() ? files.qcamera : files.road_cam, + (flags & REPLAY_FLAG_QCAMERA) || files.road_cam.empty() ? files.qcamera : files.road_cam, flags & REPLAY_FLAG_DCAM ? files.driver_cam : "", flags & REPLAY_FLAG_ECAM ? files.wide_road_cam : "", - files.rlog.isEmpty() ? files.qlog : files.rlog, + files.rlog.empty() ? files.qlog : files.rlog, }; for (int i = 0; i < file_list.size(); ++i) { - if (!file_list[i].isEmpty() && (!(flags & REPLAY_FLAG_NO_VIPC) || i >= MAX_CAMERAS)) { + if (!file_list[i].empty() && (!(flags & REPLAY_FLAG_NO_VIPC) || i >= MAX_CAMERAS)) { ++loading_; - synchronizer_.addFuture(QtConcurrent::run(this, &Segment::loadFile, i, file_list[i].toStdString())); + synchronizer_.addFuture(QtConcurrent::run(this, &Segment::loadFile, i, file_list[i])); } } } diff --git a/tools/replay/route.h b/tools/replay/route.h index acc73d509a1f6d..9c1cc37e2a9ac5 100644 --- a/tools/replay/route.h +++ b/tools/replay/route.h @@ -21,42 +21,42 @@ enum class RouteLoadError { }; struct RouteIdentifier { - QString dongle_id; - QString timestamp; + std::string dongle_id; + std::string timestamp; int begin_segment = 0; int end_segment = -1; - QString str; + std::string str; }; struct SegmentFile { - QString rlog; - QString qlog; - QString road_cam; - QString driver_cam; - QString wide_road_cam; - QString qcamera; + std::string rlog; + std::string qlog; + std::string road_cam; + std::string driver_cam; + std::string wide_road_cam; + std::string qcamera; }; class Route { public: - Route(const QString &route, const QString &data_dir = {}); + Route(const std::string &route, const std::string &data_dir = {}); bool load(); RouteLoadError lastError() const { return err_; } - inline const QString &name() const { return route_.str; } + inline const std::string &name() const { return route_.str; } inline const QDateTime datetime() const { return date_time_; } - inline const QString &dir() const { return data_dir_; } + inline const std::string &dir() const { return data_dir_; } inline const RouteIdentifier &identifier() const { return route_; } inline const std::map &segments() const { return segments_; } inline const SegmentFile &at(int n) { return segments_.at(n); } - static RouteIdentifier parseRoute(const QString &str); + static RouteIdentifier parseRoute(const std::string &str); protected: bool loadFromLocal(); bool loadFromServer(int retries = 3); bool loadFromJson(const QString &json); - void addFileToSegment(int seg_num, const QString &file); + void addFileToSegment(int seg_num, const std::string &file); RouteIdentifier route_ = {}; - QString data_dir_; + std::string data_dir_; std::map segments_; QDateTime date_time_; RouteLoadError err_ = RouteLoadError::None; diff --git a/tools/replay/tests/test_replay.cc b/tools/replay/tests/test_replay.cc index 3e5bfdf984e309..2fe468e726d23c 100644 --- a/tools/replay/tests/test_replay.cc +++ b/tools/replay/tests/test_replay.cc @@ -122,12 +122,12 @@ std::string download_demo_route() { assert(remote_route.load()); // Create a local route from remote for testing - const std::string route_name = DEMO_ROUTE.mid(17).toStdString(); + const std::string route_name = std::string(DEMO_ROUTE).substr(17); for (int i = 0; i < 2; ++i) { std::string log_path = util::string_format("%s/%s--%d/", data_dir.c_str(), route_name.c_str(), i); util::create_directories(log_path, 0755); - REQUIRE(download_to_file(remote_route.at(i).rlog.toStdString(), log_path + "rlog.bz2")); - REQUIRE(download_to_file(remote_route.at(i).qcamera.toStdString(), log_path + "qcamera.ts")); + REQUIRE(download_to_file(remote_route.at(i).rlog, log_path + "rlog.bz2")); + REQUIRE(download_to_file(remote_route.at(i).qcamera, log_path + "qcamera.ts")); } } @@ -139,7 +139,7 @@ TEST_CASE("Local route") { std::string data_dir = download_demo_route(); auto flags = GENERATE(0, REPLAY_FLAG_QCAMERA); - Route route(DEMO_ROUTE, QString::fromStdString(data_dir)); + Route route(DEMO_ROUTE, data_dir); REQUIRE(route.load()); REQUIRE(route.segments().size() == 2); for (int i = 0; i < TEST_REPLAY_SEGMENTS; ++i) { diff --git a/tools/replay/util.cc b/tools/replay/util.cc index 0ae0c05f9b8dab..91f6af4f845575 100644 --- a/tools/replay/util.cc +++ b/tools/replay/util.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -390,6 +391,35 @@ std::string sha256(const std::string &str) { return util::hexdump(hash, SHA256_DIGEST_LENGTH); } +std::vector split(std::string_view source, char delimiter) { + std::vector fields; + size_t last = 0; + for (size_t i = 0; i < source.length(); ++i) { + if (source[i] == delimiter) { + fields.emplace_back(source.substr(last, i - last)); + last = i + 1; + } + } + fields.emplace_back(source.substr(last)); + return fields; +} + +std::string join(const std::vector &elements, char separator) { + std::ostringstream oss; + for (size_t i = 0; i < elements.size(); ++i) { + if (i != 0) oss << separator; + oss << elements[i]; + } + return oss.str(); +} + +std::string extractFileName(const std::string &file) { + size_t queryPos = file.find_first_of("?"); + std::string path = (queryPos != std::string::npos) ? file.substr(0, queryPos) : file; + size_t lastSlash = path.find_last_of("/\\"); + return (lastSlash != std::string::npos) ? path.substr(lastSlash + 1) : path; +} + // MonotonicBuffer void *MonotonicBuffer::allocate(size_t bytes, size_t alignment) { diff --git a/tools/replay/util.h b/tools/replay/util.h index d6c26a34716ab1..317b964181a6a5 100644 --- a/tools/replay/util.h +++ b/tools/replay/util.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "cereal/messaging/messaging.h" enum CameraType { @@ -57,3 +59,6 @@ typedef std::function Download void installDownloadProgressHandler(DownloadProgressHandler); bool httpDownload(const std::string &url, const std::string &file, size_t chunk_size = 0, std::atomic *abort = nullptr); std::string formattedDataSize(size_t size); +std::vector split(std::string_view source, char delimiter); +std::string join(const std::vector &elements, char separator); +std::string extractFileName(const std::string& file);