From 3f233edc3931cfcf91e02f1b82846dc4e4a94018 Mon Sep 17 00:00:00 2001 From: "A.Burak Tektas" <15087318+Abtektas@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:05:42 +0100 Subject: [PATCH] feat: read appended data A process that appends data to a ULog file (for example PX4's hardfault_log after a crash) stores the file offset where the new data starts in the flag bits message. The log may have been cut in the middle of a message at that offset. The reader reported "File contains appended offsets - this is not supported" and kept parsing through the offset. A cut message was then joined with the appended bytes: on pyulog's sample_appended.ulg this gives one data sample with garbage values, and appended messages after a cut could be lost. Read up to each appended offset separately, drop the cut message and any recovery state there, and continue with the appended data. Offsets that point into the file header are ignored with an error. Assisted-by: Claude:claude-opus-5-5 --- test/ulog_parsing_test.cpp | 80 ++++++++++++++++++++++++++++++++++++++ ulog_cpp/reader.cpp | 65 ++++++++++++++++++++++++++++--- ulog_cpp/reader.hpp | 7 ++++ 3 files changed, 147 insertions(+), 5 deletions(-) diff --git a/test/ulog_parsing_test.cpp b/test/ulog_parsing_test.cpp index 4773055..4b19bde 100644 --- a/test/ulog_parsing_test.cpp +++ b/test/ulog_parsing_test.cpp @@ -4,6 +4,8 @@ ****************************************************************************/ #include +#include +#include #include #include #include @@ -266,6 +268,84 @@ TEST_CASE("ULog parsing - test corruption") } } +TEST_CASE("ULog parsing - appended data after a cut message") +{ + // A crash dump (e.g. PX4 hardfault_log) cuts the log, possibly in the middle of a message, + // and appends new messages at the offset stored in the flag bits. + for (const int cut_bytes : {0, 5}) { // bytes of the cut message that stay in the file + for (const int chunk_size : {1, 7, 1 << 20}) { + CAPTURE(cut_bytes); + CAPTURE(chunk_size); + + std::vector written_data; + TestWriter writer([&](const uint8_t* data, int length) { + const int prev_size = written_data.size(); + written_data.resize(written_data.size() + length); + memcpy(written_data.data() + prev_size, data, length); + }); + + const ulog_cpp::MessageFormat format{"message", + {{"uint64_t", "timestamp"}, {"uint16_t", "x"}}}; + const uint16_t msg_id = 1; + auto make_data = [&](uint8_t value) { + std::vector payload(10); + payload[0] = value; // timestamp + payload[8] = value; // x + return ulog_cpp::Data{msg_id, payload}; + }; + const ulog_cpp::Data data_before_cut = make_data(1); + const ulog_cpp::Data data_cut = make_data(2); + const ulog_cpp::Data data_appended = make_data(3); + const ulog_cpp::Logging logging{ulog_cpp::Logging::Level::Emergency, "appended", 4'000}; + + writer.fileHeader(ulog_cpp::FileHeader{}); + writer.messageFormat(format); + writer.headerComplete(); + writer.addLoggedMessage(ulog_cpp::AddLoggedMessage{0, msg_id, "message"}); + writer.data(data_before_cut); + + // Write a message, then cut the file after 'cut_bytes' of it + const size_t cut_message_start = written_data.size(); + writer.data(data_cut); + written_data.resize(cut_message_start + cut_bytes); + const uint64_t appended_offset = written_data.size(); + + writer.logging(logging); + writer.data(data_appended); + REQUIRE_EQ(writer.num_errors, 0); + + // Set the appended data flag and offset in the flag bits message + const size_t flag_bits_start = sizeof(ulog_cpp::ulog_file_header_s); + written_data[flag_bits_start + + offsetof(ulog_cpp::ulog_message_flag_bits_s, incompat_flags)] |= + ULOG_INCOMPAT_FLAG0_DATA_APPENDED_MASK; + memcpy(written_data.data() + flag_bits_start + + offsetof(ulog_cpp::ulog_message_flag_bits_s, appended_offsets), + &appended_offset, sizeof(appended_offset)); + + const auto data_container = std::make_shared( + ulog_cpp::DataContainer::StorageConfig::FullLog); + ulog_cpp::Reader reader{data_container}; + // The reader expects the file header and flag bits in the first chunk + const size_t header_size = flag_bits_start + sizeof(ulog_cpp::ulog_message_flag_bits_s); + reader.readChunk(written_data.data(), static_cast(header_size)); + for (size_t i = header_size; i < written_data.size(); i += chunk_size) { + const int length = std::min(chunk_size, static_cast(written_data.size() - i)); + reader.readChunk(written_data.data() + i, length); + } + + CHECK(data_container->parsingErrors().empty()); + CHECK_FALSE(data_container->hadFatalError()); + REQUIRE_EQ(data_container->logging().size(), 1); + CHECK_EQ(logging, data_container->logging()[0]); + const auto& samples = data_container->subscriptionsByMessageId().at(msg_id)->rawSamples(); + REQUIRE_EQ(samples.size(), 2); + CHECK_EQ(data_before_cut, samples[0]); + CHECK_EQ(data_appended, samples[1]); + } + } +} + TEST_CASE("ULog parsing - corruption recovery rejects wrong-size data for a known msg_id") { // Same technique as the "test corruption" case above, but instead of only inserting zero diff --git a/ulog_cpp/reader.cpp b/ulog_cpp/reader.cpp index 8a9bcfe..2294ec4 100644 --- a/ulog_cpp/reader.cpp +++ b/ulog_cpp/reader.cpp @@ -5,6 +5,8 @@ #include "reader.hpp" +#include +#include #include #include "raw_messages.hpp" @@ -76,6 +78,32 @@ void Reader::readChunk(const uint8_t* data, int length) _total_num_read += num_read; } + // Appended data starts at the stored offsets. The message at such an offset may have been cut, so + // read up to each offset separately and restart message parsing there. + while (length > 0 && _state != State::InvalidData && + _next_appended_offset_index < _appended_offsets.size()) { + const uint64_t offset = _appended_offsets[_next_appended_offset_index]; + const auto num_read = static_cast(_total_num_read); + if (offset > num_read) { + const int num_until_offset = + static_cast(std::min(offset - num_read, static_cast(length))); + readMessages(data, num_until_offset); + data += num_until_offset; + length -= num_until_offset; + } + if (static_cast(_total_num_read) >= offset) { + startAppendedData(); + ++_next_appended_offset_index; + } + } + + if (length > 0 && _state != State::InvalidData) { + readMessages(data, length); + } +} + +void Reader::readMessages(const uint8_t* data, int length) +{ static constexpr int kULogHeaderLength = static_cast(sizeof(ulog_message_header_s)); while (length > 0 && !_need_recovery) { @@ -251,7 +279,7 @@ void Reader::tryToRecover(const uint8_t* data, int length) "%i: recovered, recursive call (index = %i, length = %i, partial buf len = %i)\n", _total_num_read, index, length, _partial_message_buffer_length); _need_recovery = false; - readChunk(data, length); + readMessages(data, length); return; } @@ -261,6 +289,19 @@ void Reader::tryToRecover(const uint8_t* data, int length) } } +void Reader::startAppendedData() +{ + DBG_PRINTF("%i: start of appended data\n", _total_num_read); + // Discard the message that was cut at the offset (if any) and any data kept for recovery + _partial_message_buffer_length = 0; + _need_recovery = false; + if (_state == State::ReadHeader) { + // Appended data contains messages of the Data section + _state = State::ReadData; + _data_handler_interface->headerComplete(); + } +} + void Reader::corruptionDetected() { if (!_corruption_reported) { @@ -318,11 +359,25 @@ int Reader::readFlagBits(const uint8_t* data, int length) reinterpret_cast(data); if (static_cast(flag_bits->msg_type) == ULogMessageType::FLAG_BITS) { // This is expected to be the first message after the file magic - if (flag_bits->appended_offsets[0] != 0) { - // TODO: handle appended data - _data_handler_interface->error("File contains appended offsets - this is not supported", - true); + // Appended data must start after the flag bits message + const uint64_t data_start = + static_cast(_total_num_read) + flag_bits->msg_size + ULOG_MSG_HEADER_LEN; + for (size_t i = 0; + i < sizeof(flag_bits->appended_offsets) / sizeof(flag_bits->appended_offsets[0]); ++i) { + uint64_t offset = 0; + memcpy(&offset, + reinterpret_cast(flag_bits) + + offsetof(ulog_message_flag_bits_s, appended_offsets) + (i * sizeof(offset)), + sizeof(offset)); + if (offset >= data_start) { + _appended_offsets.push_back(offset); + } else if (offset != 0) { + _data_handler_interface->error("Invalid appended data offset, ignoring it", true); + } } + std::sort(_appended_offsets.begin(), _appended_offsets.end()); + _appended_offsets.erase(std::unique(_appended_offsets.begin(), _appended_offsets.end()), + _appended_offsets.end()); // Check incompat flags bool has_incompat_flags = false; if (flag_bits->incompat_flags[0] & ~(ULOG_INCOMPAT_FLAG0_DATA_APPENDED_MASK)) { diff --git a/ulog_cpp/reader.hpp b/ulog_cpp/reader.hpp index d426a20..99e8290 100644 --- a/ulog_cpp/reader.hpp +++ b/ulog_cpp/reader.hpp @@ -4,8 +4,10 @@ ****************************************************************************/ #pragma once +#include #include #include +#include #include "data_handler_interface.hpp" @@ -36,6 +38,8 @@ class Reader { void corruptionDetected(); int appendToPartialBuffer(const uint8_t* data, int length); void tryToRecover(const uint8_t* data, int length); + void readMessages(const uint8_t* data, int length); + void startAppendedData(); void readHeaderMessage(const uint8_t* message); void readDataMessage(const uint8_t* message); @@ -63,6 +67,9 @@ class Reader { ///< buffer data) ulog_file_header_s _file_header{}; + + std::vector _appended_offsets; ///< sorted file offsets where appended data starts + size_t _next_appended_offset_index{0}; }; } // namespace ulog_cpp