diff --git a/test/ulog_parsing_test.cpp b/test/ulog_parsing_test.cpp index 4773055..4b19bde 100644 --- a/test/ulog_parsing_test.cpp +++ b/test/ulog_parsing_test.cpp @@ -4,6 +4,8 @@ ****************************************************************************/ #include +#include +#include #include #include #include @@ -266,6 +268,84 @@ TEST_CASE("ULog parsing - test corruption") } } +TEST_CASE("ULog parsing - appended data after a cut message") +{ + // A crash dump (e.g. PX4 hardfault_log) cuts the log, possibly in the middle of a message, + // and appends new messages at the offset stored in the flag bits. + for (const int cut_bytes : {0, 5}) { // bytes of the cut message that stay in the file + for (const int chunk_size : {1, 7, 1 << 20}) { + CAPTURE(cut_bytes); + CAPTURE(chunk_size); + + std::vector written_data; + TestWriter writer([&](const uint8_t* data, int length) { + const int prev_size = written_data.size(); + written_data.resize(written_data.size() + length); + memcpy(written_data.data() + prev_size, data, length); + }); + + const ulog_cpp::MessageFormat format{"message", + {{"uint64_t", "timestamp"}, {"uint16_t", "x"}}}; + const uint16_t msg_id = 1; + auto make_data = [&](uint8_t value) { + std::vector payload(10); + payload[0] = value; // timestamp + payload[8] = value; // x + return ulog_cpp::Data{msg_id, payload}; + }; + const ulog_cpp::Data data_before_cut = make_data(1); + const ulog_cpp::Data data_cut = make_data(2); + const ulog_cpp::Data data_appended = make_data(3); + const ulog_cpp::Logging logging{ulog_cpp::Logging::Level::Emergency, "appended", 4'000}; + + writer.fileHeader(ulog_cpp::FileHeader{}); + writer.messageFormat(format); + writer.headerComplete(); + writer.addLoggedMessage(ulog_cpp::AddLoggedMessage{0, msg_id, "message"}); + writer.data(data_before_cut); + + // Write a message, then cut the file after 'cut_bytes' of it + const size_t cut_message_start = written_data.size(); + writer.data(data_cut); + written_data.resize(cut_message_start + cut_bytes); + const uint64_t appended_offset = written_data.size(); + + writer.logging(logging); + writer.data(data_appended); + REQUIRE_EQ(writer.num_errors, 0); + + // Set the appended data flag and offset in the flag bits message + const size_t flag_bits_start = sizeof(ulog_cpp::ulog_file_header_s); + written_data[flag_bits_start + + offsetof(ulog_cpp::ulog_message_flag_bits_s, incompat_flags)] |= + ULOG_INCOMPAT_FLAG0_DATA_APPENDED_MASK; + memcpy(written_data.data() + flag_bits_start + + offsetof(ulog_cpp::ulog_message_flag_bits_s, appended_offsets), + &appended_offset, sizeof(appended_offset)); + + const auto data_container = std::make_shared( + ulog_cpp::DataContainer::StorageConfig::FullLog); + ulog_cpp::Reader reader{data_container}; + // The reader expects the file header and flag bits in the first chunk + const size_t header_size = flag_bits_start + sizeof(ulog_cpp::ulog_message_flag_bits_s); + reader.readChunk(written_data.data(), static_cast(header_size)); + for (size_t i = header_size; i < written_data.size(); i += chunk_size) { + const int length = std::min(chunk_size, static_cast(written_data.size() - i)); + reader.readChunk(written_data.data() + i, length); + } + + CHECK(data_container->parsingErrors().empty()); + CHECK_FALSE(data_container->hadFatalError()); + REQUIRE_EQ(data_container->logging().size(), 1); + CHECK_EQ(logging, data_container->logging()[0]); + const auto& samples = data_container->subscriptionsByMessageId().at(msg_id)->rawSamples(); + REQUIRE_EQ(samples.size(), 2); + CHECK_EQ(data_before_cut, samples[0]); + CHECK_EQ(data_appended, samples[1]); + } + } +} + TEST_CASE("ULog parsing - corruption recovery rejects wrong-size data for a known msg_id") { // Same technique as the "test corruption" case above, but instead of only inserting zero diff --git a/ulog_cpp/reader.cpp b/ulog_cpp/reader.cpp index 8a9bcfe..2294ec4 100644 --- a/ulog_cpp/reader.cpp +++ b/ulog_cpp/reader.cpp @@ -5,6 +5,8 @@ #include "reader.hpp" +#include +#include #include #include "raw_messages.hpp" @@ -76,6 +78,32 @@ void Reader::readChunk(const uint8_t* data, int length) _total_num_read += num_read; } + // Appended data starts at the stored offsets. The message at such an offset may have been cut, so + // read up to each offset separately and restart message parsing there. + while (length > 0 && _state != State::InvalidData && + _next_appended_offset_index < _appended_offsets.size()) { + const uint64_t offset = _appended_offsets[_next_appended_offset_index]; + const auto num_read = static_cast(_total_num_read); + if (offset > num_read) { + const int num_until_offset = + static_cast(std::min(offset - num_read, static_cast(length))); + readMessages(data, num_until_offset); + data += num_until_offset; + length -= num_until_offset; + } + if (static_cast(_total_num_read) >= offset) { + startAppendedData(); + ++_next_appended_offset_index; + } + } + + if (length > 0 && _state != State::InvalidData) { + readMessages(data, length); + } +} + +void Reader::readMessages(const uint8_t* data, int length) +{ static constexpr int kULogHeaderLength = static_cast(sizeof(ulog_message_header_s)); while (length > 0 && !_need_recovery) { @@ -251,7 +279,7 @@ void Reader::tryToRecover(const uint8_t* data, int length) "%i: recovered, recursive call (index = %i, length = %i, partial buf len = %i)\n", _total_num_read, index, length, _partial_message_buffer_length); _need_recovery = false; - readChunk(data, length); + readMessages(data, length); return; } @@ -261,6 +289,19 @@ void Reader::tryToRecover(const uint8_t* data, int length) } } +void Reader::startAppendedData() +{ + DBG_PRINTF("%i: start of appended data\n", _total_num_read); + // Discard the message that was cut at the offset (if any) and any data kept for recovery + _partial_message_buffer_length = 0; + _need_recovery = false; + if (_state == State::ReadHeader) { + // Appended data contains messages of the Data section + _state = State::ReadData; + _data_handler_interface->headerComplete(); + } +} + void Reader::corruptionDetected() { if (!_corruption_reported) { @@ -318,11 +359,25 @@ int Reader::readFlagBits(const uint8_t* data, int length) reinterpret_cast(data); if (static_cast(flag_bits->msg_type) == ULogMessageType::FLAG_BITS) { // This is expected to be the first message after the file magic - if (flag_bits->appended_offsets[0] != 0) { - // TODO: handle appended data - _data_handler_interface->error("File contains appended offsets - this is not supported", - true); + // Appended data must start after the flag bits message + const uint64_t data_start = + static_cast(_total_num_read) + flag_bits->msg_size + ULOG_MSG_HEADER_LEN; + for (size_t i = 0; + i < sizeof(flag_bits->appended_offsets) / sizeof(flag_bits->appended_offsets[0]); ++i) { + uint64_t offset = 0; + memcpy(&offset, + reinterpret_cast(flag_bits) + + offsetof(ulog_message_flag_bits_s, appended_offsets) + (i * sizeof(offset)), + sizeof(offset)); + if (offset >= data_start) { + _appended_offsets.push_back(offset); + } else if (offset != 0) { + _data_handler_interface->error("Invalid appended data offset, ignoring it", true); + } } + std::sort(_appended_offsets.begin(), _appended_offsets.end()); + _appended_offsets.erase(std::unique(_appended_offsets.begin(), _appended_offsets.end()), + _appended_offsets.end()); // Check incompat flags bool has_incompat_flags = false; if (flag_bits->incompat_flags[0] & ~(ULOG_INCOMPAT_FLAG0_DATA_APPENDED_MASK)) { diff --git a/ulog_cpp/reader.hpp b/ulog_cpp/reader.hpp index d426a20..99e8290 100644 --- a/ulog_cpp/reader.hpp +++ b/ulog_cpp/reader.hpp @@ -4,8 +4,10 @@ ****************************************************************************/ #pragma once +#include #include #include +#include #include "data_handler_interface.hpp" @@ -36,6 +38,8 @@ class Reader { void corruptionDetected(); int appendToPartialBuffer(const uint8_t* data, int length); void tryToRecover(const uint8_t* data, int length); + void readMessages(const uint8_t* data, int length); + void startAppendedData(); void readHeaderMessage(const uint8_t* message); void readDataMessage(const uint8_t* message); @@ -63,6 +67,9 @@ class Reader { ///< buffer data) ulog_file_header_s _file_header{}; + + std::vector _appended_offsets; ///< sorted file offsets where appended data starts + size_t _next_appended_offset_index{0}; }; } // namespace ulog_cpp