Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions test/ulog_parsing_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
****************************************************************************/
#include <doctest/doctest.h>

#include <algorithm>
#include <cstddef>
#include <filesystem>
#include <ulog_cpp/data_container.hpp>
#include <ulog_cpp/reader.hpp>
Expand Down Expand Up @@ -266,6 +268,84 @@ TEST_CASE("ULog parsing - test corruption")
}
}

TEST_CASE("ULog parsing - appended data after a cut message")
{
// A crash dump (e.g. PX4 hardfault_log) cuts the log, possibly in the middle of a message,
// and appends new messages at the offset stored in the flag bits.
for (const int cut_bytes : {0, 5}) { // bytes of the cut message that stay in the file
for (const int chunk_size : {1, 7, 1 << 20}) {
CAPTURE(cut_bytes);
CAPTURE(chunk_size);

std::vector<uint8_t> written_data;
TestWriter writer([&](const uint8_t* data, int length) {
const int prev_size = written_data.size();
written_data.resize(written_data.size() + length);
memcpy(written_data.data() + prev_size, data, length);
});

const ulog_cpp::MessageFormat format{"message",
{{"uint64_t", "timestamp"}, {"uint16_t", "x"}}};
const uint16_t msg_id = 1;
auto make_data = [&](uint8_t value) {
std::vector<uint8_t> payload(10);
payload[0] = value; // timestamp
payload[8] = value; // x
return ulog_cpp::Data{msg_id, payload};
};
const ulog_cpp::Data data_before_cut = make_data(1);
const ulog_cpp::Data data_cut = make_data(2);
const ulog_cpp::Data data_appended = make_data(3);
const ulog_cpp::Logging logging{ulog_cpp::Logging::Level::Emergency, "appended", 4'000};

writer.fileHeader(ulog_cpp::FileHeader{});
writer.messageFormat(format);
writer.headerComplete();
writer.addLoggedMessage(ulog_cpp::AddLoggedMessage{0, msg_id, "message"});
writer.data(data_before_cut);

// Write a message, then cut the file after 'cut_bytes' of it
const size_t cut_message_start = written_data.size();
writer.data(data_cut);
written_data.resize(cut_message_start + cut_bytes);
const uint64_t appended_offset = written_data.size();

writer.logging(logging);
writer.data(data_appended);
REQUIRE_EQ(writer.num_errors, 0);

// Set the appended data flag and offset in the flag bits message
const size_t flag_bits_start = sizeof(ulog_cpp::ulog_file_header_s);
written_data[flag_bits_start +
offsetof(ulog_cpp::ulog_message_flag_bits_s, incompat_flags)] |=
ULOG_INCOMPAT_FLAG0_DATA_APPENDED_MASK;
memcpy(written_data.data() + flag_bits_start +
offsetof(ulog_cpp::ulog_message_flag_bits_s, appended_offsets),
&appended_offset, sizeof(appended_offset));

const auto data_container = std::make_shared<ulog_cpp::DataContainer>(
ulog_cpp::DataContainer::StorageConfig::FullLog);
ulog_cpp::Reader reader{data_container};
// The reader expects the file header and flag bits in the first chunk
const size_t header_size = flag_bits_start + sizeof(ulog_cpp::ulog_message_flag_bits_s);
reader.readChunk(written_data.data(), static_cast<int>(header_size));
for (size_t i = header_size; i < written_data.size(); i += chunk_size) {
const int length = std::min<int>(chunk_size, static_cast<int>(written_data.size() - i));
reader.readChunk(written_data.data() + i, length);
}

CHECK(data_container->parsingErrors().empty());
CHECK_FALSE(data_container->hadFatalError());
REQUIRE_EQ(data_container->logging().size(), 1);
CHECK_EQ(logging, data_container->logging()[0]);
const auto& samples = data_container->subscriptionsByMessageId().at(msg_id)->rawSamples();
REQUIRE_EQ(samples.size(), 2);
CHECK_EQ(data_before_cut, samples[0]);
CHECK_EQ(data_appended, samples[1]);
}
}
}

TEST_CASE("ULog parsing - corruption recovery rejects wrong-size data for a known msg_id")
{
// Same technique as the "test corruption" case above, but instead of only inserting zero
Expand Down
65 changes: 60 additions & 5 deletions ulog_cpp/reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

#include "reader.hpp"

#include <algorithm>
#include <cstddef>
#include <cstring>

#include "raw_messages.hpp"
Expand Down Expand Up @@ -76,6 +78,32 @@ void Reader::readChunk(const uint8_t* data, int length)
_total_num_read += num_read;
}

// Appended data starts at the stored offsets. The message at such an offset may have been cut, so
// read up to each offset separately and restart message parsing there.
while (length > 0 && _state != State::InvalidData &&
_next_appended_offset_index < _appended_offsets.size()) {
const uint64_t offset = _appended_offsets[_next_appended_offset_index];
const auto num_read = static_cast<uint64_t>(_total_num_read);
if (offset > num_read) {
const int num_until_offset =
static_cast<int>(std::min<uint64_t>(offset - num_read, static_cast<uint64_t>(length)));
readMessages(data, num_until_offset);
data += num_until_offset;
length -= num_until_offset;
}
if (static_cast<uint64_t>(_total_num_read) >= offset) {
startAppendedData();
++_next_appended_offset_index;
}
}

if (length > 0 && _state != State::InvalidData) {
readMessages(data, length);
}
}

void Reader::readMessages(const uint8_t* data, int length)
{
static constexpr int kULogHeaderLength = static_cast<int>(sizeof(ulog_message_header_s));

while (length > 0 && !_need_recovery) {
Expand Down Expand Up @@ -251,7 +279,7 @@ void Reader::tryToRecover(const uint8_t* data, int length)
"%i: recovered, recursive call (index = %i, length = %i, partial buf len = %i)\n",
_total_num_read, index, length, _partial_message_buffer_length);
_need_recovery = false;
readChunk(data, length);
readMessages(data, length);

return;
}
Expand All @@ -261,6 +289,19 @@ void Reader::tryToRecover(const uint8_t* data, int length)
}
}

void Reader::startAppendedData()
{
DBG_PRINTF("%i: start of appended data\n", _total_num_read);
// Discard the message that was cut at the offset (if any) and any data kept for recovery
_partial_message_buffer_length = 0;
_need_recovery = false;
if (_state == State::ReadHeader) {
// Appended data contains messages of the Data section
_state = State::ReadData;
_data_handler_interface->headerComplete();
}
}

void Reader::corruptionDetected()
{
if (!_corruption_reported) {
Expand Down Expand Up @@ -318,11 +359,25 @@ int Reader::readFlagBits(const uint8_t* data, int length)
reinterpret_cast<const ulog_message_flag_bits_s*>(data);
if (static_cast<ULogMessageType>(flag_bits->msg_type) == ULogMessageType::FLAG_BITS) {
// This is expected to be the first message after the file magic
if (flag_bits->appended_offsets[0] != 0) {
// TODO: handle appended data
_data_handler_interface->error("File contains appended offsets - this is not supported",
true);
// Appended data must start after the flag bits message
const uint64_t data_start =
static_cast<uint64_t>(_total_num_read) + flag_bits->msg_size + ULOG_MSG_HEADER_LEN;
for (size_t i = 0;
i < sizeof(flag_bits->appended_offsets) / sizeof(flag_bits->appended_offsets[0]); ++i) {
uint64_t offset = 0;
memcpy(&offset,
reinterpret_cast<const uint8_t*>(flag_bits) +
offsetof(ulog_message_flag_bits_s, appended_offsets) + (i * sizeof(offset)),
sizeof(offset));
if (offset >= data_start) {
_appended_offsets.push_back(offset);
} else if (offset != 0) {
_data_handler_interface->error("Invalid appended data offset, ignoring it", true);
}
}
std::sort(_appended_offsets.begin(), _appended_offsets.end());
_appended_offsets.erase(std::unique(_appended_offsets.begin(), _appended_offsets.end()),
_appended_offsets.end());
// Check incompat flags
bool has_incompat_flags = false;
if (flag_bits->incompat_flags[0] & ~(ULOG_INCOMPAT_FLAG0_DATA_APPENDED_MASK)) {
Expand Down
7 changes: 7 additions & 0 deletions ulog_cpp/reader.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
****************************************************************************/
#pragma once

#include <cstdint>
#include <memory>
#include <set>
#include <vector>

#include "data_handler_interface.hpp"

Expand Down Expand Up @@ -36,6 +38,8 @@ class Reader {
void corruptionDetected();
int appendToPartialBuffer(const uint8_t* data, int length);
void tryToRecover(const uint8_t* data, int length);
void readMessages(const uint8_t* data, int length);
void startAppendedData();

void readHeaderMessage(const uint8_t* message);
void readDataMessage(const uint8_t* message);
Expand Down Expand Up @@ -63,6 +67,9 @@ class Reader {
///< buffer data)

ulog_file_header_s _file_header{};

std::vector<uint64_t> _appended_offsets; ///< sorted file offsets where appended data starts
size_t _next_appended_offset_index{0};
};

} // namespace ulog_cpp
Loading