-
Notifications
You must be signed in to change notification settings - Fork 125
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initial POC for MCAP Support with c++ and python
- Loading branch information
1 parent
b00ad2b
commit fb2d23f
Showing
23 changed files
with
6,066 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
# Mcap Tracefile Support | ||
This folder contains the official upstream MCAP c++ library as well as convince classes for c++ OSI trace file writer and reader support. | ||
|
||
# Work in Progress | ||
- This is a work in progress and just a proof of concept. | ||
- Major cleaning and refactoring is needed. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
#ifndef MCAP_TRACEFILE_H | ||
#define MCAP_TRACEFILE_H | ||
#include <../mcap/include/mcap/mcap.hpp> | ||
#include <filesystem> | ||
|
||
namespace osi3 | ||
{ | ||
class TracefileMcapReader { | ||
public: | ||
explicit TracefileMcapReader(std::filesystem::path mcap_file); | ||
~TracefileMcapReader() { reader_.close(); }; | ||
bool has_next() const; | ||
std::pair<mcap::Message, mcap::ChannelPtr> read_next(); | ||
private: | ||
mcap::McapReader reader_; | ||
std::unique_ptr<mcap::LinearMessageView> message_view_ = nullptr; | ||
std::optional<mcap::LinearMessageView::Iterator> message_it_; | ||
}; | ||
|
||
|
||
} // namespace osi3 | ||
|
||
|
||
#endif //MCAP_TRACEFILE_H |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
#ifndef OSI_TRACEFILE_MCAP_WRITER_H | ||
#define OSI_TRACEFILE_MCAP_WRITER_H | ||
#include <../mcap/include/mcap/mcap.hpp> | ||
#include <filesystem> | ||
|
||
namespace osi3 | ||
{ | ||
class TracefileMcapWriter { | ||
public: | ||
explicit TracefileMcapWriter(std::filesystem::path mcap_file); | ||
~TracefileMcapWriter() { writer_.close(); }; | ||
template <typename T> void write(T top_level_message); | ||
void close() { writer_.close(); } | ||
|
||
private: | ||
void addOSITopLevelMessagesAsSchemata(); | ||
void addCommonMetadata(); | ||
mcap::McapWriter writer_; | ||
std::map<std::string, int> name_to_schema_ids_; | ||
|
||
}; | ||
|
||
|
||
} // namespace osi3 | ||
|
||
|
||
#endif //OSI_TRACEFILE_MCAP_WRITER_H |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
MIT License | ||
|
||
Copyright (c) Foxglove Technologies Inc | ||
|
||
Permission is hereby granted, free of charge, to any person obtaining a copy | ||
of this software and associated documentation files (the "Software"), to deal | ||
in the Software without restriction, including without limitation the rights | ||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
copies of the Software, and to permit persons to whom the Software is | ||
furnished to do so, subject to the following conditions: | ||
|
||
The above copyright notice and this permission notice shall be included in all | ||
copies or substantial portions of the Software. | ||
|
||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
SOFTWARE. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
from conans import ConanFile, tools | ||
|
||
|
||
class McapConan(ConanFile): | ||
name = "mcap" | ||
version = "1.4.0" | ||
url = "https://github.com/foxglove/mcap" | ||
homepage = "https://github.com/foxglove/mcap" | ||
description = "A C++ implementation of the MCAP file format" | ||
license = "MIT" | ||
topics = ("mcap", "serialization", "deserialization", "recording") | ||
|
||
settings = ("os", "compiler", "build_type", "arch") | ||
requires = ("lz4/1.9.4", "zstd/1.5.2") | ||
generators = "cmake" | ||
|
||
def validate(self): | ||
tools.check_min_cppstd(self, "17") | ||
|
||
def configure(self): | ||
pass | ||
|
||
def package(self): | ||
self.copy(pattern="LICENSE", dst="licenses") | ||
self.copy("include/*") | ||
|
||
def package_id(self): | ||
self.info.header_only() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,108 @@ | ||
#include <array> | ||
#include <cstddef> | ||
#include <cstdint> | ||
|
||
namespace mcap::internal { | ||
|
||
/** | ||
* Compute CRC32 lookup tables as described at: | ||
* https://github.com/komrad36/CRC#option-6-1-byte-tabular | ||
* | ||
* An iteration of CRC computation can be performed on 8 bits of input at once. By pre-computing a | ||
* table of the values of CRC(?) for all 2^8 = 256 possible byte values, during the final | ||
* computation we can replace a loop over 8 bits with a single lookup in the table. | ||
* | ||
* For further speedup, we can also pre-compute the values of CRC(?0) for all possible bytes when a | ||
* zero byte is appended. Then we can process two bytes of input at once by computing CRC(AB) = | ||
* CRC(A0) ^ CRC(B), using one lookup in the CRC(?0) table and one lookup in the CRC(?) table. | ||
* | ||
* The same technique applies for any number of bytes to be processed at once, although the speed | ||
* improvements diminish. | ||
* | ||
* @param Polynomial The binary representation of the polynomial to use (reversed, i.e. most | ||
* significant bit represents x^0). | ||
* @param NumTables The number of bytes of input that will be processed at once. | ||
*/ | ||
template <size_t Polynomial, size_t NumTables> | ||
struct CRC32Table { | ||
private: | ||
std::array<uint32_t, 256 * NumTables> table = {}; | ||
|
||
public: | ||
constexpr CRC32Table() { | ||
for (uint32_t i = 0; i < 256; i++) { | ||
uint32_t r = i; | ||
r = ((r & 1) * Polynomial) ^ (r >> 1); | ||
r = ((r & 1) * Polynomial) ^ (r >> 1); | ||
r = ((r & 1) * Polynomial) ^ (r >> 1); | ||
r = ((r & 1) * Polynomial) ^ (r >> 1); | ||
r = ((r & 1) * Polynomial) ^ (r >> 1); | ||
r = ((r & 1) * Polynomial) ^ (r >> 1); | ||
r = ((r & 1) * Polynomial) ^ (r >> 1); | ||
r = ((r & 1) * Polynomial) ^ (r >> 1); | ||
table[i] = r; | ||
} | ||
for (size_t i = 256; i < table.size(); i++) { | ||
uint32_t value = table[i - 256]; | ||
table[i] = table[value & 0xff] ^ (value >> 8); | ||
} | ||
} | ||
|
||
constexpr uint32_t operator[](size_t index) const { | ||
return table[index]; | ||
} | ||
}; | ||
|
||
inline uint32_t getUint32LE(const std::byte* data) { | ||
return (uint32_t(data[0]) << 0) | (uint32_t(data[1]) << 8) | (uint32_t(data[2]) << 16) | | ||
(uint32_t(data[3]) << 24); | ||
} | ||
|
||
static constexpr CRC32Table<0xedb88320, 8> CRC32_TABLE; | ||
|
||
/** | ||
* Initialize a CRC32 to all 1 bits. | ||
*/ | ||
static constexpr uint32_t CRC32_INIT = 0xffffffff; | ||
|
||
/** | ||
* Update a streaming CRC32 calculation. | ||
* | ||
* For performance, this implementation processes the data 8 bytes at a time, using the algorithm | ||
* presented at: https://github.com/komrad36/CRC#option-9-8-byte-tabular | ||
*/ | ||
inline uint32_t crc32Update(const uint32_t prev, const std::byte* const data, const size_t length) { | ||
// Process bytes one by one until we reach the proper alignment. | ||
uint32_t r = prev; | ||
size_t offset = 0; | ||
for (; (uintptr_t(data + offset) & alignof(uint32_t)) != 0 && offset < length; offset++) { | ||
r = CRC32_TABLE[(r ^ uint8_t(data[offset])) & 0xff] ^ (r >> 8); | ||
} | ||
if (offset == length) { | ||
return r; | ||
} | ||
|
||
// Process 8 bytes (2 uint32s) at a time. | ||
size_t remainingBytes = length - offset; | ||
for (; remainingBytes >= 8; offset += 8, remainingBytes -= 8) { | ||
r ^= getUint32LE(data + offset); | ||
uint32_t r2 = getUint32LE(data + offset + 4); | ||
r = CRC32_TABLE[0 * 256 + ((r2 >> 24) & 0xff)] ^ CRC32_TABLE[1 * 256 + ((r2 >> 16) & 0xff)] ^ | ||
CRC32_TABLE[2 * 256 + ((r2 >> 8) & 0xff)] ^ CRC32_TABLE[3 * 256 + ((r2 >> 0) & 0xff)] ^ | ||
CRC32_TABLE[4 * 256 + ((r >> 24) & 0xff)] ^ CRC32_TABLE[5 * 256 + ((r >> 16) & 0xff)] ^ | ||
CRC32_TABLE[6 * 256 + ((r >> 8) & 0xff)] ^ CRC32_TABLE[7 * 256 + ((r >> 0) & 0xff)]; | ||
} | ||
|
||
// Process any remaining bytes one by one. | ||
for (; offset < length; offset++) { | ||
r = CRC32_TABLE[(r ^ uint8_t(data[offset])) & 0xff] ^ (r >> 8); | ||
} | ||
return r; | ||
} | ||
|
||
/** Finalize a CRC32 by inverting the output value. */ | ||
inline uint32_t crc32Final(uint32_t crc) { | ||
return crc ^ 0xffffffff; | ||
} | ||
|
||
} // namespace mcap::internal |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,120 @@ | ||
#pragma once | ||
|
||
#include <string> | ||
|
||
namespace mcap { | ||
|
||
/** | ||
* @brief Status codes for MCAP readers and writers. | ||
*/ | ||
enum class StatusCode { | ||
Success = 0, | ||
NotOpen, | ||
InvalidSchemaId, | ||
InvalidChannelId, | ||
FileTooSmall, | ||
ReadFailed, | ||
MagicMismatch, | ||
InvalidFile, | ||
InvalidRecord, | ||
InvalidOpCode, | ||
InvalidChunkOffset, | ||
InvalidFooter, | ||
DecompressionFailed, | ||
DecompressionSizeMismatch, | ||
UnrecognizedCompression, | ||
OpenFailed, | ||
MissingStatistics, | ||
InvalidMessageReadOptions, | ||
NoMessageIndexesAvailable, | ||
UnsupportedCompression, | ||
}; | ||
|
||
/** | ||
* @brief Wraps a status code and string message carrying additional context. | ||
*/ | ||
struct [[nodiscard]] Status { | ||
StatusCode code; | ||
std::string message; | ||
|
||
Status() | ||
: code(StatusCode::Success) {} | ||
|
||
Status(StatusCode code) | ||
: code(code) { | ||
switch (code) { | ||
case StatusCode::Success: | ||
break; | ||
case StatusCode::NotOpen: | ||
message = "not open"; | ||
break; | ||
case StatusCode::InvalidSchemaId: | ||
message = "invalid schema id"; | ||
break; | ||
case StatusCode::InvalidChannelId: | ||
message = "invalid channel id"; | ||
break; | ||
case StatusCode::FileTooSmall: | ||
message = "file too small"; | ||
break; | ||
case StatusCode::ReadFailed: | ||
message = "read failed"; | ||
break; | ||
case StatusCode::MagicMismatch: | ||
message = "magic mismatch"; | ||
break; | ||
case StatusCode::InvalidFile: | ||
message = "invalid file"; | ||
break; | ||
case StatusCode::InvalidRecord: | ||
message = "invalid record"; | ||
break; | ||
case StatusCode::InvalidOpCode: | ||
message = "invalid opcode"; | ||
break; | ||
case StatusCode::InvalidChunkOffset: | ||
message = "invalid chunk offset"; | ||
break; | ||
case StatusCode::InvalidFooter: | ||
message = "invalid footer"; | ||
break; | ||
case StatusCode::DecompressionFailed: | ||
message = "decompression failed"; | ||
break; | ||
case StatusCode::DecompressionSizeMismatch: | ||
message = "decompression size mismatch"; | ||
break; | ||
case StatusCode::UnrecognizedCompression: | ||
message = "unrecognized compression"; | ||
break; | ||
case StatusCode::OpenFailed: | ||
message = "open failed"; | ||
break; | ||
case StatusCode::MissingStatistics: | ||
message = "missing statistics"; | ||
break; | ||
case StatusCode::InvalidMessageReadOptions: | ||
message = "message read options conflict"; | ||
break; | ||
case StatusCode::NoMessageIndexesAvailable: | ||
message = "file has no message indices"; | ||
break; | ||
case StatusCode::UnsupportedCompression: | ||
message = "unsupported compression"; | ||
break; | ||
default: | ||
message = "unknown"; | ||
break; | ||
} | ||
} | ||
|
||
Status(StatusCode code, const std::string& message) | ||
: code(code) | ||
, message(message) {} | ||
|
||
bool ok() const { | ||
return code == StatusCode::Success; | ||
} | ||
}; | ||
|
||
} // namespace mcap |
Oops, something went wrong.