Illustrated C/C++ source sheets, symbol index, API relationship graph, compiler console and debug trace

LIVE AI / FLEXIBLE SENSING / FLIGHT CONTEXT

Sensoraia API

Live AI and sensing, connected to the flight.
See, listen, inspect and return supported results while airborne.

Loading...
Searching...
No Matches
map_storage.h
Go to the documentation of this file.
1/*
2 * Copyright 2026 OpenUAS
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18/**
19 * @file map_storage.h
20 * @brief Integrity checks and bounded disk-backed intermediates for surveys.
21 * @details Checkpoints are identified by content hashes and published through
22 * fsync plus rename so resume can distinguish complete work from interrupted
23 * writes. SHA-256 here detects accidental mismatch; it is not an authenticity
24 * or security boundary.
25 */
26#ifndef PANGEA_MAP_STORAGE_H
27#define PANGEA_MAP_STORAGE_H
28
29#include "sha256fst.h"
30#include <opencv2/core.hpp>
31#include <filesystem>
32#include <fstream>
33#include <array>
34#include <algorithm>
35#include <functional>
36#include <cerrno>
37#include <cstdlib>
38#include <fcntl.h>
39#include <unistd.h>
40#include <sys/stat.h>
41#include <sys/statvfs.h>
42
43namespace pangea_survey {
44/** @brief Own a working-file descriptor through success and exception paths. */
47public:
48 /** @brief Adopt a descriptor without duplicating it.
49 * @param descriptor Owned descriptor, or -1 for no open file. */
51 /** @brief Noncopyable: ownership cannot be shared by copying the FD number. */
53 /** @brief Nonassignable: use reset() to replace the owned descriptor. */
55 /** @brief Close an owned descriptor; use close() when the status matters. */
57 /** @brief Borrow the descriptor without transferring ownership.
58 * @return Open descriptor number, or -1. */
59 int get() const { return descriptor_; }
60 /** @brief Relinquish the old descriptor and adopt a distinct replacement.
61 * @param descriptor New owned descriptor, or -1 to leave the slot empty.
62 * @details The old close status is not reported; do not pass get() back in. */
63 void reset(int descriptor) {
66 }
67 /** @brief Close explicitly and report the system call result.
68 * @return Close status; the owner is empty even when close fails.
69 * @details Clear ownership first so destruction cannot close a reused number. */
70 int close() {
71 const int descriptor = descriptor_;
72 descriptor_ = -1;
73 return ::close(descriptor);
74 }
75};
76/** @brief Compute a lowercase SHA-256 digest for an in-memory byte range. */
77inline std::string dataDigest(const void* data, std::size_t size) {
78 std::array<char, SHA256_HEX_SIZE> digest{};
79 sha256_hex(data, size, digest.data());
80 return digest.data();
81}
82/**
83 * @brief Hash a file in bounded chunks, optionally honoring cancellation.
84 * @param[in] path Input file.
85 * @param[in] stopped Optional cooperative cancellation predicate.
86 * @return Lowercase SHA-256 digest.
87 * @throws std::runtime_error If the file cannot be read or cancellation is requested.
88 */
89inline std::string fileDigest(const std::filesystem::path& path, const std::function<bool()>& stopped = {}) {
90 if (stopped && stopped()) throw std::runtime_error("Checksum verification canceled");
91 StorageDescriptor input(open(path.c_str(), O_RDONLY | O_NONBLOCK | O_CLOEXEC | O_NOFOLLOW));
92 struct stat before{};
93 if (input.get() < 0 || fstat(input.get(), &before) != 0 || !S_ISREG(before.st_mode) || before.st_size < 0)
94 throw std::runtime_error("Cannot hash regular working input: " + path.string());
95 sha256 context; sha256_init(&context);
96 std::array<char, 65536> buffer{};
97 auto remaining = static_cast<std::uint64_t>(before.st_size);
98 while (remaining) {
99 if (stopped && stopped()) throw std::runtime_error("Checksum verification canceled");
100 const auto count = read(input.get(), buffer.data(), std::min<std::uint64_t>(remaining, buffer.size()));
101 if (count < 0 && errno == EINTR) continue;
102 if (count <= 0) throw std::runtime_error("Cannot finish input checksum");
103 sha256_append(&context, buffer.data(), static_cast<std::size_t>(count));
104 remaining -= static_cast<std::uint64_t>(count);
105 }
106 struct stat after{};
107 if (fstat(input.get(), &after) != 0 || before.st_size != after.st_size ||
108 before.st_mtim.tv_sec != after.st_mtim.tv_sec || before.st_mtim.tv_nsec != after.st_mtim.tv_nsec ||
109 before.st_ctim.tv_sec != after.st_ctim.tv_sec || before.st_ctim.tv_nsec != after.st_ctim.tv_nsec)
110 throw std::runtime_error("Working input changed during checksum");
111 std::array<char, SHA256_HEX_SIZE> digest{};
112 sha256_finalize_hex(&context, digest.data());
113 return digest.data();
114}
115/**
116 * @brief Require requested free space plus a fixed safety reserve.
117 * @param[in] directory Filesystem on which work will be created.
118 * @param[in] bytes Estimated bytes needed by the next operation.
119 * @throws std::runtime_error If free space cannot be queried or is insufficient.
120 */
121inline void storageHeadroom(const std::filesystem::path& directory, std::uint64_t bytes) {
122 struct statvfs status{};
123 constexpr std::uint64_t reserve = 512ull * 1024 * 1024;
124 if (bytes > UINT64_MAX - reserve || statvfs(directory.c_str(), &status) != 0 || !status.f_frsize ||
125 status.f_bavail < (bytes + reserve) / status.f_frsize + ((bytes + reserve) % status.f_frsize != 0))
126 throw std::runtime_error("Insufficient working disk space; checkpoint retained");
127}
128/**
129 * @brief Fsync a completed working file, rename it, then sync its directory.
130 * @param[in] temporary Completed sibling temporary file.
131 * @param[in] target Final checkpoint path on the same filesystem.
132 * @param replace True to rename over the target; false to publish without replacement.
133 * @throws std::runtime_error If synchronization, rename, or directory access fails.
134 */
135inline void publishWorkingFile(const std::filesystem::path& temporary, const std::filesystem::path& target,
136 bool replace = true) {
137 StorageDescriptor descriptor(open(temporary.c_str(), O_RDONLY | O_NONBLOCK | O_CLOEXEC | O_NOFOLLOW));
138 struct stat completed{};
139 if (descriptor.get() < 0 || fstat(descriptor.get(), &completed) != 0 || !S_ISREG(completed.st_mode))
140 throw std::runtime_error("Cannot open completed regular working file");
141 const int status = fsync(descriptor.get());
142 const int closed = descriptor.close();
143 if (status != 0 || closed != 0) throw std::runtime_error("Cannot synchronize working file");
144 if (replace) std::filesystem::rename(temporary, target);
145 else {
146 if (link(temporary.c_str(), target.c_str()) != 0)
147 throw std::runtime_error("Cannot exclusively publish working file");
148 if (unlink(temporary.c_str()) != 0)
149 throw std::runtime_error("Cannot remove published working-file temporary");
150 }
151 const auto parent = target.has_parent_path() ? target.parent_path() : std::filesystem::path(".");
152 StorageDescriptor directory(open(parent.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC));
153 if (directory.get() < 0) throw std::runtime_error("Cannot open checkpoint directory");
154 const int synced = fsync(directory.get());
155 if (synced != 0) throw std::runtime_error("Cannot synchronize checkpoint directory");
156}
157/** @brief Save checkpoint text through the durable working-file publisher. */
158inline void saveWorkingText(const std::filesystem::path& target, const std::string& text) {
159 auto temporary = target.string() + ".partial.XXXXXX";
160 StorageDescriptor output(mkstemp(temporary.data()));
161 if (output.get() < 0) throw std::runtime_error("Cannot create working metadata");
162 try {
163 if (fcntl(output.get(), F_SETFD, FD_CLOEXEC) != 0)
164 throw std::runtime_error("Cannot protect working metadata descriptor");
165 std::size_t written = 0;
166 while (written < text.size()) {
167 const auto count = write(output.get(), text.data() + written,
168 std::min<std::size_t>(text.size() - written, 65536));
169 if (count < 0 && errno == EINTR) continue;
170 if (count <= 0) throw std::runtime_error("Cannot save working metadata");
171 written += static_cast<std::size_t>(count);
172 }
173 if (output.close() != 0) throw std::runtime_error("Cannot close working metadata");
174 publishWorkingFile(temporary, target);
175 } catch (...) {
176 unlink(temporary.c_str());
177 throw;
178 }
179}
180/**
181 * @brief Read bounded rectangular regions from a lossless BGRA disk raster.
182 * @details Disk-backed source pixels trade I/O for a small resident working
183 * set; each read is capped so callers cannot accidentally materialize an
184 * unbounded source image through this interface.
185 */
187 std::filesystem::path path_;
189public:
190 /**
191 * @brief Open a raster whose byte length must match its declared dimensions.
192 * @param[in] path Raster file containing row-major BGRA bytes.
193 * @param[in] width Raster width in pixels.
194 * @param[in] height Raster height in pixels.
195 * @throws std::runtime_error If dimensions or file length are invalid.
196 */
197 DiskRaster(std::filesystem::path path, int width, int height)
198 : path_(std::move(path)), width_(width), height_(height) {
199 if (width < 1 || height < 1 || width > 8192 || height > 8192 ||
200 std::filesystem::file_size(path_) != static_cast<std::uint64_t>(width) * height * 4)
201 throw std::runtime_error("Invalid lossless working raster");
202 }
203 /**
204 * @brief Read a rectangle into a new BGRA matrix.
205 * @param[in] rectangle Nonempty in-bounds raster region no larger than 16 MiB.
206 * @return Owned BGRA pixels for the requested rectangle.
207 * @throws std::runtime_error If the rectangle exceeds bounds or file reading fails.
208 */
209 cv::Mat read(const cv::Rect& rectangle) const {
210 if (rectangle.empty() || (rectangle & cv::Rect(0, 0, width_, height_)) != rectangle ||
211 static_cast<std::uint64_t>(rectangle.area()) * 4 > 16ull * 1024 * 1024)
212 throw std::runtime_error("Working raster read exceeds bounded region");
213 std::ifstream input(path_, std::ios::binary);
214 cv::Mat result(rectangle.size(), CV_8UC4);
215 for (int row = 0; row < rectangle.height; ++row) {
216 input.seekg((static_cast<std::uint64_t>(rectangle.y + row) * width_ + rectangle.x) * 4);
217 input.read(reinterpret_cast<char*>(result.ptr(row)), rectangle.width * 4);
218 if (!input) throw std::runtime_error("Cannot read lossless working raster");
219 }
220 return result;
221 }
222};
223}
224#endif
size_t size
Definition chdk_pipe.c:143
uint8_t bytes[CHDK_CONTROL_REPLY_SIZE]
Definition chdk_pipe.c:144
int descriptor
Definition chdk_pipe.c:142
int(* open)(const struct usbfs_device *, struct usbfs_handle *)
Definition chdkbridge.c:56
cv::Mat read(const cv::Rect &rectangle) const
Read a rectangle into a new BGRA matrix.
DiskRaster(std::filesystem::path path, int width, int height)
Open a raster whose byte length must match its declared dimensions.
std::filesystem::path path_
Own a working-file descriptor through success and exception paths.
Definition map_storage.h:45
~StorageDescriptor()
Close an owned descriptor; use close() when the status matters.
Definition map_storage.h:56
int close()
Close explicitly and report the system call result.
Definition map_storage.h:70
int get() const
Borrow the descriptor without transferring ownership.
Definition map_storage.h:59
void reset(int descriptor)
Relinquish the old descriptor and adopt a distinct replacement.
Definition map_storage.h:63
StorageDescriptor(const StorageDescriptor &)=delete
Noncopyable: ownership cannot be shared by copying the FD number.
StorageDescriptor & operator=(const StorageDescriptor &)=delete
Nonassignable: use reset() to replace the owned descriptor.
StorageDescriptor(int descriptor=-1)
Adopt a descriptor without duplicating it.
Definition map_storage.h:50
static struct ear_inference_result queued_evidence completed
uint8_t row[LWIR_MAX_WIDTH *3U]
Definition lwir_image.c:63
std::string digest(const void *data, size_t size)
Definition pangea.cpp:615
void storageHeadroom(const std::filesystem::path &directory, std::uint64_t bytes)
Require requested free space plus a fixed safety reserve.
std::string fileDigest(const std::filesystem::path &path, const std::function< bool()> &stopped={})
Hash a file in bounded chunks, optionally honoring cancellation.
Definition map_storage.h:89
void publishWorkingFile(const std::filesystem::path &temporary, const std::filesystem::path &target, bool replace=true)
Fsync a completed working file, rename it, then sync its directory.
std::string dataDigest(const void *data, std::size_t size)
Compute a lowercase SHA-256 digest for an in-memory byte range.
Definition map_storage.h:77
void saveWorkingText(const std::filesystem::path &target, const std::string &text)
Save checkpoint text through the durable working-file publisher.
static size_t count
Definition pose_log.c:79
static FILE * output
Definition pose_log.c:77
static uint64_t synced
Definition pose_log.c:81
void sha256_hex(const void *data, size_t length, char *output)
Hash one memory region to a lowercase hexadecimal string.
Definition sha256fst.c:272
void sha256_finalize_hex(struct sha256 *context, char *output)
Finish a message and write its lowercase hexadecimal digest.
Definition sha256fst.c:249
void sha256_append(struct sha256 *context, const void *data, size_t length)
Append an arbitrarily aligned chunk; empty chunks are harmless.
Definition sha256fst.c:190
void sha256_init(struct sha256 *context)
Initialize or reset a context for a new message.
Definition sha256fst.c:168
Allocation-free, portable SHA-256 checksums for memory and streams.
struct sha256 sha256
Caller-owned incremental checksum state; treat members as read-only.
struct usher_control_result result
Definition usher.c:377