Skip to content

Commit

Permalink
feat(mount): add log on OS notification area
Browse files Browse the repository at this point in the history
This change adds a log on the specific OS notification area,
specifically for errors associated with the reading, writing and master
disconnection processes, which allows the user to have a proper feedback
on the each error.

Signed-off-by: rolysr <rolysr@leil.io>
  • Loading branch information
rolysr committed Feb 4, 2025
1 parent e072eba commit e3fabec
Show file tree
Hide file tree
Showing 9 changed files with 251 additions and 9 deletions.
2 changes: 2 additions & 0 deletions src/mount/fuse/main.cc
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,8 @@ static int mainloop(struct fuse_args *args, struct fuse_cmdline_opts *fuse_opts,
params.debug_mode = gMountOptions.debug;
params.direct_io = gMountOptions.directio;
params.ignore_flush = gMountOptions.ignoreflush;
params.log_notifications_area = gMountOptions.lognotificationarea;
params.message_suppression_period = gMountOptions.messagesuppressionperiod;

if (!gMountOptions.meta) {
SaunaClient::fs_init(params);
Expand Down
9 changes: 8 additions & 1 deletion src/mount/fuse/mount_config.cc
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ struct fuse_opt gSfsOptsStage2[] = {
SFS_OPT("nostdmountoptions", nostdmountoptions, 1),
SFS_OPT("sfsignoreflush=%d", ignoreflush, 0),
SFS_OPT("limitglibcmallocarenas=%d", limitglibcmallocarenas, 0),
SFS_OPT("sfslognotificationarea=%s", lognotificationarea, 0),
SFS_OPT("sfsmessagesuppressionperiod=%u", messagesuppressionperiod, 0),

SFS_OPT("enablefilelocks=%u", filelocks, 0),
SFS_OPT("nonempty", nonemptymount, 1),
Expand Down Expand Up @@ -239,6 +241,9 @@ void usage(const char *progname) {
" -o limitglibcmallocarenas=N limit glibc malloc arenas to given value - prevents "
"from using huge amount of virtual memory. Use it in constrained memory "
"environments (default: %u)\n"
" -o sfslognotificationarea=0|1 enable/disable logging to Linux notification area (default: %d)\n"
" -o sfsmessagesuppressionperiod=N set period of message suppression in seconds for logging on "
"notification area (default: %u)\n"
"\n",
SaunaClient::FsInitParams::kDefaultCacheExpirationTime,
SaunaClient::FsInitParams::kDefaultReadaheadMaxWindowSize,
Expand Down Expand Up @@ -269,7 +274,9 @@ void usage(const char *progname) {
SaunaClient::FsInitParams::kDefaultIoRetries,
SaunaClient::FsInitParams::kDefaultSymlinkCacheTimeout,
SaunaClient::FsInitParams::kDefaultSubfolder,
SaunaClient::FsInitParams::kDefaultLimitGlibcMallocArenas
SaunaClient::FsInitParams::kDefaultLimitGlibcMallocArenas,
SaunaClient::FsInitParams::kDefaultLogNotificationArea,
SaunaClient::FsInitParams::kDefaultMessageSuppressionPeriod
);
printf(
"CMODE can be set to:\n"
Expand Down
2 changes: 2 additions & 0 deletions src/mount/fuse/mount_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ struct sfsopts_ {
bool directio;
int ignoreflush;
unsigned limitglibcmallocarenas;
int lognotificationarea;
unsigned messagesuppressionperiod;

sfsopts_()
: masterhost(NULL),
Expand Down
2 changes: 2 additions & 0 deletions src/mount/mastercomm.cc
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
#include "common/sockets.h"
#include "slogger/slogger.h"
#include "mount/exports.h"
#include "mount/notification_area_logging.h"
#include "mount/stats.h"
#include "protocol/cltoma.h"
#include "protocol/matocl.h"
Expand Down Expand Up @@ -206,6 +207,7 @@ static inline void setDisconnect(bool value) {
if(value) {
SaunaClient::masterDisconnectedCallback();
safs_pretty_syslog(LOG_WARNING,"master: disconnected");
addNotificationMessage("SaunaFS master disconnected");
}
}

Expand Down
188 changes: 188 additions & 0 deletions src/mount/notification_area_logging.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
/*
Copyright 2023 Leil Storage OÜ
SaunaFS is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3.
SaunaFS is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with SaunaFS If not, see <http://www.gnu.org/licenses/>.
*/

#pragma once

#include "common/platform.h"

#ifdef _WIN32
#include <shellapi.h>
#include <windows.h>
#include <string>
#else
#include <cstdlib>
#endif

#include <chrono>
#include <condition_variable>
#include <mutex>
#include <string>
#include <thread>
#include <unordered_map>

inline bool gShowMessagesOnNotificationArea = false;

struct MessageCache {
std::string message;
std::chrono::steady_clock::time_point timestamp;
};

inline std::jthread notificationThread;

inline std::unordered_map<std::string, MessageCache> messageCache;
inline std::map<int, MessageCache> fullPathFromInodeCache;
inline std::chrono::seconds gMessageSuppressionPeriodSeconds;
inline std::mutex messageCacheMutex;
inline std::condition_variable cv;
inline std::atomic<bool> stopThread = false;

#ifdef _WIN32
inline void ShowWindowsNotification(const std::string &message) {
// Create a hidden window
HWND hWnd = CreateWindowEx(0, "STATIC", "HiddenWindow", 0, 0, 0, 0, 0,
HWND_MESSAGE, NULL, NULL, NULL);

// Define the NOTIFYICONDATA structure
NOTIFYICONDATA nid = {};
nid.cbSize = sizeof(NOTIFYICONDATA);
nid.hWnd = hWnd; // Handle to the hidden window
nid.uID = 1001; // Unique ID for the notification icon
nid.uFlags = NIF_INFO | NIF_ICON; // Display an info balloon and an icon
nid.dwInfoFlags = NIIF_INFO; // Info icon
strcpy_s(nid.szInfo, message.c_str()); // Set the message
strcpy_s(nid.szInfoTitle, "SaunaFS Windows Client"); // Set the title
nid.hIcon = LoadIcon(NULL, IDI_INFORMATION); // Load a standard icon

// Display the notification
if (!Shell_NotifyIcon(NIM_ADD, &nid)) {
// Handle error
safs::log_debug(
"ShowMessageOnWindowsNotificationArea: Failed to "
"display notification icon");
}

// Clean up
DestroyWindow(hWnd);
}
#else
inline void ShowLinuxNotification(const std::string &message) {
std::string command =
R"(notify-send "SaunaFS Linux Client" ")" + message + "\"";

if (std::system(command.c_str()) != 0) {
safs::log_debug(
"addNotificationMessage: Failed to show notification using "
"notify-send");
}
}
#endif

inline void NotificationThread() {
while (!stopThread) {
std::unique_lock<std::mutex> lock(messageCacheMutex);
cv.wait(lock, [] { return !messageCache.empty() || stopThread; });

if (stopThread) { break; }

auto now = std::chrono::steady_clock::now();
for (auto it = messageCache.begin(); it != messageCache.end();) {
if ((now - it->second.timestamp) >=
gMessageSuppressionPeriodSeconds) {
// Show the message
#ifdef _WIN32
ShowWindowsNotification(it->second.message);
#else
ShowLinuxNotification(it->second.message);
#endif
it = messageCache.erase(it);
} else {
++it;
}
}

for (auto it = fullPathFromInodeCache.begin();
it != fullPathFromInodeCache.end();) {
if ((now - it->second.timestamp) >=
gMessageSuppressionPeriodSeconds) {
it = fullPathFromInodeCache.erase(it);
} else {
++it;
}
}
}
}

inline void addNotificationMessage(const std::string &message) {
if (stopThread || !gShowMessagesOnNotificationArea) { return; }
auto now = std::chrono::steady_clock::now();

std::unique_lock<std::mutex> lock(messageCacheMutex);
auto it = messageCache.find(message);
if (it != messageCache.end() &&
(now - it->second.timestamp) < gMessageSuppressionPeriodSeconds) {
// Suppress the message
return;
}

// Update the cache with the current message and timestamp
messageCache[message] = {message, now};
cv.notify_one();
}

inline void addPathByInodeBasedNotificationMessage(const std::string &message,
uint32_t inode) {
if (stopThread || !gShowMessagesOnNotificationArea) { return; }
auto now = std::chrono::steady_clock::now();

std::string fullPath;
std::unique_lock<std::mutex> lock(messageCacheMutex);
auto it = fullPathFromInodeCache.find(inode);
if (it != fullPathFromInodeCache.end() &&
(now - it->second.timestamp) < gMessageSuppressionPeriodSeconds) {
fullPath = it->second.message;
} else {
fs_fullpath(inode, 0, 0, fullPath);
fullPathFromInodeCache[inode] = {fullPath, now};
}
lock.unlock();

addNotificationMessage(message + ": " + fullPath);
}

inline void StartNotificationThread() {
stopThread = false;
notificationThread = std::jthread(NotificationThread);
}

inline void StopNotificationThread() {
{
std::unique_lock<std::mutex> lock(messageCacheMutex);
stopThread = true;
cv.notify_all();
}
if (notificationThread.joinable()) { notificationThread.join(); }
}

inline void notifications_area_logging_init(
bool log_notification_area, unsigned message_suppression_period) {
gShowMessagesOnNotificationArea = log_notification_area;
gMessageSuppressionPeriodSeconds =
std::chrono::seconds(message_suppression_period);

StartNotificationThread();
}

inline void notifications_area_logging_term() { StopNotificationThread(); }
13 changes: 13 additions & 0 deletions src/mount/readdata.cc
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
#include "mount/chunk_locator.h"
#include "mount/chunk_reader.h"
#include "mount/mastercomm.h"
#include "mount/notification_area_logging.h"
#include "mount/readahead_adviser.h"
#include "mount/readdata_cache.h"
#include "mount/memory_info.h"
Expand All @@ -66,6 +67,7 @@ uint64_t successfulTimesRequestedMemory = 0;
constexpr double kTimesRequestedMemoryLowerSuccessRate = 0.3;
constexpr double kTimesRequestedMemoryUpperSuccessRate = 0.8;
constexpr uint32_t kMinCacheExpirationTime = 1;
constexpr uint32_t kMinTryCounterToShowReadErrorMessage = 9;

std::unique_ptr<IMemoryInfo> createMemoryInfo() {
std::unique_ptr<IMemoryInfo> memoryInfo;
Expand Down Expand Up @@ -753,6 +755,9 @@ int read_to_buffer(ReadRecord *rrec, uint64_t current_offset,
try_counter = 0;
} catch (UnrecoverableReadException &ex) {
print_error_msg(reader, try_counter, ex);
addPathByInodeBasedNotificationMessage(
"Unrecoverable read error: " + std::string(ex.what()),
rrec->inode);
std::unique_lock usedMemoryLock(gReadCacheMemoryMutex);
decreaseUsedReadCacheMemory(total_read_cache_bytes_to_reserve);
usedMemoryLock.unlock();
Expand All @@ -767,11 +772,19 @@ int read_to_buffer(ReadRecord *rrec, uint64_t current_offset,
}
force_prepare = true;
if (try_counter > maxRetries) {
addPathByInodeBasedNotificationMessage(
"Read error: Exceeded max retries:" +
std::string(ex.what()),
rrec->inode);
std::unique_lock usedMemoryLock(gReadCacheMemoryMutex);
decreaseUsedReadCacheMemory(total_read_cache_bytes_to_reserve);
usedMemoryLock.unlock();
return SAUNAFS_ERROR_IO;
} else {
if (try_counter > kMinTryCounterToShowReadErrorMessage) {
addPathByInodeBasedNotificationMessage(
"Read error: " + std::string(ex.what()), rrec->inode);
}
std::unique_lock usedMemoryLock(gReadCacheMemoryMutex);
decreaseUsedReadCacheMemory(last_read_cache_bytes_to_reserve);
total_read_cache_bytes_to_reserve -= last_read_cache_bytes_to_reserve;
Expand Down
5 changes: 5 additions & 0 deletions src/mount/sauna_client.cc
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
#include "mount/io_limit_group.h"
#include "mount/mastercomm.h"
#include "mount/masterproxy.h"
#include "mount/notification_area_logging.h"
#include "mount/oplog.h"
#include "mount/readdata.h"
#include "mount/special_inode.h"
Expand Down Expand Up @@ -3540,6 +3541,9 @@ void fs_init(FsInitParams &params) {
set_debug_mode(params.debug_mode);
#endif

notifications_area_logging_init(params.log_notifications_area,
params.message_suppression_period);

init(params.debug_mode, params.keep_cache, params.direntry_cache_timeout, params.direntry_cache_size,
params.entry_cache_timeout, params.attr_cache_timeout, params.mkdir_copy_sgid,
params.sugid_clear_mode, params.use_rw_lock,
Expand All @@ -3559,6 +3563,7 @@ void fs_term() {
::fs_term();
symlink_cache_term();
socketrelease();
notifications_area_logging_term();
}

} // namespace SaunaClient
26 changes: 18 additions & 8 deletions src/mount/sauna_client.h
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ struct FsInitParams {
static constexpr float kDefaultBandwidthOveruse = 1.0;
static constexpr unsigned kDefaultChunkserverWriteTo = 5000;
static constexpr bool kDefaultIgnoreFlush = false;
static constexpr int kDefaultLogNotificationArea = 0;
static constexpr unsigned kDefaultMessageSuppressionPeriod = 10;
#ifdef _WIN32
static constexpr unsigned kDefaultWriteCacheSize = 50;
static constexpr unsigned kDefaultCleanAcquiredFilesPeriod = 0;
Expand Down Expand Up @@ -162,9 +164,13 @@ struct FsInitParams {
enable_status_updater_thread(kDefaultEnableStatusUpdaterThread),
ignore_utimens_update(kDefaultIgnoreUtimensUpdate),
#endif
ignore_flush(kDefaultIgnoreFlush), verbose(kDefaultVerbose), direct_io(kDirectIO)
,limit_glibc_malloc_arenas(kDefaultLimitGlibcMallocArenas)
{
ignore_flush(kDefaultIgnoreFlush),
verbose(kDefaultVerbose),
direct_io(kDirectIO),
limit_glibc_malloc_arenas(kDefaultLimitGlibcMallocArenas),
log_notifications_area(kDefaultLogNotificationArea),
message_suppression_period(kDefaultMessageSuppressionPeriod) {

}

FsInitParams(const std::string &bind_host, const std::string &host, const std::string &port, const std::string &mountpoint)
Expand Down Expand Up @@ -201,11 +207,13 @@ struct FsInitParams {
enable_status_updater_thread(kDefaultEnableStatusUpdaterThread),
ignore_utimens_update(kDefaultIgnoreUtimensUpdate),
#endif
ignore_flush(kDefaultIgnoreFlush), verbose(kDefaultVerbose), direct_io(kDirectIO)
#ifndef _WIN32
,limit_glibc_malloc_arenas(kDefaultLimitGlibcMallocArenas)
#endif
{
ignore_flush(kDefaultIgnoreFlush),
verbose(kDefaultVerbose),
direct_io(kDirectIO),
limit_glibc_malloc_arenas(kDefaultLimitGlibcMallocArenas),
log_notifications_area(kDefaultLogNotificationArea),
message_suppression_period(kDefaultMessageSuppressionPeriod) {

}

std::string bind_host;
Expand Down Expand Up @@ -265,6 +273,8 @@ struct FsInitParams {
bool verbose;
bool direct_io;
unsigned limit_glibc_malloc_arenas;
int log_notifications_area;
unsigned message_suppression_period;

std::string io_limits_config_file;
};
Expand Down
Loading

0 comments on commit e3fabec

Please sign in to comment.