diff --git a/src/mount/fuse/main.cc b/src/mount/fuse/main.cc
index 27496f97b..75166b4d1 100644
--- a/src/mount/fuse/main.cc
+++ b/src/mount/fuse/main.cc
@@ -261,6 +261,8 @@ static int mainloop(struct fuse_args *args, struct fuse_cmdline_opts *fuse_opts,
params.debug_mode = gMountOptions.debug;
params.direct_io = gMountOptions.directio;
params.ignore_flush = gMountOptions.ignoreflush;
+ params.log_notifications_area = gMountOptions.lognotificationarea;
+ params.message_suppression_period = gMountOptions.messagesuppressionperiod;
if (!gMountOptions.meta) {
SaunaClient::fs_init(params);
diff --git a/src/mount/fuse/mount_config.cc b/src/mount/fuse/mount_config.cc
index 171d1a8b2..b1fde187e 100644
--- a/src/mount/fuse/mount_config.cc
+++ b/src/mount/fuse/mount_config.cc
@@ -94,6 +94,8 @@ struct fuse_opt gSfsOptsStage2[] = {
SFS_OPT("nostdmountoptions", nostdmountoptions, 1),
SFS_OPT("sfsignoreflush=%d", ignoreflush, 0),
SFS_OPT("limitglibcmallocarenas=%d", limitglibcmallocarenas, 0),
+ SFS_OPT("sfslognotificationarea=%d", lognotificationarea, 0),
+ SFS_OPT("sfsmessagesuppressionperiod=%u", messagesuppressionperiod, 0),
SFS_OPT("enablefilelocks=%u", filelocks, 0),
SFS_OPT("nonempty", nonemptymount, 1),
@@ -239,6 +241,9 @@ void usage(const char *progname) {
" -o limitglibcmallocarenas=N limit glibc malloc arenas to given value - prevents "
"from using huge amount of virtual memory. Use it in constrained memory "
"environments (default: %u)\n"
+" -o sfslognotificationarea=0|1 enable/disable logging to Linux notification area (default: %d)\n"
+" -o sfsmessagesuppressionperiod=N set period of message suppression in seconds for logging on "
+ "notification area (default: %u)\n"
"\n",
SaunaClient::FsInitParams::kDefaultCacheExpirationTime,
SaunaClient::FsInitParams::kDefaultReadaheadMaxWindowSize,
@@ -269,7 +274,9 @@ void usage(const char *progname) {
SaunaClient::FsInitParams::kDefaultIoRetries,
SaunaClient::FsInitParams::kDefaultSymlinkCacheTimeout,
SaunaClient::FsInitParams::kDefaultSubfolder,
- SaunaClient::FsInitParams::kDefaultLimitGlibcMallocArenas
+ SaunaClient::FsInitParams::kDefaultLimitGlibcMallocArenas,
+ SaunaClient::FsInitParams::kDefaultLogNotificationArea,
+ SaunaClient::FsInitParams::kDefaultMessageSuppressionPeriod
);
printf(
"CMODE can be set to:\n"
diff --git a/src/mount/fuse/mount_config.h b/src/mount/fuse/mount_config.h
index 25b11021b..d6c6785c7 100644
--- a/src/mount/fuse/mount_config.h
+++ b/src/mount/fuse/mount_config.h
@@ -116,6 +116,8 @@ struct sfsopts_ {
bool directio;
int ignoreflush;
unsigned limitglibcmallocarenas;
+ int lognotificationarea;
+ unsigned messagesuppressionperiod;
sfsopts_()
: masterhost(NULL),
diff --git a/src/mount/mastercomm.cc b/src/mount/mastercomm.cc
index 29fefb644..3258f762f 100644
--- a/src/mount/mastercomm.cc
+++ b/src/mount/mastercomm.cc
@@ -51,6 +51,7 @@
#include "common/sockets.h"
#include "slogger/slogger.h"
#include "mount/exports.h"
+#include "mount/notification_area_logging.h"
#include "mount/stats.h"
#include "protocol/cltoma.h"
#include "protocol/matocl.h"
@@ -206,6 +207,7 @@ static inline void setDisconnect(bool value) {
if(value) {
SaunaClient::masterDisconnectedCallback();
safs_pretty_syslog(LOG_WARNING,"master: disconnected");
+ addNotificationMessage("SaunaFS master disconnected");
}
}
diff --git a/src/mount/notification_area_logging.h b/src/mount/notification_area_logging.h
new file mode 100644
index 000000000..7a7518221
--- /dev/null
+++ b/src/mount/notification_area_logging.h
@@ -0,0 +1,189 @@
+/*
+ Copyright 2023 Leil Storage OÜ
+
+ SaunaFS is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3.
+
+ SaunaFS is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with SaunaFS If not, see .
+ */
+
+#pragma once
+
+#include "common/platform.h"
+
+#ifdef _WIN32
+#include
+#include
+#include
+#else
+#include
+#endif
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+inline bool gShowMessagesOnNotificationArea = false;
+
+struct MessageCache {
+ std::string message;
+ std::chrono::steady_clock::time_point timestamp;
+};
+
+inline std::jthread notificationThread;
+
+inline std::unordered_map messageCache;
+inline std::map fullPathFromInodeCache;
+inline std::chrono::seconds gMessageSuppressionPeriodSeconds;
+inline std::mutex messageCacheMutex;
+inline std::condition_variable cv;
+inline std::atomic stopThread = false;
+
+#ifdef _WIN32
+inline void ShowWindowsNotification(const std::string &message) {
+ // Create a hidden window
+ HWND hWnd = CreateWindowEx(0, "STATIC", "HiddenWindow", 0, 0, 0, 0, 0,
+ HWND_MESSAGE, NULL, NULL, NULL);
+
+ // Define the NOTIFYICONDATA structure
+ NOTIFYICONDATA nid = {};
+ nid.cbSize = sizeof(NOTIFYICONDATA);
+ nid.hWnd = hWnd; // Handle to the hidden window
+ nid.uID = 1001; // Unique ID for the notification icon
+ nid.uFlags = NIF_INFO | NIF_ICON; // Display an info balloon and an icon
+ nid.dwInfoFlags = NIIF_INFO; // Info icon
+ strcpy_s(nid.szInfo, message.c_str()); // Set the message
+ strcpy_s(nid.szInfoTitle, "SaunaFS Windows Client"); // Set the title
+ nid.hIcon = LoadIcon(NULL, IDI_INFORMATION); // Load a standard icon
+
+ // Display the notification
+ if (!Shell_NotifyIcon(NIM_ADD, &nid)) {
+ // Handle error
+ safs::log_debug(
+ "ShowMessageOnWindowsNotificationArea: Failed to "
+ "display notification icon");
+ }
+
+ // Clean up
+ DestroyWindow(hWnd);
+}
+#else
+inline void ShowLinuxNotification(const std::string &message) {
+ std::string command =
+ R"(notify-send "SaunaFS Linux Client" ")" + message + "\"";
+
+ if (std::system(command.c_str()) != 0) {
+ safs::log_debug(
+ "addNotificationMessage: Failed to show notification using "
+ "notify-send");
+ }
+}
+#endif
+
+inline void NotificationThread() {
+ while (!stopThread) {
+ std::unique_lock lock(messageCacheMutex);
+ cv.wait(lock, [] { return !messageCache.empty() || stopThread; });
+
+ if (stopThread) { break; }
+
+ auto now = std::chrono::steady_clock::now();
+ for (auto it = messageCache.begin(); it != messageCache.end();) {
+ if ((now - it->second.timestamp) >=
+ gMessageSuppressionPeriodSeconds) {
+ // Show the message
+#ifdef _WIN32
+ ShowWindowsNotification(it->second.message);
+#else
+ ShowLinuxNotification(it->second.message);
+#endif
+ it = messageCache.erase(it);
+ } else {
+ ++it;
+ }
+ }
+
+ for (auto it = fullPathFromInodeCache.begin();
+ it != fullPathFromInodeCache.end();) {
+ if ((now - it->second.timestamp) >=
+ gMessageSuppressionPeriodSeconds) {
+ it = fullPathFromInodeCache.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+}
+
+inline void addNotificationMessage(const std::string &message) {
+ if (stopThread || !gShowMessagesOnNotificationArea) { return; }
+ auto now = std::chrono::steady_clock::now();
+
+ std::unique_lock lock(messageCacheMutex);
+ auto it = messageCache.find(message);
+ if (it != messageCache.end() &&
+ (now - it->second.timestamp) < gMessageSuppressionPeriodSeconds) {
+ // Suppress the message
+ return;
+ }
+
+ // Update the cache with the current message and timestamp
+ messageCache[message] = {message, now};
+ cv.notify_one();
+}
+
+inline void addPathByInodeBasedNotificationMessage(const std::string &message,
+ uint32_t inode) {
+ if (stopThread || !gShowMessagesOnNotificationArea) { return; }
+ auto now = std::chrono::steady_clock::now();
+
+ std::string fullPath;
+ std::unique_lock lock(messageCacheMutex);
+ auto it = fullPathFromInodeCache.find(inode);
+ if (it != fullPathFromInodeCache.end() &&
+ (now - it->second.timestamp) < gMessageSuppressionPeriodSeconds) {
+ fullPath = it->second.message;
+ } else {
+ fs_fullpath(inode, 0, 0, fullPath);
+ fullPathFromInodeCache[inode] = {fullPath, now};
+ }
+ lock.unlock();
+
+ addNotificationMessage(message + ": " + fullPath);
+}
+
+inline void StartNotificationThread() {
+ stopThread = false;
+ notificationThread = std::jthread(NotificationThread);
+}
+
+inline void StopNotificationThread() {
+ {
+ std::unique_lock lock(messageCacheMutex);
+ stopThread = true;
+ cv.notify_all();
+ }
+ if (notificationThread.joinable()) { notificationThread.join(); }
+}
+
+inline void notifications_area_logging_init(
+ bool log_notification_area, unsigned message_suppression_period) {
+ gShowMessagesOnNotificationArea = log_notification_area;
+ gMessageSuppressionPeriodSeconds =
+ std::chrono::seconds(message_suppression_period);
+ if (gShowMessagesOnNotificationArea) { StartNotificationThread(); }
+}
+
+inline void notifications_area_logging_term() {
+ if (gShowMessagesOnNotificationArea) { StopNotificationThread(); }
+}
diff --git a/src/mount/readdata.cc b/src/mount/readdata.cc
index 517258982..c0cce85d7 100644
--- a/src/mount/readdata.cc
+++ b/src/mount/readdata.cc
@@ -46,6 +46,7 @@
#include "mount/chunk_locator.h"
#include "mount/chunk_reader.h"
#include "mount/mastercomm.h"
+#include "mount/notification_area_logging.h"
#include "mount/readahead_adviser.h"
#include "mount/readdata_cache.h"
#include "mount/memory_info.h"
@@ -66,6 +67,7 @@ uint64_t successfulTimesRequestedMemory = 0;
constexpr double kTimesRequestedMemoryLowerSuccessRate = 0.3;
constexpr double kTimesRequestedMemoryUpperSuccessRate = 0.8;
constexpr uint32_t kMinCacheExpirationTime = 1;
+constexpr uint32_t kMinTryCounterToShowReadErrorMessage = 9;
std::unique_ptr createMemoryInfo() {
std::unique_ptr memoryInfo;
@@ -753,6 +755,9 @@ int read_to_buffer(ReadRecord *rrec, uint64_t current_offset,
try_counter = 0;
} catch (UnrecoverableReadException &ex) {
print_error_msg(reader, try_counter, ex);
+ addPathByInodeBasedNotificationMessage(
+ "Unrecoverable read error: " + std::string(ex.what()),
+ rrec->inode);
std::unique_lock usedMemoryLock(gReadCacheMemoryMutex);
decreaseUsedReadCacheMemory(total_read_cache_bytes_to_reserve);
usedMemoryLock.unlock();
@@ -767,11 +772,19 @@ int read_to_buffer(ReadRecord *rrec, uint64_t current_offset,
}
force_prepare = true;
if (try_counter > maxRetries) {
+ addPathByInodeBasedNotificationMessage(
+ "Read error: Exceeded max retries:" +
+ std::string(ex.what()),
+ rrec->inode);
std::unique_lock usedMemoryLock(gReadCacheMemoryMutex);
decreaseUsedReadCacheMemory(total_read_cache_bytes_to_reserve);
usedMemoryLock.unlock();
return SAUNAFS_ERROR_IO;
} else {
+ if (try_counter > kMinTryCounterToShowReadErrorMessage) {
+ addPathByInodeBasedNotificationMessage(
+ "Read error: " + std::string(ex.what()), rrec->inode);
+ }
std::unique_lock usedMemoryLock(gReadCacheMemoryMutex);
decreaseUsedReadCacheMemory(last_read_cache_bytes_to_reserve);
total_read_cache_bytes_to_reserve -= last_read_cache_bytes_to_reserve;
diff --git a/src/mount/sauna_client.cc b/src/mount/sauna_client.cc
index 7094173b8..4c21185a5 100644
--- a/src/mount/sauna_client.cc
+++ b/src/mount/sauna_client.cc
@@ -64,6 +64,7 @@
#include "mount/io_limit_group.h"
#include "mount/mastercomm.h"
#include "mount/masterproxy.h"
+#include "mount/notification_area_logging.h"
#include "mount/oplog.h"
#include "mount/readdata.h"
#include "mount/special_inode.h"
@@ -3540,6 +3541,9 @@ void fs_init(FsInitParams ¶ms) {
set_debug_mode(params.debug_mode);
#endif
+ notifications_area_logging_init(params.log_notifications_area,
+ params.message_suppression_period);
+
init(params.debug_mode, params.keep_cache, params.direntry_cache_timeout, params.direntry_cache_size,
params.entry_cache_timeout, params.attr_cache_timeout, params.mkdir_copy_sgid,
params.sugid_clear_mode, params.use_rw_lock,
@@ -3559,6 +3563,7 @@ void fs_term() {
::fs_term();
symlink_cache_term();
socketrelease();
+ notifications_area_logging_term();
}
} // namespace SaunaClient
diff --git a/src/mount/sauna_client.h b/src/mount/sauna_client.h
index e2751a043..86cbff010 100644
--- a/src/mount/sauna_client.h
+++ b/src/mount/sauna_client.h
@@ -86,6 +86,8 @@ struct FsInitParams {
static constexpr float kDefaultBandwidthOveruse = 1.0;
static constexpr unsigned kDefaultChunkserverWriteTo = 5000;
static constexpr bool kDefaultIgnoreFlush = false;
+ static constexpr int kDefaultLogNotificationArea = 0;
+ static constexpr unsigned kDefaultMessageSuppressionPeriod = 10;
#ifdef _WIN32
static constexpr unsigned kDefaultWriteCacheSize = 50;
static constexpr unsigned kDefaultCleanAcquiredFilesPeriod = 0;
@@ -162,9 +164,13 @@ struct FsInitParams {
enable_status_updater_thread(kDefaultEnableStatusUpdaterThread),
ignore_utimens_update(kDefaultIgnoreUtimensUpdate),
#endif
- ignore_flush(kDefaultIgnoreFlush), verbose(kDefaultVerbose), direct_io(kDirectIO)
- ,limit_glibc_malloc_arenas(kDefaultLimitGlibcMallocArenas)
- {
+ ignore_flush(kDefaultIgnoreFlush),
+ verbose(kDefaultVerbose),
+ direct_io(kDirectIO),
+ limit_glibc_malloc_arenas(kDefaultLimitGlibcMallocArenas),
+ log_notifications_area(kDefaultLogNotificationArea),
+ message_suppression_period(kDefaultMessageSuppressionPeriod) {
+
}
FsInitParams(const std::string &bind_host, const std::string &host, const std::string &port, const std::string &mountpoint)
@@ -201,11 +207,13 @@ struct FsInitParams {
enable_status_updater_thread(kDefaultEnableStatusUpdaterThread),
ignore_utimens_update(kDefaultIgnoreUtimensUpdate),
#endif
- ignore_flush(kDefaultIgnoreFlush), verbose(kDefaultVerbose), direct_io(kDirectIO)
-#ifndef _WIN32
- ,limit_glibc_malloc_arenas(kDefaultLimitGlibcMallocArenas)
-#endif
- {
+ ignore_flush(kDefaultIgnoreFlush),
+ verbose(kDefaultVerbose),
+ direct_io(kDirectIO),
+ limit_glibc_malloc_arenas(kDefaultLimitGlibcMallocArenas),
+ log_notifications_area(kDefaultLogNotificationArea),
+ message_suppression_period(kDefaultMessageSuppressionPeriod) {
+
}
std::string bind_host;
@@ -265,6 +273,8 @@ struct FsInitParams {
bool verbose;
bool direct_io;
unsigned limit_glibc_malloc_arenas;
+ int log_notifications_area;
+ unsigned message_suppression_period;
std::string io_limits_config_file;
};
diff --git a/src/mount/writedata.cc b/src/mount/writedata.cc
index a68e04ad7..13f17e226 100644
--- a/src/mount/writedata.cc
+++ b/src/mount/writedata.cc
@@ -56,6 +56,7 @@
#include "mount/mastercomm.h"
#include "mount/readdata.h"
#include "mount/tweaks.h"
+#include "mount/notification_area_logging.h"
#include "mount/write_cache_block.h"
#include "protocol/cltocs.h"
#include "protocol/SFSCommunication.h"
@@ -391,6 +392,9 @@ class InodeChunkWriter {
static const uint32_t kMaximumTimeWhenJobsWaiting = 10;
// For the last 'kTimeToFinishOperations' seconds of maximumTime we won't start new operations
static const uint32_t kTimeToFinishOperations = 5;
+ // Minimum tries counter value before showing write error message on
+ // notifications area
+ static const uint32_t kMinTryCounterToShowWriteErrorMessage = 4;
};
void InodeChunkWriter::processJob(inodedata* inodeData) {
@@ -468,6 +472,8 @@ void InodeChunkWriter::processJob(inodedata* inodeData) {
write_job_delayed_end(inodeData_, SAUNAFS_STATUS_OK, (canWait ? 1 : 0), lock);
} catch (Exception& e) {
std::string errorString = e.what();
+ addPathByInodeBasedNotificationMessage(
+ "Write error: " + std::string(e.what()), inodeData_->inode);
Glock lock(gMutex);
if (e.status() != SAUNAFS_ERROR_LOCKED) {
inodeData_->trycnt++;
@@ -486,6 +492,7 @@ void InodeChunkWriter::processJob(inodedata* inodeData) {
safs_pretty_syslog(LOG_WARNING, "write file error, inode: %" PRIu32 ", index: %" PRIu32 " - %s",
inodeData_->inode, chunkIndex_, errorString.c_str());
+
if (inodeData_->trycnt >= maxretries) {
// Convert error to an unrecoverable error
throw UnrecoverableWriteException(e.message(), e.status());
@@ -495,6 +502,8 @@ void InodeChunkWriter::processJob(inodedata* inodeData) {
}
}
} catch (UnrecoverableWriteException& e) {
+ addPathByInodeBasedNotificationMessage(
+ "Write error: " + std::string(e.what()), inodeData_->inode);
Glock lock(gMutex);
if (e.status() == SAUNAFS_ERROR_ENOENT) {
write_job_end(inodeData_, SAUNAFS_ERROR_EBADF, lock);
@@ -506,6 +515,10 @@ void InodeChunkWriter::processJob(inodedata* inodeData) {
write_job_end(inodeData_, SAUNAFS_ERROR_IO, lock);
}
} catch (Exception& e) {
+ if (inodeData_->trycnt > kMinTryCounterToShowWriteErrorMessage) {
+ addPathByInodeBasedNotificationMessage(
+ "Write error: " + std::string(e.what()), inodeData_->inode);
+ }
Glock lock(gMutex);
int waitTime = 1;
if (inodeData_->trycnt > 10) {