Feature/nfs integration

driver-fs-state/setup.sh

@@ -38,6 +38,7 @@ PML_SRC="./mcfs-main.pml"
 PML_TEMP="./.pml_tmp"
 PML_START_PATN="\/\* The persistent content of the file systems \*\/"
 PML_END_PATN="\/\* Abstract state signatures of the file systems \*\/"
+NFS_SUFFIX="server";
 
 exclude_files=()
 # Create file system and device key-value map
@@ -391,7 +392,17 @@ for i in $(seq 0 $(($n_fs-1))); do
         if [ -d /mnt/test-$fs-i$i-s$SWARM_ID ]; then
             runcmd rm -rf /mnt/test-$fs-i$i-s$SWARM_ID;
         fi
-        runcmd mkdir -p /mnt/test-$fs-i$i-s$SWARM_ID;
+
+        if [ -d /mnt/test-$fs-i$i-s$SWARM_ID-$NFS_SUFFIX ]; then
+            runcmd rm -rf /mnt/test-$fs-i$i-s$SWARM_ID-$NFS_SUFFIX;
+        fi
+
+        if [ "$fs" = 'nfs' ]; then
+            runcmd mkdir -p /mnt/test-$fs-i$i-s$SWARM_ID-$NFS-SUFFIX;
+        fi
+
+
+
     fi
 done
 

fs-state/fileutil.c

@@ -755,7 +755,8 @@ static bool check_equal_eligible(void)
                 strcmp(get_fslist()[i], XFS_NAME) == 0   || 
                 strcmp(get_fslist()[i], VERIFS1_NAME) == 0 || 
                 strcmp(get_fslist()[i], NILFS2_NAME) == 0 ||
-                strcmp(get_fslist()[i], NOVA_NAME) == 0 ) {
+                strcmp(get_fslist()[i], NOVA_NAME) == 0 ||
+                strcmp(get_fslist()[i], NFS_NAME) == 0) {
             return false;
         }
     }

fs-state/init_globals.h

@@ -51,12 +51,12 @@ typedef struct all_dev_nums {
 static const char *fs_all[] = {"btrfs", "ext2", "ext4", "f2fs", 
                                "jffs2", "ramfs", "tmpfs", "verifs1", 
                                "verifs2", "xfs", "nilfs2", "jfs",
-                               "nova"};
+                               "nova", "nfs"};
 
 static const char *dev_all[]= {"ram", "ram", "ram", "ram", 
                                 "mtdblock", "", "", "", 
                                 "", "ram", "ram", "ram",
-                                "pmem"};
+                                "pmem", "ram"};
 #define ALL_FS nelem(fs_all)
 
 static inline int get_dev_from_fs(char *fs_type) {
@@ -81,6 +81,7 @@ typedef struct all_global_params {
     int _swarm_id;
     unsigned int _n_fs;
     char **fslist;
+
     char **fssuffix;
     char **devlist;
     size_t *devsize_kb;

fs-state/mount.c

@@ -77,6 +77,22 @@ void mountall()
             char cmdbuf[PATH_MAX];
             snprintf(cmdbuf, PATH_MAX, "mount -t NOVA -o noatime %s %s", get_devlist()[i], get_basepaths()[i]);
             ret = execute_cmd_status(cmdbuf);                       
+        } else if(is_nfs(get_fslist()[i])) {
+            char cmdbuf[PATH_MAX]; 
+            char serverbasepath[strlen(get_basepaths()[i]) + NFS_SERVER_SUFFIX_LEN + 1];
+            char clientmountpath[strlen(NFS_EXPORT_IP) + strlen(get_basepaths()[i]) + NFS_SERVER_SUFFIX_LEN + 1];
+
+            snprintf(serverbasepath, sizeof(serverbasepath), "%s-%s",get_basepaths()[i], NFS_SERVER_SUFFIX);
+            snprintf(clientmountpath,  sizeof(clientmountpath), "%s:%s",NFS_EXPORT_IP, serverbasepath);
+
+            ret = mount(get_devlist()[i], serverbasepath, "ext4", MS_NOATIME, "");
+            snprintf(cmdbuf, PATH_MAX,
+             "exportfs -o rw,sync,no_root_squash %s", clientmountpath);
+             ret = execute_cmd_status(cmdbuf);
+
+            //Compared to a mount shell command, we also need to specify addr again in mount options for the C system call (https://stackoverflow.com/questions/28350912/nfs-mount-system-call-in-linux)
+            ret = mount(clientmountpath, get_basepaths()[i], get_fslist()[i], MS_NOATIME, "rw,sync,vers=3,addr=127.0.0.1");
+            //TODO: gaahuja nfsv3 vs v4 handling, error handling                     
         } else {
             ret = mount(get_devlist()[i], get_basepaths()[i], get_fslist()[i], MS_NOATIME, "");
         }        
@@ -119,6 +135,8 @@ void unmount_all(bool strict)
 {
     bool has_failure = false;
     int ret;
+    int retNFS = -1;
+    int retServerFS = -1;
 #ifndef NO_FS_STAT
     record_fs_stat();
 #endif
@@ -138,11 +156,35 @@ void unmount_all(bool strict)
         */
 
         while (retry_limit > 0) {
-            ret = umount2(get_basepaths()[i], 0);
+            if(is_nfs(get_fslist()[i])) {
+            char cmdbuf[PATH_MAX]; 
+
+            char serverbasepath[strlen(get_basepaths()[i]) + NFS_SERVER_SUFFIX_LEN + 1];
+            char clientmountpath[strlen(NFS_EXPORT_IP) + strlen(get_basepaths()[i]) + NFS_SERVER_SUFFIX_LEN + 1];
+
+            snprintf(serverbasepath, sizeof(serverbasepath), "%s-%s",get_basepaths()[i], NFS_SERVER_SUFFIX);
+            snprintf(clientmountpath,  sizeof(clientmountpath), "%s:%s",NFS_EXPORT_IP, serverbasepath);
+
+            snprintf(cmdbuf, PATH_MAX, "umount %s", clientmountpath);
+             if(retNFS != 0) {
+                retNFS = execute_cmd_status(cmdbuf);
+                //fprintf(stderr, "Unmounted nfs at %s, retNFS=%d err=%s \n", clientmountpath, retNFS, errnoname(errno));
+                snprintf(cmdbuf, PATH_MAX,
+                    "exportfs -u %s", clientmountpath);
+                execute_cmd_status(cmdbuf);
+            }
+            if(retNFS == 0) {
+             retServerFS = umount2(serverbasepath,0);
+            }
+            ret = retNFS | retServerFS;
+            //fprintf(stderr, "retNFS=%d retServer=%d errNo=%d errName=%s\n", retNFS, retServerFS, errno, errnoname(errno));
+            }
+            else {
+                ret = umount2(get_basepaths()[i], 0);
+            }
             if (ret == 0) {
                 break; // Success, exit the retry loop
             }        
-
             /* If unmounting failed due to device being busy, again up to
             * retry_limit times with 100 * 2^n ms (n = num_retries) */
             if (errno == EBUSY) {

fs-state/setup.c

@@ -413,6 +413,33 @@ static int setup_nova(const char *devname, const char *basepath, const size_t si
     ret = umount2(basepath, 0);    
     return ret;
 }
+
+static int setup_nfs(const char *serverfsname, const char *devname, const char *basepath, const size_t size_kb)
+{
+    //TODO: gaahuja add support for other underlying fs than ext4/ext2, other networking setup? change nfs-ram mapping in init_globals.c to be extensible
+    int ret;
+    char cmdbuf[PATH_MAX]; 
+    FILE *exports; 
+
+    fprintf(stderr, "Will setup %s with underlying fs= %s\n",
+        __FUNCTION__, serverfsname);
+
+    ret =  setup_generic(serverfsname, devname, size_kb);
+
+    snprintf(cmdbuf, PATH_MAX,
+            "systemctl restart nfs-kernel-server");
+
+    ret = execute_cmd_status(cmdbuf);
+    if( ret!=0 ) {
+        fprintf(stderr, "Cannot restart nfs server (ret=%d \n",
+                ret);
+        exit(1);
+    }
+
+    return ret;
+
+}
+
 void setup_filesystems()
 {
     int ret;
@@ -443,10 +470,14 @@ void setup_filesystems()
         {
             ret = setup_nilfs2(get_devlist()[i], get_devsize_kb()[i]);
         }
-         else if (strcmp(get_fslist()[i], "nova") == 0)
+        else if (strcmp(get_fslist()[i], "nova") == 0)
         {
             ret = setup_nova(get_devlist()[i], get_basepaths()[i], get_devsize_kb()[i]);
         }
+        else if (strcmp(get_fslist()[i], "nfs") == 0)
+        {
+            ret = setup_nfs("ext4", get_devlist()[i], get_basepaths()[i], get_devsize_kb()[i]);
+        }
         // TODO: we need to consider VeriFS1 and VeriFS2 separately here
         else if (is_verifs(get_fslist()[i]))
         {

fs-state/setup.h

@@ -48,12 +48,17 @@
 
 #define VERIFS_PREFIX       "veri"
 #define NOVA_NAME           "nova"
+#define NFS_NAME           "nfs"
 #define BTRFS_NAME          "btrfs"
 #define XFS_NAME            "xfs"
 #define VERIFS1_NAME        "verifs1"
 #define NILFS2_NAME         "nilfs2"
 #define VERIFS_PREFIX_LEN   (sizeof(VERIFS_PREFIX) - 1)
 
+#define NFS_EXPORT_IP "127.0.0.1"
+#define NFS_SERVER_SUFFIX "server"
+#define NFS_SERVER_SUFFIX_LEN (sizeof(NFS_SERVER_SUFFIX)-1)
+
 static inline bool is_verifs(const char *fsname)
 {
     return strncmp(fsname, VERIFS_PREFIX, VERIFS_PREFIX_LEN) == 0;
@@ -64,6 +69,12 @@ static inline bool is_nova(const char *fsname)
     return strcmp(fsname, NOVA_NAME) == 0;
 }
 
+static inline bool is_nfs(const char *fsname)
+{
+    return strcmp(fsname, NFS_NAME) == 0;
+}
+
+
 void setup_filesystems();
 int mkdir_p(const char *path, mode_t dir_mode, mode_t file_mode);
 

fs-state/setup.sh

@@ -42,7 +42,7 @@ PML_END_PATN="\/\* Abstract state signatures of the file systems \*\/"
 exclude_files=()
 # Create file system and device key-value map
 declare -A FS_DEV_MAP
-FS_DEV_MAP+=( ["btrfs"]="ram" ["ext2"]="ram" ["ext4"]="ram" ["f2fs"]="ram" )
+FS_DEV_MAP+=( ["btrfs"]="ram" ["ext2"]="ram" ["ext4"]="ram" ["f2fs"]="ram" ["nfs"]="ram")
 FS_DEV_MAP+=( ["jffs2"]="mtdblock" ["ramfs"]="" ["tmpfs"]="" )
 FS_DEV_MAP+=( ["verifs1"]="" ["verifs2"]="" ["xfs"]="ram" ["nilfs2"]="ram" ["jfs"]="ram")
 FS_DEV_MAP+=( ["nova"]="pmem" )
@@ -396,6 +396,24 @@ unset_nova() {
     :
 }
 
+setup_nfs() {
+    #TODO gaahuja : add flexibility for nfs versioning, ip flexibility, choosing underlying server file system, export args to c code?
+    # DEVFILE="$1";
+    #SERVERDEVFILE = "/nfsserver/$DEVFILE" 
+    # NFS_SERVER_FS="ext4"
+    # echo "BEFORE NFS SETUP"
+    # setup_ext $NFS_SERVER_FS $DEVFILE 0;
+    # echo "MODIFYING /cat/exports";
+    #Underlying fs created 
+    :
+}
+
+unset_nfs() {
+    DEVFILE="$1";
+    #Remove entry for nfs server
+     sed -i "/$DEVFILE/d" /etc/exports 2> /dev/null
+}
+
 # Setup mount points and each file system
 for i in $(seq 0 $(($n_fs-1))); do
     # Run individual file system setup scripts defined above
@@ -405,6 +423,7 @@ for i in $(seq 0 $(($n_fs-1))); do
     # Do not need to set up VeriFS
     if [ "${fs:0:${VERI_PREFIX_LEN}}" != "$VERIFS_PREFIX" ]; then
         # Unmount first
+
         if [ "$(mount | grep /mnt/test-$fs-i$i-s$SWARM_ID)" ]; then
             runcmd umount -f /mnt/test-$fs-i$i-s$SWARM_ID;
         fi
@@ -415,6 +434,11 @@ for i in $(seq 0 $(($n_fs-1))); do
             runcmd rm -rf /mnt/test-$fs-i$i-s$SWARM_ID;
         fi
         runcmd mkdir -p /mnt/test-$fs-i$i-s$SWARM_ID;
+        # For NFS, need to create another dir for fs mounted on the server
+
+        if [[ "$fs" == nfs* ]]; then
+            runcmd mkdir -p /mnt/test-$fs-i$i-s$SWARM_ID-server;
+        fi
     fi
 done
 

fs_bugs/nfs/mount_unmount/Makefile

@@ -0,0 +1,22 @@
+CC=gcc
+UTILITY_DIR := ../../test_utilities
+UTILITY_SRC := $(wildcard $(UTILITY_DIR)/*.c)
+UTILITY_OBJ := $(patsubst $(UTILITY_DIR)/%,%.o,$(UTILITY_SRC))
+
+override CFLAGS += -g -I../../test_utilities -I../../../fs-state -I../../../include
+override LIBS += -lssl -lcrypto -lrt -lpthread -lz
+
+driver: driver.c utility-libs
+	$(CC) -o driver driver.c utility-libs.a $(CFLAGS) $(LIBS)
+
+test: driver
+	sudo bash -x run_driver.sh
+
+utility-libs: $(UTILITY_OBJ)
+	ar rvs $@.a $^
+
+%.c.o: $(UTILITY_DIR)/%.c
+	gcc -Wall -Werror -o $@ -fPIC -c $< $(CFLAGS) $(LIBS)
+
+clean:
+	rm -f *.o *.a driver ckpt_tmp.img

fs_bugs/nfs/mount_unmount/driver.c

@@ -0,0 +1,77 @@
+/*
+ * File:   driver.c
+ * Date:   February 9, 2024
+ * Brief:  This file reproduces kernel hang in NOVA fs by repeatedly mounting-unmounting it in a tight loop.
+ * 
+ * Copyright (c) 2020-2024 Gautam Ahuja
+ * Copyright (c) 2020-2024 Yifei Liu
+ * Copyright (c) 2020-2024 Erez Zadok
+ * Copyright (c) 2020-2024 Stony Brook University
+ * Copyright (c) 2020-2024 The Research Foundation of SUNY
+ *
+ * You can redistribute it and/or modify it under the terms of the Apache
+ * License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0).
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mount.h>
+#include <sys/wait.h>
+#include <limits.h>
+#include <unistd.h>
+
+int execute_cmd_status(const char *cmd)
+{
+    if (cmd == NULL) 
+    {
+        fprintf(stderr, "Command should not be NULL.\n");
+        return -1;
+    }
+
+    int retval = system(cmd);
+    int status = WEXITSTATUS(retval);
+    return status;
+}
+
+int main(int argc, char **argv)
+{
+
+    long loop_max = 1;
+    char cmdbuf[PATH_MAX];
+    int ret = -5;
+    long loop_id = 0;
+
+
+
+    while (loop_id < loop_max)
+    {
+
+    ret = mount("/dev/ram1", "/mnt/server", "ext4", MS_NOATIME, NULL);
+    // snprintf(cmdbuf, PATH_MAX, "mount -t ext4 /dev/ram1 /mnt/server");
+    //  ret = execute_cmd_status(cmdbuf);
+    fprintf(stderr, "EXT4 mount, ret=%d\n", ret);
+
+    snprintf(cmdbuf, PATH_MAX, "exportfs -o rw,sync,no_root_squash localhost:/mnt/server");
+    ret = execute_cmd_status(cmdbuf);
+    fprintf(stderr, "Executed command = %s, ret=%d\n", cmdbuf, ret);
+
+    ret = mount("localhost:/mnt/server", "/mnt/local", "nfs", MS_NOATIME, "rw,sync,vers=3,addr=127.0.0.1");
+    fprintf(stderr, "NFS mount, ret=%d\n", ret);
+
+    ret = umount2("/mnt/local", 0);
+     fprintf(stderr, "NFS unmount, ret=%d\n", ret);
+
+
+    snprintf(cmdbuf, PATH_MAX, "exportfs -u localhost:/mnt/server");
+    ret = execute_cmd_status(cmdbuf);
+    fprintf(stderr, "Executed command = %s, ret=%d\n", cmdbuf, ret);
+
+    usleep(5000000);
+    // snprintf(cmdbuf, PATH_MAX, "umount /dev/ram1");
+    // ret = execute_cmd_status(cmdbuf);
+    ret = umount2("dev/ram1", 0);
+    fprintf(stderr, "EXT4 unmount, ret=%d\n", ret);
+    ++loop_id;
+    }
+    return 0;
+}