From c52e280d5447c8134fb7588e562b213c9637132d Mon Sep 17 00:00:00 2001
From: Chris Taylor <ctaylor@tactcomplabs.com>
Date: Mon, 5 Feb 2024 17:43:17 +0000
Subject: [PATCH] prov/opx: Add RISC-V support

Signed-off-by: Chris Taylor <ctaylor@tactcomplabs.com>
---
 prov/opx/configure.m4                         |  2 +-
 prov/opx/include/rdma/opx/fi_opx_compiler.h   | 58 ++++++++++++++++++-
 .../opx/include/rdma/opx/fi_opx_hfi1_packet.h |  7 ++-
 prov/opx/include/rdma/opx/fi_opx_timer.h      |  8 +++
 4 files changed, 70 insertions(+), 5 deletions(-)

diff --git a/prov/opx/configure.m4 b/prov/opx/configure.m4
index e5c34a59a09..4fa5ebd43f4 100644
--- a/prov/opx/configure.m4
+++ b/prov/opx/configure.m4
@@ -49,7 +49,7 @@ AC_DEFUN([FI_OPX_CONFIGURE],[
 	dnl and is not supported for non-x86 processors.
 	AS_IF([test "x$macos" = "x1"],[opx_happy=0],
 		[test "x$freebsd" = "x1"],[opx_happy=0],
-		[test x$host_cpu != xx86_64],[opx_happy=0],
+		[test x$host_cpu != xx86_64 && test x$host_cpu != xriscv && test x$host_cpu != xriscv64],[opx_happy=0],
 		[test x"$enable_opx" != x"no"],[
 
 		AC_MSG_CHECKING([for opx provider])
diff --git a/prov/opx/include/rdma/opx/fi_opx_compiler.h b/prov/opx/include/rdma/opx/fi_opx_compiler.h
index 73c105f6d7f..7fadf792768 100644
--- a/prov/opx/include/rdma/opx/fi_opx_compiler.h
+++ b/prov/opx/include/rdma/opx/fi_opx_compiler.h
@@ -59,14 +59,33 @@
 
 static inline void fi_opx_compiler_msync_writes()
 {
-	asm volatile ("sfence" :  :  : "memory");
+#if defined(__riscv)
+
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+ 	asm volatile ("fence ow,ow" :  :  : "memory");
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
+ 	asm volatile ("sfence" :  :  : "memory");
+#endif
 }
 
 static inline void fi_opx_compiler_msync_reads()
 {
-	asm volatile ("lfence" :  :  : "memory");
-}
+#if defined(__riscv)
 
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+	asm volatile ("fence ir,ir" :  :  : "memory");
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
+ 	asm volatile ("lfence" :  :  : "memory");
+#endif
+}
 
 #define fi_opx_compiler_barrier()		__asm__ __volatile__ ( "" ::: "memory" )
 
@@ -87,9 +106,19 @@ void fi_opx_compiler_store_u64(volatile uint64_t * const variable, const uint64_
 static inline
 void fi_opx_compiler_inc_u64(volatile uint64_t * const variable)
 {
+#if defined(__riscv)
+
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+        (*variable) += 1;
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
 	__asm__ __volatile__ ("lock ; incq %0"
 				: "=m" (*variable)
 				: "m" (*variable));
+#endif
 	return;
 }
 
@@ -97,18 +126,41 @@ static inline
 uint64_t fi_opx_compiler_fetch_and_inc_u64(volatile uint64_t * const variable)
 {
 	uint64_t value = 1;
+#if defined(__riscv)
+
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+	const uint64_t tmp = (*variable);
+	(*variable) = value;
+	value = tmp;
+	(*variable) = (*variable) + value;
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
 	__asm__ __volatile__ ("lock ; xadd %0,%1"
 				: "=r" (value), "=m" (*variable)
 				: "0" (value), "m" (*variable));
+#endif
 	return value;
 }
 
 static inline
 void fi_opx_compiler_dec_u64(volatile uint64_t * const variable)
 {
+#if defined(__riscv)
+
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+	(*variable) -= 1;
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
 	__asm__ __volatile__ ("lock ; decq %0"
 				: "=m" (*variable)
 				: "m" (*variable));
+#endif
 	return;
 }
 
diff --git a/prov/opx/include/rdma/opx/fi_opx_hfi1_packet.h b/prov/opx/include/rdma/opx/fi_opx_hfi1_packet.h
index 4951172f799..43d672c53a7 100644
--- a/prov/opx/include/rdma/opx/fi_opx_hfi1_packet.h
+++ b/prov/opx/include/rdma/opx/fi_opx_hfi1_packet.h
@@ -43,7 +43,12 @@
 #include "rdma/fabric.h"	/* only for 'fi_addr_t' ... which is a typedef to uint64_t */
 #include "rdma/opx/fi_opx_addr.h"
 
-
+#if defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64)
+#ifndef PAGE_SIZE
+/* 4K pages default */
+#define PAGE_SIZE 4096
+#endif
+#endif
 
 #define FI_OPX_ADDR_SEP_RX_MAX			(4)
 #define FI_OPX_HFI1_PACKET_MTU			(8192)
diff --git a/prov/opx/include/rdma/opx/fi_opx_timer.h b/prov/opx/include/rdma/opx/fi_opx_timer.h
index a1c9876c1b8..6738c9d2d10 100644
--- a/prov/opx/include/rdma/opx/fi_opx_timer.h
+++ b/prov/opx/include/rdma/opx/fi_opx_timer.h
@@ -62,6 +62,14 @@ static inline uint64_t fi_opx_timer_get_cycles()
 	cycles = ((uint64_t)a) | (((uint64_t)d) << 32);
 	return cycles;
 }
+#elif defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64)
+__attribute__((always_inline))
+static inline uint64_t fi_opx_timer_get_cycles()
+{
+        uint64_t dst = 0;
+        asm volatile ("rdcycle %0" : "=r" (dst) );
+        return dst;
+}
 #else
 #error "Cycle timer not defined for this platform"
 #endif