rpmsg: bring back zero copy transfer

Commit-id: b16ca55
Adding RPMsg Extension layer implementing zero-copy send and receive.

Signed-off-by: Chao An <anchao@pinecone.net>

lib/include/openamp/rpmsg.h

@@ -83,11 +83,22 @@ struct rpmsg_endpoint {
 /**
  * struct rpmsg_device_ops - RPMsg device operations
  * @send_offchannel_raw: send RPMsg data
+ * @hold_rx_buffer: hold RPMsg RX buffer
+ * @release_rx_buffer: release RPMsg RX buffer
+ * @get_tx_payload_buffer: get RPMsg TX buffer
+ * @send_offchannel_nocopy: send RPMsg data without copy
  */
 struct rpmsg_device_ops {
 	int (*send_offchannel_raw)(struct rpmsg_device *rdev,
 				   uint32_t src, uint32_t dst,
 				   const void *data, int len, int wait);
+	void (*hold_rx_buffer)(struct rpmsg_device *rdev, void *rxbuf);
+	void (*release_rx_buffer)(struct rpmsg_device *rdev, void *rxbuf);
+	void *(*get_tx_payload_buffer)(struct rpmsg_device *rdev,
+				       uint32_t *len, int wait);
+	int (*send_offchannel_nocopy)(struct rpmsg_device *rdev,
+				      uint32_t src, uint32_t dst,
+				       const void *data, int len);
 };
 
 /**
@@ -265,6 +276,166 @@ static inline int rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept,
 	return rpmsg_send_offchannel_raw(ept, src, dst, data, len, false);
 }
 
+/**
+ * @brief Holds the rx buffer for usage outside the receive callback.
+ *
+ * Calling this function prevents the RPMsg receive buffer from being released
+ * back to the pool of shmem buffers. This API can only be called at rx
+ * callback context (rpmsg_rx_cb_t). With this API, the application doesn't
+ * need to copy the message in rx callback. Instead, the rx buffer base address
+ * is saved in application context and further processed in application
+ * process. After the message is processed, the application can release the rx
+ * buffer for future reuse in vring by calling the rpmsg_release_rx_buffer()
+ * function.
+ *
+ * @param: ept The rpmsg endpoint
+ * @param: rxbuf RX buffer with message payload
+ *
+ * @see rpmsg_release_rx_buffer
+ */
+void rpmsg_hold_rx_buffer(struct rpmsg_endpoint *ept, void *rxbuf);
+
+/**
+ * @brief Releases the rx buffer for future reuse in vring.
+ *
+ * This API can be called at process context when the message in rx buffer is
+ * processed.
+ *
+ * @ept: the rpmsg endpoint
+ * @rxbuf: rx buffer with message payload
+ *
+ * @see rpmsg_hold_rx_buffer
+ */
+void rpmsg_release_rx_buffer(struct rpmsg_endpoint *ept, void *rxbuf);
+
+/**
+ * @brief Gets the tx buffer for message payload.
+ *
+ * This API can only be called at process context to get the tx buffer in vring.
+ * By this way, the application can directly put its message into the vring tx
+ * buffer without copy from an application buffer.
+ * It is the application responsibility to correctly fill the allocated tx
+ * buffer by data and passing correct parameters to the rpmsg_send_nocopy() or
+ * rpmsg_sendto_nocopy() function to perform data no-copy-send mechanism.
+ *
+ * @ept:  Pointer to rpmsg endpoint
+ * @len:  Pointer to store tx buffer size
+ * @wait: Boolean, wait or not for buffer to become available
+ *
+ * @return The tx buffer address on success and NULL on failure
+ *
+ * @see rpmsg_send_offchannel_nocopy
+ * @see rpmsg_sendto_nocopy
+ * @see rpmsg_send_nocopy
+ */
+void *rpmsg_get_tx_payload_buffer(struct rpmsg_endpoint *ept,
+				  uint32_t *len, int wait);
+
+/**
+ * rpmsg_send_offchannel_nocopy() - send a message in tx buffer reserved by
+ * rpmsg_get_tx_payload_buffer() across to the remote processor.
+ *
+ * This function sends buf of length len to the remote dst address,
+ * and uses src as the source address.
+ * The message will be sent to the remote processor which the ept
+ * endpoint belongs to.
+ * The application has to take the responsibility for:
+ *  1. tx buffer reserved (rpmsg_get_tx_payload_buffer() )
+ *  2. filling the data to be sent into the pre-allocated tx buffer
+ *  3. not exceeding the buffer size when filling the data
+ *  4. data cache coherency
+ *
+ * After the rpmsg_send_offchannel_nocopy() function is issued the tx buffer is
+ * no more owned by the sending task and must not be touched anymore unless the
+ * rpmsg_send_offchannel_nocopy() function fails and returns an error. In that
+ * case application should try to re-issue the rpmsg_send_offchannel_nocopy()
+ * again.
+ *
+ * @ept:  The rpmsg endpoint
+ * @src:  The rpmsg endpoint local address
+ * @dst:  The rpmsg endpoint remote address
+ * @data: TX buffer with message filled
+ * @len:  Length of payload
+ *
+ * @return number of bytes it has sent or negative error value on failure.
+ *
+ * @see rpmsg_get_tx_payload_buffer
+ * @see rpmsg_sendto_nocopy
+ * @see rpmsg_send_nocopy
+ */
+int rpmsg_send_offchannel_nocopy(struct rpmsg_endpoint *ept, uint32_t src,
+				 uint32_t dst, const void *data, int len);
+
+/**
+ * @brief rpmsg_sendto_nocopy() - sends a message in tx buffer allocated by
+ * rpmsg_get_tx_payload_buffer() across to the remote processor, specify dst.
+ *
+ * This function sends buf of length len to the remote dst address.
+ * The message will be sent to the remote processor which the ept
+ * endpoint belongs to, using ept's source address.
+ * The application has to take the responsibility for:
+ *  1. tx buffer allocation (rpmsg_get_tx_payload_buffer() )
+ *  2. filling the data to be sent into the pre-allocated tx buffer
+ *  3. not exceeding the buffer size when filling the data
+ *  4. data cache coherency
+ *
+ * After the rpmsg_sendto_nocopy() function is issued the tx buffer is no more
+ * owned by the sending task and must not be touched anymore unless the
+ * rpmsg_sendto_nocopy() function fails and returns an error. In that case the
+ * application should try to re-issue the rpmsg_sendto_nocopy() again.
+ *
+ * @ept:  The rpmsg endpoint
+ * @data: TX buffer with message filled
+ * @len:  Length of payload
+ * @dst:  Destination address
+ *
+ * @return number of bytes it has sent or negative error value on failure.
+ *
+ * @see rpmsg_get_tx_payload_buffer
+ * @see rpmsg_send_offchannel_nocopy
+ * @see rpmsg_send_nocopy
+ */
+static inline int rpmsg_sendto_nocopy(struct rpmsg_endpoint *ept,
+				      const void *data, int len, uint32_t dst)
+{
+	return rpmsg_send_offchannel_nocopy(ept, ept->addr, dst, data, len);
+}
+
+/**
+ * rpmsg_send_nocopy() - send a message in tx buffer reserved by
+ * rpmsg_get_tx_payload_buffer() across to the remote processor.
+ *
+ * This function sends buf of length len on the ept endpoint.
+ * The message will be sent to the remote processor which the ept
+ * endpoint belongs to, using ept's source and destination addresses.
+ * The application has to take the responsibility for:
+ *  1. tx buffer reserved (rpmsg_get_tx_payload_buffer() )
+ *  2. filling the data to be sent into the pre-allocated tx buffer
+ *  3. not exceeding the buffer size when filling the data
+ *  4. data cache coherency
+ *
+ * After the rpmsg_send_nocopy() function is issued the tx buffer is no more
+ * owned by the sending task and must not be touched anymore unless the
+ * rpmsg_send_nocopy() function fails and returns an error. In that case the
+ * application should try to re-issue the rpmsg_send_nocopy() again.
+ *
+ * @ept:  The rpmsg endpoint
+ * @data: TX buffer with message filled
+ * @len:  Length of payload
+ *
+ * @return number of bytes it has sent or negative error value on failure.
+ *
+ * @see rpmsg_get_tx_payload_buffer
+ * @see rpmsg_send_offchannel_nocopy
+ * @see rpmsg_sendto_nocopy
+ */
+static inline int rpmsg_send_nocopy(struct rpmsg_endpoint *ept,
+				    const void *data, int len)
+{
+	return rpmsg_send_offchannel_nocopy(ept, ept->addr,
+					    ept->dest_addr, data, len);
+}
+
 /**
  * rpmsg_init_ept - initialize rpmsg endpoint
  *

lib/rpmsg/rpmsg.c

@@ -144,6 +144,65 @@ int rpmsg_send_ns_message(struct rpmsg_endpoint *ept, unsigned long flags)
 		return RPMSG_SUCCESS;
 }
 
+void rpmsg_hold_rx_buffer(struct rpmsg_endpoint *ept, void *rxbuf)
+{
+	struct rpmsg_device *rdev;
+
+	if (!ept || !ept->rdev || !rxbuf)
+		return;
+
+	rdev = ept->rdev;
+
+	if (rdev->ops.hold_rx_buffer)
+		rdev->ops.hold_rx_buffer(rdev, rxbuf);
+}
+
+void rpmsg_release_rx_buffer(struct rpmsg_endpoint *ept, void *rxbuf)
+{
+	struct rpmsg_device *rdev;
+
+	if (!ept || !ept->rdev || !rxbuf)
+		return;
+
+	rdev = ept->rdev;
+
+	if (rdev->ops.release_rx_buffer)
+		rdev->ops.release_rx_buffer(rdev, rxbuf);
+}
+
+void *rpmsg_get_tx_payload_buffer(struct rpmsg_endpoint *ept,
+				  uint32_t *len, int wait)
+{
+	struct rpmsg_device *rdev;
+
+	if (!ept || !ept->rdev || !len)
+		return NULL;
+
+	rdev = ept->rdev;
+
+	if (rdev->ops.get_tx_payload_buffer)
+		return rdev->ops.get_tx_payload_buffer(rdev, len, wait);
+
+	return NULL;
+}
+
+int rpmsg_send_offchannel_nocopy(struct rpmsg_endpoint *ept, uint32_t src,
+				 uint32_t dst, const void *data, int len)
+{
+	struct rpmsg_device *rdev;
+
+	if (!ept || !ept->rdev || !data || dst == RPMSG_ADDR_ANY)
+		return RPMSG_ERR_PARAM;
+
+	rdev = ept->rdev;
+
+	if (rdev->ops.send_offchannel_nocopy)
+		return rdev->ops.send_offchannel_nocopy(rdev, src, dst,
+							data, len);
+
+	return RPMSG_ERR_PARAM;
+}
+
 struct rpmsg_endpoint *rpmsg_get_endpoint(struct rpmsg_device *rdev,
 					  const char *name, uint32_t addr,
 					  uint32_t dest_addr)

lib/rpmsg/rpmsg_internal.h

@@ -35,7 +35,12 @@ extern "C" {
 	} while (0)
 #endif
 
+#define RPMSG_BUF_HELD (1U << 31) /* Flag to suggest to hold the buffer */
+
+#define RPMSG_LOCATE_HDR(p) \
+	((struct rpmsg_hdr *)((unsigned char *)(p) - sizeof(struct rpmsg_hdr)))
 #define RPMSG_LOCATE_DATA(p) ((unsigned char *)(p) + sizeof(struct rpmsg_hdr))
+
 /**
  * enum rpmsg_ns_flags - dynamic name service announcement flags
  *