libzrdma: Add support for query interface and some other optimization

Thia patch add the following verbs:
query_device_ex
query_port
attach_mcast
detach_mcast
This patch also add the following function:
add support for new vendor id and device id
add support for private verbs interface
use ZXDH_DEBUG_MASK to control debug level log
use ZXDH_WRITE_IMM_SPILIT_RNABLE to fix write with immediate bug

on-behalf-of: @ZTE [email protected]

Signed-off-by: zte_lifuyan <[email protected]>

Author: ztefuyan

debian/libibverbs-dev.install

@@ -13,6 +13,8 @@ usr/include/infiniband/sa.h
 usr/include/infiniband/tm_types.h
 usr/include/infiniband/verbs.h
 usr/include/infiniband/verbs_api.h
+usr/include/infiniband/zxdh_devids.h
+usr/include/infiniband/zxdh_dv.h
 usr/lib/*/lib*-rdmav*.a
 usr/lib/*/libefa.a
 usr/lib/*/libefa.so
@@ -26,12 +28,14 @@ usr/lib/*/libmlx4.a
 usr/lib/*/libmlx4.so
 usr/lib/*/libmlx5.a
 usr/lib/*/libmlx5.so
+usr/lib/*/libzrdma.so
 usr/lib/*/pkgconfig/libefa.pc
 usr/lib/*/pkgconfig/libhns.pc
 usr/lib/*/pkgconfig/libibverbs.pc
 usr/lib/*/pkgconfig/libmana.pc
 usr/lib/*/pkgconfig/libmlx4.pc
 usr/lib/*/pkgconfig/libmlx5.pc
+usr/lib/*//pkgconfig/libzrdma.pc
 usr/share/man/man3/efadv_*.3
 usr/share/man/man3/hnsdv_*.3
 usr/share/man/man3/ibv_*

debian/libibverbs1.install

@@ -1,2 +1,4 @@
 usr/lib/*/libibverbs*.so.*
+usr/lib/*/libzrdma.so.*
+usr/lib/*/libzrdma.a
 usr/share/doc/rdma-core/libibverbs.md usr/share/doc/libibverbs1/

kernel-headers/CMakeLists.txt

@@ -27,6 +27,7 @@ publish_internal_headers(rdma
   rdma/siw-abi.h
   rdma/vmw_pvrdma-abi.h
   rdma/zxdh-abi.h
+  rdma/zxdh_user_ioctl_cmds.h
   rdma/zxdh_user_ioctl_verbs.h
   )
 

providers/zrdma/CMakeLists.txt

@@ -25,3 +25,11 @@ if(TARGET zrdma-rdmav${IBVERBS_PABI_VERSION})
   rdma_pkg_config("zrdma-rdmav${IBVERBS_PABI_VERSION}" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
 endif()
 
+install(CODE "
+  execute_process(
+    COMMAND \${CMAKE_COMMAND} -E create_symlink
+      libzrdma-rdmav${IBVERBS_PABI_VERSION}.so
+      \$ENV{DESTDIR}\${CMAKE_INSTALL_LIBDIR}/libzrdma.so
+  )
+")
+

providers/zrdma/libzrdma.map

@@ -1,5 +1,16 @@
 /* Export symbols should be added below according to
    Documentation/versioning.md document. */
 ZRDMA_1.0 {
+	global:
+		zxdh_get_log_trace_switch;
 	local: *;
 };
+
+ZRDMA_1.1 {
+	global:
+		zxdh_set_log_trace_switch;
+		zxdh_modify_qp_udp_sport;
+		zxdh_query_qpc;
+		zxdh_modify_qpc;
+		zxdh_reset_qp;
+} ZRDMA_1.0;

providers/zrdma/private_verbs_cmd.c

@@ -69,8 +69,92 @@ static int _zxdh_query_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc)
 	return 0;
 }
 
+static void copy_modify_qpc_fields(struct zxdh_modify_qpc_req *req_cmd,
+				   uint64_t attr_mask,
+				   struct zxdh_rdma_qpc *qpc)
+{
+	if (attr_mask & ZXDH_TX_READ_RETRY_FLAG_SET) {
+		req_cmd->retry_flag = qpc->retry_flag;
+		req_cmd->rnr_retry_flag = qpc->rnr_retry_flag;
+		req_cmd->read_retry_flag = qpc->read_retry_flag;
+		req_cmd->cur_retry_count = qpc->cur_retry_count;
+	}
+	if (attr_mask & ZXDH_RETRY_CQE_SQ_OPCODE)
+		req_cmd->retry_cqe_sq_opcode = qpc->retry_cqe_sq_opcode;
+
+	if (attr_mask & ZXDH_ERR_FLAG_SET) {
+		req_cmd->err_flag = qpc->err_flag;
+		req_cmd->ack_err_flag = qpc->ack_err_flag;
+	}
+	if (attr_mask & ZXDH_PACKAGE_ERR_FLAG)
+		req_cmd->package_err_flag = qpc->package_err_flag;
+}
+
+static int _zxdh_reset_qp(struct ibv_qp *qp, uint64_t opcode)
+{
+	DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ,
+			       ZXDH_IB_METHOD_QP_RESET_QP, 2);
+	fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_RESET_QP_HANDLE, qp->handle);
+	fill_attr_in_uint64(cmd, ZXDH_IB_ATTR_QP_RESET_OP_CODE, opcode);
+	return execute_ioctl(qp->context, cmd);
+}
+
+static int _zxdh_modify_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc,
+			    uint64_t qpc_mask)
+{
+	DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ,
+			       ZXDH_IB_METHOD_QP_MODIFY_QPC, 3);
+	struct zxdh_modify_qpc_req req = { 0 };
+
+	copy_modify_qpc_fields(&req, qpc_mask, qpc);
+	fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_QUERY_HANDLE, qp->handle);
+	fill_attr_in_uint64(cmd, ZXDH_IB_ATTR_QP_MODIFY_QPC_MASK, qpc_mask);
+	fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_QP_MODIFY_QPC_REQ, &req);
+	return execute_ioctl(qp->context, cmd);
+}
+
+static int _zxdh_modify_qp_udp_sport(struct ibv_context *ibctx,
+				     uint16_t udp_sport, uint32_t qpn)
+{
+	if (udp_sport <= MIN_UDP_SPORT || qpn <= MIN_QP_QPN)
+		return -EINVAL;
+
+	DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ,
+			       ZXDH_IB_METHOD_QP_MODIFY_UDP_SPORT, 2);
+	fill_attr_in(cmd, ZXDH_IB_ATTR_QP_UDP_PORT, &udp_sport,
+		     sizeof(udp_sport));
+	fill_attr_in_uint32(cmd, ZXDH_IB_ATTR_QP_QPN, qpn);
+	return execute_ioctl(ibctx, cmd);
+}
+
+static int _zxdh_get_log_trace_switch(struct ibv_context *ibctx,
+				      uint8_t *switch_status)
+{
+	DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV,
+			       ZXDH_IB_METHOD_DEV_GET_LOG_TRACE, 1);
+
+	fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_LOG_TARCE_SWITCH,
+			  switch_status);
+	return execute_ioctl(ibctx, cmd);
+}
+
+static int _zxdh_set_log_trace_switch(struct ibv_context *ibctx,
+				      uint8_t switch_status)
+{
+	DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV,
+			       ZXDH_IB_METHOD_DEV_SET_LOG_TRACE, 1);
+	fill_attr_in(cmd, ZXDH_IB_ATTR_DEV_SET_LOG_TARCE_SWITCH, &switch_status,
+		     sizeof(switch_status));
+	return execute_ioctl(ibctx, cmd);
+}
+
 static struct zxdh_uvcontext_ops zxdh_ctx_ops = {
+	.modify_qp_udp_sport = _zxdh_modify_qp_udp_sport,
+	.get_log_trace_switch = _zxdh_get_log_trace_switch,
+	.set_log_trace_switch = _zxdh_set_log_trace_switch,
 	.query_qpc = _zxdh_query_qpc,
+	.modify_qpc = _zxdh_modify_qpc,
+	.reset_qp = _zxdh_reset_qp,
 };
 
 static inline struct zxdh_uvcontext *to_zxdhtx(struct ibv_context *ibctx)

providers/zrdma/private_verbs_cmd.h

@@ -36,16 +36,6 @@
 #include "zxdh_zrdma.h"
 #include "zxdh_dv.h"
 
-enum zxdh_rdma_tool_flags {
-	ZXDH_QP_EXTEND_OP = 1 << 0,
-	ZXDH_CAPTURE = 1 << 1,
-	ZXDH_GET_HW_DATA = 1 << 2,
-	ZXDH_GET_HW_OBJECT_DATA = 1 << 3,
-	ZXDH_CHECK_HW_HEALTH = 1 << 4,
-	ZXDH_RDMA_TOOL_CFG_DEV_PARAM = 1 << 5,
-	ZXDH_RDMA_TOOL_SHOW_RES_MAP = 1 << 5,
-};
-
 struct zxdh_uvcontext_ops {
 	int (*modify_qp_udp_sport)(struct ibv_context *ibctx,
 				   uint16_t udp_sport, uint32_t qpn);

providers/zrdma/zxdh_devids.h

@@ -33,13 +33,20 @@
 #ifndef __ZXDH_DEVIDS_H__
 #define __ZXDH_DEVIDS_H__
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* ZXDH VENDOR ID */
 #define PCI_VENDOR_ID_ZXDH_EVB 0x16c3
 #define PCI_VENDOR_ID_ZXDH_E312 0x1cf2
+#define PCI_VENDOR_ID_ZXDH_E312S_D 0x1cf2
 #define PCI_VENDOR_ID_ZXDH_E310 0x1cf2
 #define PCI_VENDOR_ID_ZXDH_E310_RDMA 0x1cf2
 #define PCI_VENDOR_ID_ZXDH_E316 0x1cf2
 #define PCI_VENDOR_ID_ZXDH_X512 0x1cf2
+#define PCI_VENDOR_ID_ZXDH_E312_TY_CLOUD 0x1b18
+#define PCI_VENDOR_ID_ZXDH_E310_TY_CLOUD  0x1b18
 /* ZXDH Devices ID */
 #define ZXDH_DEV_ID_ADAPTIVE_EVB_PF 0x8040 /* ZXDH EVB PF DEVICE ID*/
 #define ZXDH_DEV_ID_ADAPTIVE_EVB_VF 0x8041 /* ZXDH EVB VF DEVICE ID*/
@@ -53,4 +60,15 @@
 #define ZXDH_DEV_ID_ADAPTIVE_E316_VF 0x807f /* ZXDH E316 VF DEVICE ID*/
 #define ZXDH_DEV_ID_ADAPTIVE_X512_PF 0x806B /* ZXDH X512 PF DEVICE ID*/
 #define ZXDH_DEV_ID_ADAPTIVE_X512_VF 0x806C /* ZXDH X512 VF DEVICE ID*/
-#endif /* __ZXDH_DEVIDS_H__ */
+#define ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_PF 0x1110 /* ZXDH E312 TY CLOUD PF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_VF 0x1111 /* ZXDH E312 TY CLOUD VF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_PF 0x1100 /* ZXDH E310 TY CLOUD PF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_VF 0x1101 /* ZXDH E310 TY CLOUD VF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E312S_D_PF 0x80a2 /* ZXDH E310 TY CLOUD PF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E312S_D_VF 0x80a3 /* ZXDH E310 TY CLOUD VF DEVICE ID*/
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ZXDH_DEVIDS_H */