iw_cxgbe/libcxgb4: Support 64B CQEs

The larger CQE exports the RSS header to userspace.  Note that this
changes the ABI between the driver and library, so the ABI version is
bumped.  A new user->kernel create_cq request message is defined to
disallow a CQE size mismatch between iw_cxgbe and libcxgb4.

Sponsored by:	Chelsio Communications
Co-authored-by:	Steve Wise <[email protected]>

contrib/ofed/libcxgb4/cxgb4-abi.h

@@ -47,6 +47,11 @@ struct c4iw_alloc_pd_resp {
 	uint32_t pdid;
 };
 
+struct c4iw_create_cq_req {
+	struct ibv_create_cq ibv_req;
+	__u64 cqe_size;
+};
+
 struct c4iw_create_cq_resp {
 	struct ibv_create_cq_resp ibv_resp;
 	__u64 key;

contrib/ofed/libcxgb4/libcxgb4.h

@@ -97,7 +97,7 @@ struct c4iw_pd {
 struct c4iw_mr {
 	struct ibv_mr ibv_mr;
 	uint64_t va_fbo;
-	uint32_t len;
+	uint64_t len;
 };
 
 static inline u32 c4iw_mmid(u32 stag)

contrib/ofed/libcxgb4/t4.h

@@ -63,6 +63,7 @@
 #include "t4_regs.h"
 #include "t4_chip_type.h"
 #include "t4fw_interface.h"
+#include "t4_msg.h"
 
 #ifdef DEBUG
 #define DBGLOG(s)
@@ -192,9 +193,10 @@ static inline void init_wr_hdr(union t4_wr *wqe, u16 wrid,
 #define T4_ERR_INTERNAL_ERR                0x1F /* internal error (opcode  */
 						/* mismatch) */
 /*
- * CQE defs
+ * 64B CQE entries.
  */
 struct t4_cqe {
+	struct rss_header rss;
 	__be32 header;
 	__be32 len;
 	union {
@@ -212,7 +214,7 @@ struct t4_cqe {
 			__be32 wrid_low;
 		} gen;
 	} u;
-	__be64 reserved;
+	__be64 reserved[4];
 	__be64 bits_type_ts;
 };
 

contrib/ofed/libcxgb4/verbs.c

@@ -168,20 +168,28 @@ int c4iw_dereg_mr(struct ibv_mr *mr)
 struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe,
 			      struct ibv_comp_channel *channel, int comp_vector)
 {
-	struct ibv_create_cq cmd;
+	struct c4iw_create_cq_req cmd;
 	struct c4iw_create_cq_resp resp;
 	struct c4iw_cq *chp;
 	struct c4iw_dev *dev = to_c4iw_dev(context->device);
 	int ret;
 
+	if (dev->abi_version < 5) {
+		fprintf(stderr, "libcxgb4 FATAL ERROR: downlevel iw_cxgb4 "
+			"module.  Cannot support RDMA with this driver/lib"
+			" combination.  Update your drivers!\n");
+		return NULL;
+	}
+
 	chp = calloc(1, sizeof *chp);
 	if (!chp) {
 		return NULL;
 	}
 
+	cmd.cqe_size = sizeof *chp->cq.queue;
 	resp.reserved = 0;
 	ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
-				&chp->ibv_cq, &cmd, sizeof cmd,
+				&chp->ibv_cq, &cmd.ibv_req, sizeof cmd,
 				&resp.ibv_resp, sizeof resp);
 	if (ret)
 		goto err1;

lib/ofed/libcxgb4/Makefile

@@ -13,6 +13,7 @@ verbs.c
 
 LIBADD=		ibverbs pthread
 CFLAGS+=	-I${_spath}
+CFLAGS+=	-I${SRCTOP}/sys/dev/cxgbe/common
 CFLAGS+=	-I${SRCTOP}/sys/dev/cxgbe/firmware
 
 .include <bsd.lib.mk>

sys/dev/cxgbe/iw_cxgbe/cq.c

@@ -156,11 +156,10 @@ create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
 			F_FW_RI_RES_WR_IQANDST |
 			V_FW_RI_RES_WR_IQANDSTINDEX(sc->sge.ofld_rxq[0].iq.abs_id));
 	res->u.cq.iqdroprss_to_iqesize = cpu_to_be16(
-			F_FW_RI_RES_WR_IQDROPRSS |
 			V_FW_RI_RES_WR_IQPCIECH(2) |
 			V_FW_RI_RES_WR_IQINTCNTTHRESH(0) |
 			F_FW_RI_RES_WR_IQO |
-			V_FW_RI_RES_WR_IQESIZE(1));
+			V_FW_RI_RES_WR_IQESIZE(ilog2(sizeof *cq->queue) - 4));
 	res->u.cq.iqsize = cpu_to_be16(cq->size);
 	res->u.cq.iqaddr = cpu_to_be64(cq->dma_addr);
 
@@ -715,7 +714,7 @@ skip_cqe:
 static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
 {
 	struct c4iw_qp *qhp = NULL;
-	struct t4_cqe cqe = {0, 0}, *rd_cqe;
+	struct t4_cqe uninitialized_var(cqe), *rd_cqe;
 	struct t4_wq *wq;
 	u32 credit = 0;
 	u8 cqe_flushed;

sys/dev/cxgbe/iw_cxgbe/t4.h

@@ -182,7 +182,12 @@ static inline void init_wr_hdr(union t4_wr *wqe, u16 wrid,
 /*
  * CQE defs
  */
+
+/*
+ * 64B CQE entries.
+ */
 struct t4_cqe {
+	struct rss_header rss;
 	__be32 header;
 	__be32 len;
 	union {
@@ -201,7 +206,7 @@ struct t4_cqe {
 		} gen;
 		u64 drain_cookie;
 	} u;
-	__be64 reserved;
+	__be64 reserved[4];
 	__be64 bits_type_ts;
 };
 

sys/dev/cxgbe/iw_cxgbe/user.h

@@ -34,7 +34,7 @@
 #ifndef __C4IW_USER_H__
 #define __C4IW_USER_H__
 
-#define C4IW_UVERBS_ABI_VERSION	3
+#define C4IW_UVERBS_ABI_VERSION	5
 
 /*
  * Make sure that all structs defined in this file remain laid out so
@@ -43,6 +43,10 @@
  * In particular do not use pointer types -- pass pointers in __u64
  * instead.
  */
+struct c4iw_create_cq_req {
+	__u64 cqe_size;
+};
+
 struct c4iw_create_cq_resp {
 	__u64 key;
 	__u64 gts_key;