diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h index 1cf92bde..a41b6330 100644 --- a/src/drivers/net/mlx_ipoib/arbel.h +++ b/src/drivers/net/mlx_ipoib/arbel.h @@ -119,8 +119,20 @@ struct arbel_recv_work_queue { union arbel_recv_wqe *wqe; }; +/** An Arbel queue pair */ +struct arbel_queue_pair { + /** Infiniband queue pair */ + struct ib_queue_pair qp; + /** Send work queue */ + struct arbel_send_work_queue send; + /** Receive work queue */ + struct arbel_recv_work_queue recv; +}; + /** An Arbel completion queue */ struct arbel_completion_queue { + /** Infiniband completion queue */ + struct ib_completion_queue cq; /** Doorbell record number */ unsigned int doorbell_idx; /** Completion queue entries */ @@ -154,6 +166,7 @@ struct arbel { */ #define ARBEL_HCR_QUERY_DEV_LIM 0x0003 +#define ARBEL_HCR_SW2HW_CQ 0x0016 #define ARBEL_HCR_BASE 0x80680 #define ARBEL_HCR_REG(x) ( ARBEL_HCR_BASE + 4 * (x) ) diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index 180ec6da..4fcc6a3c 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -52,57 +52,54 @@ static struct io_buffer *static_ipoib_tx_ring[NUM_IPOIB_SND_WQES]; static struct io_buffer *static_ipoib_rx_ring[NUM_IPOIB_RCV_WQES]; static struct arbel static_arbel; -static struct arbel_send_work_queue static_arbel_ipoib_send_wq = { - .doorbell_idx = IPOIB_SND_QP_DB_IDX, -}; -static struct arbel_send_work_queue static_arbel_ipoib_recv_wq = { - .doorbell_idx = IPOIB_RCV_QP_DB_IDX, -}; -static struct arbel_completion_queue static_arbel_ipoib_send_cq = { - .doorbell_idx = IPOIB_SND_CQ_CI_DB_IDX, -}; -static struct arbel_completion_queue static_arbel_ipoib_recv_cq = { - .doorbell_idx = IPOIB_RCV_CQ_CI_DB_IDX, -}; +static struct arbel_completion_queue static_ipoib_send_cq; +static struct arbel_completion_queue static_ipoib_recv_cq; -static struct ib_completion_queue static_ipoib_send_cq; -static struct ib_completion_queue static_ipoib_recv_cq; -static struct ib_device static_ibdev = { - .dev_priv = &static_arbel, -}; -static struct ib_queue_pair static_ipoib_qp = { +static struct arbel_queue_pair static_ipoib_qp = { + .qp = { + .send = { + .qp = &static_ipoib_qp.qp, + .is_send = 1, + .cq = &static_ipoib_send_cq.cq, + .num_wqes = NUM_IPOIB_SND_WQES, + .iobufs = static_ipoib_tx_ring, + .list = LIST_HEAD_INIT (static_ipoib_qp.qp.send.list), + }, + .recv = { + .qp = &static_ipoib_qp.qp, + .is_send = 0, + .cq = &static_ipoib_recv_cq.cq, + .num_wqes = NUM_IPOIB_RCV_WQES, + .iobufs = static_ipoib_rx_ring, + .list = LIST_HEAD_INIT (static_ipoib_qp.qp.recv.list), + }, + }, .send = { - .qp = &static_ipoib_qp, - .is_send = 1, - .cq = &static_ipoib_send_cq, - .num_wqes = NUM_IPOIB_SND_WQES, - .iobufs = static_ipoib_tx_ring, - .dev_priv = &static_arbel_ipoib_send_wq, - .list = LIST_HEAD_INIT ( static_ipoib_qp.send.list ), + .doorbell_idx = IPOIB_SND_QP_DB_IDX, }, .recv = { - .qp = &static_ipoib_qp, - .is_send = 0, - .cq = &static_ipoib_recv_cq, - .num_wqes = NUM_IPOIB_RCV_WQES, - .iobufs = static_ipoib_rx_ring, - .dev_priv = &static_arbel_ipoib_recv_wq, - .list = LIST_HEAD_INIT ( static_ipoib_qp.recv.list ), + .doorbell_idx = IPOIB_RCV_QP_DB_IDX, }, }; -static struct ib_completion_queue static_ipoib_send_cq = { - .cqn = 1234, /* Only used for debug messages */ - .num_cqes = NUM_IPOIB_SND_CQES, - .dev_priv = &static_arbel_ipoib_send_cq, - .work_queues = LIST_HEAD_INIT ( static_ipoib_send_cq.work_queues ), +static struct arbel_completion_queue static_ipoib_send_cq = { + .cq = { + .cqn = 1234, /* Only used for debug messages */ + .num_cqes = NUM_IPOIB_SND_CQES, + .work_queues = LIST_HEAD_INIT (static_ipoib_send_cq.cq.work_queues), + }, + .doorbell_idx = IPOIB_SND_CQ_CI_DB_IDX, }; -static struct ib_completion_queue static_ipoib_recv_cq = { - .cqn = 2345, /* Only used for debug messages */ - .num_cqes = NUM_IPOIB_RCV_CQES, - .dev_priv = &static_arbel_ipoib_recv_cq, - .work_queues = LIST_HEAD_INIT ( static_ipoib_recv_cq.work_queues ), +static struct arbel_completion_queue static_ipoib_recv_cq = { + .cq = { + .cqn = 2345, /* Only used for debug messages */ + .num_cqes = NUM_IPOIB_RCV_CQES, + .work_queues = LIST_HEAD_INIT (static_ipoib_recv_cq.cq.work_queues), + }, + .doorbell_idx = IPOIB_RCV_CQ_CI_DB_IDX, +}; +static struct ib_device static_ibdev = { + .priv = &static_arbel, }; - /** @@ -152,7 +149,7 @@ static int mlx_transmit_direct ( struct net_device *netdev, }; memcpy ( &av.gid, ( ( void * ) bav ) + 16, 16 ); - rc = arbel_post_send ( &static_ibdev, &static_ipoib_qp, &av, iobuf ); + rc = arbel_post_send ( &static_ibdev, &static_ipoib_qp.qp, &av, iobuf ); return rc; } @@ -206,7 +203,8 @@ static void mlx_refill_rx ( struct net_device *netdev ) { if ( ! iobuf ) break; DBG ( "Posting RX buffer %p:\n", iobuf ); - if ( ( rc = arbel_post_recv ( &static_ibdev, &static_ipoib_qp, + if ( ( rc = arbel_post_recv ( &static_ibdev, + &static_ipoib_qp.qp, iobuf ) ) != 0 ) { free_iob ( iobuf ); break; @@ -238,9 +236,9 @@ static void mlx_poll ( struct net_device *netdev ) { } /* Poll completion queues */ - arbel_poll_cq ( &static_ibdev, &static_ipoib_send_cq, + arbel_poll_cq ( &static_ibdev, &static_ipoib_send_cq.cq, temp_complete_send, temp_complete_recv ); - arbel_poll_cq ( &static_ibdev, &static_ipoib_recv_cq, + arbel_poll_cq ( &static_ibdev, &static_ipoib_recv_cq.cq, temp_complete_send, temp_complete_recv ); // mlx_poll_cq ( netdev, mlx->rcv_cqh, mlx_rx_complete ); @@ -383,27 +381,63 @@ static int arbel_cmd ( struct arbel *arbel, unsigned long command, return 0; } -static int arbel_cmd_query_dev_lim ( struct arbel *arbel, - struct arbelprm_query_dev_lim *out ) { +static inline int +arbel_cmd_query_dev_lim ( struct arbel *arbel, + struct arbelprm_query_dev_lim *dev_lim ) { return arbel_cmd ( arbel, ARBEL_HCR_OUT_CMD ( ARBEL_HCR_QUERY_DEV_LIM, - 1, sizeof ( *out ) ), - 0, NULL, 0, out ); + 1, sizeof ( *dev_lim ) ), + 0, NULL, 0, dev_lim ); } +static inline int +arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn, + const struct arbelprm_completion_queue_context *cqctx ) { + return arbel_cmd ( arbel, + ARBEL_HCR_IN_CMD ( ARBEL_HCR_SW2HW_CQ, + 1, sizeof ( *cqctx ) ), + 0, cqctx, cqn, NULL ); +} + +/*************************************************************************** + * + * Completion queue operations + * + *************************************************************************** + */ + /** * Create completion queue * * @v ibdev Infiniband device * @v */ -static int arbel_create_cq ( struct ib_device *ibdev ) { - struct arbelprm_completion_queue_context *cqctx; +static int arbel_create_cq ( struct ib_device *ibdev, + struct ib_completion_queue **new_cq ) { + struct arbel *arbel = ibdev->priv; + struct arbelprm_completion_queue_context cqctx; + struct ib_completion_queue *cq; + cq = zalloc ( sizeof ( *cq ) ); + if ( ! cq ) + return -ENOMEM; + + + memset ( &cqctx, 0, sizeof ( cqctx ) ); + + + return arbel_cmd_sw2hw_cq ( arbel, 0, &cqctx ); } +/*************************************************************************** + * + * Work request operations + * + *************************************************************************** + */ + /** * Ring doorbell register in UAR * @@ -438,9 +472,11 @@ static int arbel_post_send ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *av, struct io_buffer *iobuf ) { - struct arbel *arbel = ibdev->dev_priv; + struct arbel *arbel = ibdev->priv; + struct arbel_queue_pair *arbel_qp + = container_of ( qp, struct arbel_queue_pair, qp ); struct ib_work_queue *wq = &qp->send; - struct arbel_send_work_queue *arbel_send_wq = wq->dev_priv; + struct arbel_send_work_queue *arbel_send_wq = &arbel_qp->send; struct arbelprm_ud_send_wqe *prev_wqe; struct arbelprm_ud_send_wqe *wqe; union arbelprm_doorbell_record *db_rec; @@ -526,9 +562,11 @@ static int arbel_post_send ( struct ib_device *ibdev, static int arbel_post_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf ) { - struct arbel *arbel = ibdev->dev_priv; + struct arbel *arbel = ibdev->priv; + struct arbel_queue_pair *arbel_qp + = container_of ( qp, struct arbel_queue_pair, qp ); struct ib_work_queue *wq = &qp->recv; - struct arbel_recv_work_queue *arbel_recv_wq = wq->dev_priv; + struct arbel_recv_work_queue *arbel_recv_wq = &arbel_qp->recv; struct arbelprm_recv_wqe *wqe; union arbelprm_doorbell_record *db_rec; unsigned int wqe_idx_mask; @@ -575,12 +613,14 @@ static int arbel_complete ( struct ib_device *ibdev, union arbelprm_completion_entry *cqe, ib_completer_t complete_send, ib_completer_t complete_recv ) { - struct arbel *arbel = ibdev->dev_priv; + struct arbel *arbel = ibdev->priv; struct ib_completion completion; struct ib_work_queue *wq; - struct io_buffer *iobuf; + struct ib_queue_pair *qp; + struct arbel_queue_pair *arbel_qp; struct arbel_send_work_queue *arbel_send_wq; struct arbel_recv_work_queue *arbel_recv_wq; + struct io_buffer *iobuf; ib_completer_t complete; unsigned int opcode; unsigned long qpn; @@ -614,14 +654,16 @@ static int arbel_complete ( struct ib_device *ibdev, arbel, cq->cqn, ( is_send ? "send" : "recv" ), qpn ); return -EIO; } + qp = wq->qp; + arbel_qp = container_of ( qp, struct arbel_queue_pair, qp ); /* Identify work queue entry index */ if ( is_send ) { - arbel_send_wq = wq->dev_priv; + arbel_send_wq = &arbel_qp->send; wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_send_wq->wqe ) ) / sizeof ( arbel_send_wq->wqe[0] ) ); } else { - arbel_recv_wq = wq->dev_priv; + arbel_recv_wq = &arbel_qp->recv; wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_recv_wq->wqe ) ) / sizeof ( arbel_recv_wq->wqe[0] ) ); } @@ -637,7 +679,7 @@ static int arbel_complete ( struct ib_device *ibdev, /* Pass off to caller's completion handler */ complete = ( is_send ? complete_send : complete_recv ); - complete ( ibdev, wq->qp, &completion, iobuf ); + complete ( ibdev, qp, &completion, iobuf ); return rc; } @@ -654,8 +696,9 @@ static void arbel_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq, ib_completer_t complete_send, ib_completer_t complete_recv ) { - struct arbel *arbel = ibdev->dev_priv; - struct arbel_completion_queue *arbel_cq = cq->dev_priv; + struct arbel *arbel = ibdev->priv; + struct arbel_completion_queue *arbel_cq + = container_of ( cq, struct arbel_completion_queue, cq ); union arbelprm_doorbell_record *db_rec; union arbelprm_completion_entry *cqe; unsigned int cqe_idx_mask; @@ -757,20 +800,20 @@ static int arbel_probe ( struct pci_device *pci, static_arbel.uar = memfree_pci_dev.uar; static_arbel.db_rec = dev_ib_data.uar_context_base; static_arbel.reserved_lkey = dev_ib_data.mkey; - static_arbel_ipoib_send_wq.wqe = + static_ipoib_qp.send.wqe = ( ( struct udqp_st * ) qph )->snd_wq; - static_arbel_ipoib_recv_wq.wqe = + static_ipoib_qp.recv.wqe = ( ( struct udqp_st * ) qph )->rcv_wq; - static_arbel_ipoib_send_cq.cqe = + static_ipoib_send_cq.cqe = ( ( struct cq_st * ) ib_data.ipoib_snd_cq )->cq_buf; - static_arbel_ipoib_recv_cq.cqe = + static_ipoib_recv_cq.cqe = ( ( struct cq_st * ) ib_data.ipoib_rcv_cq )->cq_buf; - static_ipoib_qp.qpn = ib_get_qpn ( qph ); - static_ipoib_qp.priv = netdev; - list_add ( &static_ipoib_qp.send.list, - &static_ipoib_send_cq.work_queues ); - list_add ( &static_ipoib_qp.recv.list, - &static_ipoib_recv_cq.work_queues ); + static_ipoib_qp.qp.qpn = ib_get_qpn ( qph ); + static_ipoib_qp.qp.priv = netdev; + list_add ( &static_ipoib_qp.qp.send.list, + &static_ipoib_send_cq.cq.work_queues ); + list_add ( &static_ipoib_qp.qp.recv.list, + &static_ipoib_recv_cq.cq.work_queues ); struct arbelprm_query_dev_lim dev_lim; memset ( &dev_lim, 0xaa, sizeof ( dev_lim ) ); diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index 85684b63..dd8022fb 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -89,8 +89,6 @@ struct ib_work_queue { unsigned long next_idx; /** I/O buffers assigned to work queue */ struct io_buffer **iobufs; - /** Device private data */ - void *dev_priv; }; /** An Infiniband Queue Pair */ @@ -103,8 +101,6 @@ struct ib_queue_pair { struct ib_work_queue recv; /** Queue owner private data */ void *priv; - /** Device private data */ - void *dev_priv; }; /** An Infiniband Completion Queue */ @@ -123,8 +119,6 @@ struct ib_completion_queue { unsigned long next_idx; /** List of work queues completing to this queue */ struct list_head work_queues; - /** Device private data */ - void *dev_priv; }; /** An Infiniband completion */ @@ -224,8 +218,8 @@ struct ib_device_operations { /** An Infiniband device */ struct ib_device { - /** Device private data */ - void *dev_priv; + /** Driver private data */ + void *priv; };