diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c index 03dfa53a..a559d015 100644 --- a/src/drivers/net/ipoib.c +++ b/src/drivers/net/ipoib.c @@ -743,10 +743,7 @@ static void ipoib_poll ( struct net_device *netdev ) { struct ipoib_device *ipoib = netdev->priv; struct ib_device *ibdev = ipoib->ibdev; - ib_poll_cq ( ibdev, ipoib->meta.cq ); - ib_poll_cq ( ibdev, ipoib->data.cq ); - ib_qset_refill_recv ( ibdev, &ipoib->meta ); - ib_qset_refill_recv ( ibdev, &ipoib->data ); + ib_poll_eq ( ibdev ); } /** @@ -861,8 +858,8 @@ static int ipoib_open ( struct net_device *netdev ) { mac->qpn = htonl ( ipoib->data.qp->qpn ); /* Fill receive rings */ - ib_qset_refill_recv ( ibdev, &ipoib->meta ); - ib_qset_refill_recv ( ibdev, &ipoib->data ); + ib_refill_recv ( ibdev, ipoib->meta.qp ); + ib_refill_recv ( ibdev, ipoib->data.qp ); /* Join broadcast group */ if ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) { diff --git a/src/include/gpxe/ib_qset.h b/src/include/gpxe/ib_qset.h index d7a76178..a1116978 100644 --- a/src/include/gpxe/ib_qset.h +++ b/src/include/gpxe/ib_qset.h @@ -18,8 +18,6 @@ struct ib_queue_set { struct ib_completion_queue *cq; /** Queue pair */ struct ib_queue_pair *qp; - /** Receive work queue maximum fill level */ - unsigned int recv_max_fill; }; extern int ib_create_qset ( struct ib_device *ibdev, @@ -27,8 +25,6 @@ extern int ib_create_qset ( struct ib_device *ibdev, struct ib_completion_queue_operations *cq_op, unsigned int num_send_wqes, unsigned int num_recv_wqes, unsigned long qkey ); -extern void ib_qset_refill_recv ( struct ib_device *ibdev, - struct ib_queue_set *qset ); extern void ib_destroy_qset ( struct ib_device *ibdev, struct ib_queue_set *qset ); diff --git a/src/include/gpxe/ib_sma.h b/src/include/gpxe/ib_sma.h index 6d984804..dadcdff2 100644 --- a/src/include/gpxe/ib_sma.h +++ b/src/include/gpxe/ib_sma.h @@ -10,7 +10,6 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include -#include /** Infiniband Subnet Management Agent operations */ struct ib_sma_operations { @@ -33,8 +32,6 @@ struct ib_sma { struct ib_completion_queue *cq; /** SMA queue pair */ struct ib_queue_pair *qp; - /** Poll process */ - struct process poll; }; /** SMA number of send WQEs diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index 6cfceda6..41c55ac5 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -154,6 +154,10 @@ struct ib_completion_queue_operations { /** An Infiniband Completion Queue */ struct ib_completion_queue { + /** Containing Infiniband device */ + struct ib_device *ibdev; + /** List of completion queues on this Infiniband device */ + struct list_head list; /** Completion queue number */ unsigned long cqn; /** Number of completion queue entries */ @@ -310,6 +314,8 @@ struct ib_device { struct list_head list; /** Underlying device */ struct device *dev; + /** List of completion queues */ + struct list_head cqs; /** List of queue pairs */ struct list_head qps; /** Infiniband operations */ @@ -350,6 +356,8 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes, struct ib_completion_queue_operations *op ); extern void ib_destroy_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ); +extern void ib_poll_cq ( struct ib_device *ibdev, + struct ib_completion_queue *cq ); extern struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev, unsigned int num_send_wqes, struct ib_completion_queue *send_cq, unsigned int num_recv_wqes, @@ -376,6 +384,8 @@ extern void ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *av, struct io_buffer *iobuf, int rc ); +extern void ib_refill_recv ( struct ib_device *ibdev, + struct ib_queue_pair *qp ); extern int ib_open ( struct ib_device *ibdev ); extern void ib_close ( struct ib_device *ibdev ); extern int ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp, @@ -388,23 +398,13 @@ extern struct ib_device * alloc_ibdev ( size_t priv_size ); extern int register_ibdev ( struct ib_device *ibdev ); extern void unregister_ibdev ( struct ib_device *ibdev ); extern void ib_link_state_changed ( struct ib_device *ibdev ); +extern void ib_poll_eq ( struct ib_device *ibdev ); extern struct list_head ib_devices; /** Iterate over all network devices */ #define for_each_ibdev( ibdev ) \ list_for_each_entry ( (ibdev), &ib_devices, list ) -/** - * Poll completion queue - * - * @v ibdev Infiniband device - * @v cq Completion queue - */ -static inline __always_inline void -ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) { - ibdev->op->poll_cq ( ibdev, cq ); -} - /** * Check link state * diff --git a/src/net/infiniband.c b/src/net/infiniband.c index 4784f404..48572e03 100644 --- a/src/net/infiniband.c +++ b/src/net/infiniband.c @@ -43,6 +43,13 @@ FILE_LICENCE ( GPL2_OR_LATER ); /** List of Infiniband devices */ struct list_head ib_devices = LIST_HEAD_INIT ( ib_devices ); +/*************************************************************************** + * + * Completion queues + * + *************************************************************************** + */ + /** * Create completion queue * @@ -63,6 +70,8 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes, cq = zalloc ( sizeof ( *cq ) ); if ( ! cq ) goto err_alloc_cq; + cq->ibdev = ibdev; + list_add ( &cq->list, &ibdev->cqs ); cq->num_cqes = num_cqes; INIT_LIST_HEAD ( &cq->work_queues ); cq->op = op; @@ -81,6 +90,7 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes, ibdev->op->destroy_cq ( ibdev, cq ); err_dev_create_cq: + list_del ( &cq->list ); free ( cq ); err_alloc_cq: return NULL; @@ -98,9 +108,37 @@ void ib_destroy_cq ( struct ib_device *ibdev, ibdev, cq->cqn ); assert ( list_empty ( &cq->work_queues ) ); ibdev->op->destroy_cq ( ibdev, cq ); + list_del ( &cq->list ); free ( cq ); } +/** + * Poll completion queue + * + * @v ibdev Infiniband device + * @v cq Completion queue + */ +void ib_poll_cq ( struct ib_device *ibdev, + struct ib_completion_queue *cq ) { + struct ib_work_queue *wq; + + /* Poll completion queue */ + ibdev->op->poll_cq ( ibdev, cq ); + + /* Refill receive work queues */ + list_for_each_entry ( wq, &cq->work_queues, list ) { + if ( ! wq->is_send ) + ib_refill_recv ( ibdev, wq->qp ); + } +} + +/*************************************************************************** + * + * Work queues + * + *************************************************************************** + */ + /** * Create queue pair * @@ -400,6 +438,44 @@ void ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp, qp->recv.fill--; } +/** + * Refill receive work queue + * + * @v ibdev Infiniband device + * @v qp Queue pair + */ +void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { + struct io_buffer *iobuf; + int rc; + + /* Keep filling while unfilled entries remain */ + while ( qp->recv.fill < qp->recv.num_wqes ) { + + /* Allocate I/O buffer */ + iobuf = alloc_iob ( IB_MAX_PAYLOAD_SIZE ); + if ( ! iobuf ) { + /* Non-fatal; we will refill on next attempt */ + return; + } + + /* Post I/O buffer */ + if ( ( rc = ib_post_recv ( ibdev, qp, iobuf ) ) != 0 ) { + DBGC ( ibdev, "IBDEV %p could not refill: %s\n", + ibdev, strerror ( rc ) ); + free_iob ( iobuf ); + /* Give up */ + return; + } + } +} + +/*************************************************************************** + * + * Link control + * + *************************************************************************** + */ + /** * Open port * @@ -436,6 +512,13 @@ void ib_close ( struct ib_device *ibdev ) { ibdev->op->close ( ibdev ); } +/*************************************************************************** + * + * Multicast + * + *************************************************************************** + */ + /** * Attach to multicast group * @@ -495,6 +578,13 @@ void ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp, } } +/*************************************************************************** + * + * Miscellaneous + * + *************************************************************************** + */ + /** * Get Infiniband HCA information * @@ -540,6 +630,22 @@ void ib_link_state_changed ( struct ib_device *ibdev ) { ipoib_link_state_changed ( ibdev ); } +/** + * Poll event queue + * + * @v ibdev Infiniband device + */ +void ib_poll_eq ( struct ib_device *ibdev ) { + struct ib_completion_queue *cq; + + /* Poll device's event queue */ + ibdev->op->poll_eq ( ibdev ); + + /* Poll all completion queues */ + list_for_each_entry ( cq, &ibdev->cqs, list ) + ib_poll_cq ( ibdev, cq ); +} + /** * Single-step the Infiniband event queue * @@ -548,9 +654,8 @@ void ib_link_state_changed ( struct ib_device *ibdev ) { static void ib_step ( struct process *process __unused ) { struct ib_device *ibdev; - list_for_each_entry ( ibdev, &ib_devices, list ) { - ibdev->op->poll_eq ( ibdev ); - } + for_each_ibdev ( ibdev ) + ib_poll_eq ( ibdev ); } /** Infiniband event queue process */ @@ -581,6 +686,7 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) { if ( ibdev ) { drv_priv = ( ( ( void * ) ibdev ) + sizeof ( *ibdev ) ); ib_set_drvdata ( ibdev, drv_priv ); + INIT_LIST_HEAD ( &ibdev->cqs ); INIT_LIST_HEAD ( &ibdev->qps ); ibdev->lid = IB_LID_NONE; ibdev->pkey = IB_PKEY_NONE; diff --git a/src/net/infiniband/ib_qset.c b/src/net/infiniband/ib_qset.c index 799489f9..0a1e1f9d 100644 --- a/src/net/infiniband/ib_qset.c +++ b/src/net/infiniband/ib_qset.c @@ -54,9 +54,6 @@ int ib_create_qset ( struct ib_device *ibdev, struct ib_queue_set *qset, assert ( qset->cq == NULL ); assert ( qset->qp == NULL ); - /* Store queue parameters */ - qset->recv_max_fill = num_recv_wqes; - /* Allocate completion queue */ qset->cq = ib_create_cq ( ibdev, num_cqes, cq_op ); if ( ! qset->cq ) { @@ -83,37 +80,6 @@ int ib_create_qset ( struct ib_device *ibdev, struct ib_queue_set *qset, return rc; } -/** - * Refill IPoIB receive ring - * - * @v ibdev Infiniband device - * @v qset Queue set - */ -void ib_qset_refill_recv ( struct ib_device *ibdev, - struct ib_queue_set *qset ) { - struct io_buffer *iobuf; - int rc; - - while ( qset->qp->recv.fill < qset->recv_max_fill ) { - - /* Allocate I/O buffer */ - iobuf = alloc_iob ( IB_MAX_PAYLOAD_SIZE ); - if ( ! iobuf ) { - /* Non-fatal; we will refill on next attempt */ - return; - } - - /* Post I/O buffer */ - if ( ( rc = ib_post_recv ( ibdev, qset->qp, iobuf ) ) != 0 ) { - DBGC ( ibdev, "IBDEV %p could not refill: %s\n", - ibdev, strerror ( rc ) ); - free_iob ( iobuf ); - /* Give up */ - return; - } - } -} - /** * Destroy queue set * diff --git a/src/net/infiniband/ib_sma.c b/src/net/infiniband/ib_sma.c index b83d20ea..5fd1319c 100644 --- a/src/net/infiniband/ib_sma.c +++ b/src/net/infiniband/ib_sma.c @@ -27,7 +27,6 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include #include #include -#include #include /** @@ -348,36 +347,6 @@ static int ib_sma_mad ( struct ib_sma *sma, union ib_mad *mad ) { return 0; } -/** - * Refill SMA receive ring - * - * @v sma Subnet management agent - */ -static void ib_sma_refill_recv ( struct ib_sma *sma ) { - struct ib_device *ibdev = sma->ibdev; - struct io_buffer *iobuf; - int rc; - - while ( sma->qp->recv.fill < IB_SMA_NUM_RECV_WQES ) { - - /* Allocate I/O buffer */ - iobuf = alloc_iob ( IB_MAX_PAYLOAD_SIZE ); - if ( ! iobuf ) { - /* Non-fatal; we will refill on next attempt */ - return; - } - - /* Post I/O buffer */ - if ( ( rc = ib_post_recv ( ibdev, sma->qp, iobuf ) ) != 0 ) { - DBGC ( sma, "SMA %p could not refill: %s\n", - sma, strerror ( rc ) ); - free_iob ( iobuf ); - /* Give up */ - return; - } - } -} - /** * Complete SMA send * @@ -456,23 +425,6 @@ static struct ib_completion_queue_operations ib_sma_completion_ops = { .complete_recv = ib_sma_complete_recv, }; -/** - * Poll SMA - * - * @v process Process - */ -static void ib_sma_step ( struct process *process ) { - struct ib_sma *sma = - container_of ( process, struct ib_sma, poll ); - struct ib_device *ibdev = sma->ibdev; - - /* Poll the kernel completion queue */ - ib_poll_cq ( ibdev, sma->cq ); - - /* Refill the receive ring */ - ib_sma_refill_recv ( sma ); -} - /** * Create SMA * @@ -489,7 +441,6 @@ int ib_create_sma ( struct ib_sma *sma, struct ib_device *ibdev, memset ( sma, 0, sizeof ( *sma ) ); sma->ibdev = ibdev; sma->op = op; - process_init ( &sma->poll, ib_sma_step, &ibdev->refcnt ); /* Create completion queue */ sma->cq = ib_create_cq ( ibdev, IB_SMA_NUM_CQES, @@ -517,7 +468,7 @@ int ib_create_sma ( struct ib_sma *sma, struct ib_device *ibdev, } /* Fill receive ring */ - ib_sma_refill_recv ( sma ); + ib_refill_recv ( ibdev, sma->qp ); return 0; err_not_qp0: @@ -525,7 +476,6 @@ int ib_create_sma ( struct ib_sma *sma, struct ib_device *ibdev, err_create_qp: ib_destroy_cq ( ibdev, sma->cq ); err_create_cq: - process_del ( &sma->poll ); return rc; } @@ -539,5 +489,4 @@ void ib_destroy_sma ( struct ib_sma *sma ) { ib_destroy_qp ( ibdev, sma->qp ); ib_destroy_cq ( ibdev, sma->cq ); - process_del ( &sma->poll ); }