From cb9ef4dee2535c1f95d683a68832a13ccd3b01a2 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Tue, 7 Jul 2009 16:26:57 +0100 Subject: [PATCH] [ipoib] Remove the queue set abstraction Now that IPoIB has to deal with only one set of queues, the queue set abstraction becomes merely an inconvenient wrapper. --- src/drivers/net/ipoib.c | 56 +++++++++++++-------- src/include/gpxe/ib_qset.h | 31 ------------ src/net/infiniband/ib_qset.c | 97 ------------------------------------ 3 files changed, 36 insertions(+), 148 deletions(-) delete mode 100644 src/include/gpxe/ib_qset.h delete mode 100644 src/net/infiniband/ib_qset.c diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c index 9f02b4a0..4c924d1d 100644 --- a/src/drivers/net/ipoib.c +++ b/src/drivers/net/ipoib.c @@ -28,7 +28,6 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include #include #include -#include #include #include #include @@ -53,8 +52,10 @@ struct ipoib_device { struct net_device *netdev; /** Underlying Infiniband device */ struct ib_device *ibdev; - /** Queue set */ - struct ib_queue_set qset; + /** Completion queue */ + struct ib_completion_queue *cq; + /** Queue pair */ + struct ib_queue_pair *qp; /** Broadcast MAC */ struct ipoib_mac broadcast; /** Joined to multicast group @@ -361,7 +362,7 @@ static int ipoib_transmit ( struct net_device *netdev, return rc; } - return ib_post_send ( ibdev, ipoib->qset.qp, &av, iobuf ); + return ib_post_send ( ibdev, ipoib->qp, &av, iobuf ); } /** @@ -464,7 +465,7 @@ static void ipoib_irq ( struct net_device *netdev __unused, static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) { int rc; - if ( ( rc = ib_mcast_join ( ipoib->ibdev, ipoib->qset.qp, + if ( ( rc = ib_mcast_join ( ipoib->ibdev, ipoib->qp, &ipoib->broadcast.gid ) ) != 0 ) { DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n", ipoib, strerror ( rc ) ); @@ -483,7 +484,7 @@ static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) { static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) { if ( ipoib->broadcast_joined ) { - ib_mcast_leave ( ipoib->ibdev, ipoib->qset.qp, + ib_mcast_leave ( ipoib->ibdev, ipoib->qp, &ipoib->broadcast.gid ); ipoib->broadcast_joined = 0; } @@ -508,21 +509,31 @@ static int ipoib_open ( struct net_device *netdev ) { goto err_ib_open; } - /* Allocate queue set */ - if ( ( rc = ib_create_qset ( ibdev, &ipoib->qset, IPOIB_NUM_CQES, - &ipoib_cq_op, IPOIB_NUM_SEND_WQES, - IPOIB_NUM_RECV_WQES, 0 ) ) != 0 ) { - DBGC ( ipoib, "IPoIB %p could not allocate queue set: %s\n", - ipoib, strerror ( rc ) ); - goto err_create_qset; + /* Allocate completion queue */ + ipoib->cq = ib_create_cq ( ibdev, IPOIB_NUM_CQES, &ipoib_cq_op ); + if ( ! ipoib->cq ) { + DBGC ( ipoib, "IPoIB %p could not allocate completion queue\n", + ipoib ); + rc = -ENOMEM; + goto err_create_cq; } - ib_qp_set_ownerdata ( ipoib->qset.qp, ipoib ); + + /* Allocate queue pair */ + ipoib->qp = ib_create_qp ( ibdev, IPOIB_NUM_SEND_WQES, ipoib->cq, + IPOIB_NUM_RECV_WQES, ipoib->cq, 0 ); + if ( ! ipoib->qp ) { + DBGC ( ipoib, "IPoIB %p could not allocate queue pair\n", + ipoib ); + rc = -ENOMEM; + goto err_create_qp; + } + ib_qp_set_ownerdata ( ipoib->qp, ipoib ); /* Update MAC address with QPN */ - mac->qpn = htonl ( ipoib->qset.qp->qpn ); + mac->qpn = htonl ( ipoib->qp->qpn ); /* Fill receive rings */ - ib_refill_recv ( ibdev, ipoib->qset.qp ); + ib_refill_recv ( ibdev, ipoib->qp ); /* Join broadcast group */ if ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) { @@ -533,9 +544,12 @@ static int ipoib_open ( struct net_device *netdev ) { return 0; + ipoib_leave_broadcast_group ( ipoib ); err_join_broadcast: - ib_destroy_qset ( ibdev, &ipoib->qset ); - err_create_qset: + ib_destroy_qp ( ibdev, ipoib->qp ); + err_create_qp: + ib_destroy_cq ( ibdev, ipoib->cq ); + err_create_cq: ib_close ( ibdev ); err_ib_open: return rc; @@ -548,6 +562,7 @@ static int ipoib_open ( struct net_device *netdev ) { */ static void ipoib_close ( struct net_device *netdev ) { struct ipoib_device *ipoib = netdev->priv; + struct ib_device *ibdev = ipoib->ibdev; struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr ); /* Leave broadcast group */ @@ -557,10 +572,11 @@ static void ipoib_close ( struct net_device *netdev ) { mac->qpn = 0; /* Tear down the queues */ - ib_destroy_qset ( ipoib->ibdev, &ipoib->qset ); + ib_destroy_qp ( ibdev, ipoib->qp ); + ib_destroy_cq ( ibdev, ipoib->cq ); /* Close IB device */ - ib_close ( ipoib->ibdev ); + ib_close ( ibdev ); } /** IPoIB network device operations */ diff --git a/src/include/gpxe/ib_qset.h b/src/include/gpxe/ib_qset.h deleted file mode 100644 index a1116978..00000000 --- a/src/include/gpxe/ib_qset.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef _GPXE_IB_QSET_H -#define _GPXE_IB_QSET_H - -/** @file - * - * Infiniband queue sets - * - */ - -FILE_LICENCE ( GPL2_OR_LATER ); - -#include -#include - -/** An Infiniband queue set */ -struct ib_queue_set { - /** Completion queue */ - struct ib_completion_queue *cq; - /** Queue pair */ - struct ib_queue_pair *qp; -}; - -extern int ib_create_qset ( struct ib_device *ibdev, - struct ib_queue_set *qset, unsigned int num_cqes, - struct ib_completion_queue_operations *cq_op, - unsigned int num_send_wqes, - unsigned int num_recv_wqes, unsigned long qkey ); -extern void ib_destroy_qset ( struct ib_device *ibdev, - struct ib_queue_set *qset ); - -#endif /* _GPXE_IB_QSET_H */ diff --git a/src/net/infiniband/ib_qset.c b/src/net/infiniband/ib_qset.c deleted file mode 100644 index 0a1e1f9d..00000000 --- a/src/net/infiniband/ib_qset.c +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2009 Michael Brown . - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -FILE_LICENCE ( GPL2_OR_LATER ); - -#include -#include -#include -#include -#include - -/** - * @file - * - * Infiniband queue sets - * - */ - -/** - * Create queue set - * - * @v ibdev Infiniband device - * @v qset Queue set - * @v num_cqes Number of completion queue entries - * @v cq_op Completion queue operations - * @v num_send_wqes Number of send work queue entries - * @v num_recv_wqes Number of receive work queue entries - * @v qkey Queue key - * @ret rc Return status code - */ -int ib_create_qset ( struct ib_device *ibdev, struct ib_queue_set *qset, - unsigned int num_cqes, - struct ib_completion_queue_operations *cq_op, - unsigned int num_send_wqes, unsigned int num_recv_wqes, - unsigned long qkey ) { - int rc; - - /* Sanity check */ - assert ( qset->cq == NULL ); - assert ( qset->qp == NULL ); - - /* Allocate completion queue */ - qset->cq = ib_create_cq ( ibdev, num_cqes, cq_op ); - if ( ! qset->cq ) { - DBGC ( ibdev, "IBDEV %p could not allocate completion queue\n", - ibdev ); - rc = -ENOMEM; - goto err; - } - - /* Allocate queue pair */ - qset->qp = ib_create_qp ( ibdev, num_send_wqes, qset->cq, - num_recv_wqes, qset->cq, qkey ); - if ( ! qset->qp ) { - DBGC ( ibdev, "IBDEV %p could not allocate queue pair\n", - ibdev ); - rc = -ENOMEM; - goto err; - } - - return 0; - - err: - ib_destroy_qset ( ibdev, qset ); - return rc; -} - -/** - * Destroy queue set - * - * @v ibdev Infiniband device - * @v qset Queue set - */ -void ib_destroy_qset ( struct ib_device *ibdev, - struct ib_queue_set *qset ) { - - if ( qset->qp ) - ib_destroy_qp ( ibdev, qset->qp ); - if ( qset->cq ) - ib_destroy_cq ( ibdev, qset->cq ); - memset ( qset, 0, sizeof ( *qset ) ); -}