david/ipxe
david
/
ipxe
Archived
1
0
Fork 0
Code

[infiniband] Maintain queue fill level as a property of a work queue 

Both queue owners and drivers often need to keep track of the fill
level, so let's make it a generic property.
This commit is contained in:
Michael Brown 2008-10-03 03:04:21 +01:00
parent d9751edafa
commit 0de5f7af6d
3 changed files with 110 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/drivers/net/ipoib.c
View File

@ -57,8 +57,6 @@ struct ipoib_queue_set {
struct ib_completion_queue *cq;
/** Queue pair */
struct ib_queue_pair *qp;
/** Receive work queue fill level */
unsigned int recv_fill;
/** Receive work queue maximum fill level */
unsigned int recv_max_fill;
};
@ -565,7 +563,7 @@ static void ipoib_data_complete_recv ( struct ib_device *ibdev __unused,
if ( completion->syndrome ) {
netdev_rx_err ( netdev, iobuf, -EIO );
goto done;
return;
}
iob_put ( iobuf, completion->len );
@ -574,7 +572,7 @@ static void ipoib_data_complete_recv ( struct ib_device *ibdev __unused,
"contain GRH\n", ipoib );
DBGC_HD ( ipoib, iobuf->data, iob_len ( iobuf ) );
netdev_rx_err ( netdev, iobuf, -EIO );
goto done;
return;
}
iob_pull ( iobuf, sizeof ( struct ib_global_route_header ) );
@ -583,16 +581,13 @@ static void ipoib_data_complete_recv ( struct ib_device *ibdev __unused,
"contain IPoIB header\n", ipoib );
DBGC_HD ( ipoib, iobuf->data, iob_len ( iobuf ) );
netdev_rx_err ( netdev, iobuf, -EIO );
goto done;
return;
}
ipoib_pshdr = iob_push ( iobuf, sizeof ( *ipoib_pshdr ) );
/* FIXME: fill in a MAC address for the sake of AoE! */
netdev_rx ( netdev, iobuf );
done:
ipoib->data.recv_fill--;
}
/**
@ -732,7 +727,6 @@ static void ipoib_meta_complete_recv ( struct ib_device *ibdev __unused,
}
done:
ipoib->meta.recv_fill--;
free_iob ( iobuf );
}
@ -747,7 +741,7 @@ static void ipoib_refill_recv ( struct ipoib_device *ipoib,
struct io_buffer *iobuf;
int rc;
while ( qset->recv_fill < qset->recv_max_fill ) {
while ( qset->qp->recv.fill < qset->recv_max_fill ) {
iobuf = alloc_iob ( IPOIB_PKT_LEN );
if ( ! iobuf )
break;
@ -755,7 +749,6 @@ static void ipoib_refill_recv ( struct ipoib_device *ipoib,
free_iob ( iobuf );
break;
}
qset->recv_fill++;
}
}

74
src/include/gpxe/infiniband.h
View File

@ -66,6 +66,8 @@ struct ib_work_queue {
struct list_head list;
/** Number of work queue entries */
unsigned int num_wqes;
/** Number of occupied work queue entries */
unsigned int fill;
/** Next work queue entry index
*
* This is the index of the next entry to be filled (i.e. the
@ -355,70 +357,24 @@ extern void ib_destroy_qp ( struct ib_device *ibdev,
struct ib_queue_pair *qp );
extern struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
unsigned long qpn, int is_send );
extern int ib_post_send ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_address_vector *av,
struct io_buffer *iobuf );
extern int ib_post_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct io_buffer *iobuf );
extern void ib_complete_send ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_completion *completion,
struct io_buffer *iobuf );
extern void ib_complete_recv ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_completion *completion,
struct io_buffer *iobuf );
extern struct ib_device * alloc_ibdev ( size_t priv_size );
extern int register_ibdev ( struct ib_device *ibdev );
extern void unregister_ibdev ( struct ib_device *ibdev );
extern void ib_link_state_changed ( struct ib_device *ibdev );
/**
* Post send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v av Address vector
* @v iobuf I/O buffer
* @ret rc Return status code
*/
static inline __attribute__ (( always_inline )) int
ib_post_send ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_address_vector *av, struct io_buffer *iobuf ) {
return ibdev->op->post_send ( ibdev, qp, av, iobuf );
}
/**
* Post receive work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v iobuf I/O buffer
* @ret rc Return status code
*/
static inline __attribute__ (( always_inline )) int
ib_post_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct io_buffer *iobuf ) {
return ibdev->op->post_recv ( ibdev, qp, iobuf );
}
/**
* Complete send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v completion Completion
* @v iobuf I/O buffer
*/
static inline __attribute__ (( always_inline )) void
ib_complete_send ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_completion *completion,
struct io_buffer *iobuf ) {
return qp->send.cq->complete_send ( ibdev, qp, completion, iobuf );
}
/**
* Complete receive work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v completion Completion
* @v iobuf I/O buffer
*/
static inline __attribute__ (( always_inline )) void
ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_completion *completion,
struct io_buffer *iobuf ) {
return qp->recv.cq->complete_recv ( ibdev, qp, completion, iobuf );
}
/**
* Poll completion queue
*

91
src/net/infiniband.c
View File

@ -244,6 +244,97 @@ struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
return NULL;
}
/**
* Post send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v av Address vector
* @v iobuf I/O buffer
* @ret rc Return status code
*/
int ib_post_send ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_address_vector *av, struct io_buffer *iobuf ) {
int rc;
/* Check queue fill level */
if ( qp->send.fill >= qp->send.num_wqes ) {
DBGC ( ibdev, "IBDEV %p QPN %#lx send queue full\n",
ibdev, qp->qpn );
return -ENOBUFS;
}
/* Post to hardware */
if ( ( rc = ibdev->op->post_send ( ibdev, qp, av, iobuf ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p QPN %#lx could not post send WQE: "
"%s\n", ibdev, qp->qpn, strerror ( rc ) );
return rc;
}
qp->send.fill++;
return 0;
}
/**
* Post receive work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v iobuf I/O buffer
* @ret rc Return status code
*/
int ib_post_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct io_buffer *iobuf ) {
int rc;
/* Check queue fill level */
if ( qp->recv.fill >= qp->recv.num_wqes ) {
DBGC ( ibdev, "IBDEV %p QPN %#lx receive queue full\n",
ibdev, qp->qpn );
return -ENOBUFS;
}
/* Post to hardware */
if ( ( rc = ibdev->op->post_recv ( ibdev, qp, iobuf ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p QPN %#lx could not post receive WQE: "
"%s\n", ibdev, qp->qpn, strerror ( rc ) );
return rc;
}
qp->recv.fill++;
return 0;
}
/**
* Complete send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v completion Completion
* @v iobuf I/O buffer
*/
void ib_complete_send ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_completion *completion,
struct io_buffer *iobuf ) {
qp->send.cq->complete_send ( ibdev, qp, completion, iobuf );
qp->send.fill--;
}
/**
* Complete receive work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v completion Completion
* @v iobuf I/O buffer
*/
void ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_completion *completion,
struct io_buffer *iobuf ) {
qp->recv.cq->complete_recv ( ibdev, qp, completion, iobuf );
qp->recv.fill--;
}
/***************************************************************************
*
* Management datagram operations