From 8b27da9de16675f59be082168de9468346ec7183 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Sat, 15 Sep 2007 01:35:07 +0100 Subject: [PATCH] Gets a response out of the hardware. (An error completion, to be precise.) --- src/drivers/net/mlx_ipoib/bit_ops.h | 41 +++++++++++++++--- src/drivers/net/mlx_ipoib/mt25218.c | 67 ++++++++++++++++++++++++++--- src/include/gpxe/infiniband.h | 4 +- 3 files changed, 98 insertions(+), 14 deletions(-) diff --git a/src/drivers/net/mlx_ipoib/bit_ops.h b/src/drivers/net/mlx_ipoib/bit_ops.h index 74823a60..969de642 100644 --- a/src/drivers/net/mlx_ipoib/bit_ops.h +++ b/src/drivers/net/mlx_ipoib/bit_ops.h @@ -141,18 +141,33 @@ struct addr_64_st { #define MLX_BIT_OFFSET( _structure, _field ) \ offsetof ( struct _structure, _field ) +/** Dword offset of a field within a pseudo_bit_t structure */ +#define MLX_DWORD_OFFSET( _structure, _field ) \ + ( MLX_BIT_OFFSET ( _structure, _field ) / 32 ) + +/** Dword bit offset of a field within a pseudo_bit_t structure + * + * Yes, using mod-32 would work, but would lose the check for the + * error of specifying a mismatched field name and dword index. + */ +#define MLX_DWORD_BIT_OFFSET( _structure, _index, _field ) \ + ( MLX_BIT_OFFSET ( _structure, _field ) - ( 32 * (_index) ) ) + /** Bit width of a field within a pseudo_bit_t structure */ #define MLX_BIT_WIDTH( _structure, _field ) \ sizeof ( ( ( struct _structure * ) NULL )->_field ) +/** Bit mask for a field within a pseudo_bit_t structure */ +#define MLX_BIT_MASK( _structure, _field ) \ + ( ( 1 << MLX_BIT_WIDTH ( _structure, _field ) ) - 1 ) + /* * Assemble native-endian dword from named fields and values * */ #define MLX_ASSEMBLE_1( _structure, _index, _field, _value ) \ - ( (_value) << \ - ( MLX_BIT_OFFSET ( _structure, _field ) - ( 32 * (_index) ) ) ) + ( (_value) << MLX_DWORD_BIT_OFFSET ( _structure, _index, _field ) ) #define MLX_ASSEMBLE_2( _structure, _index, _field, _value, ... ) \ ( MLX_ASSEMBLE_1 ( _structure, _index, _field, _value ) | \ @@ -172,9 +187,8 @@ struct addr_64_st { */ #define MLX_MASK_1( _structure, _index, _field ) \ - MLX_ASSEMBLE_1 ( _structure, _index, _field, \ - ( ( 1 << MLX_BIT_WIDTH ( _structure, \ - _field ) ) - 1 ) ) + ( MLX_BIT_MASK ( _structure, _field ) << \ + MLX_DWORD_BIT_OFFSET ( _structure, _index, _field ) ) #define MLX_MASK_2( _structure, _index, _field, ... ) \ ( MLX_MASK_1 ( _structure, _index, _field ) | \ @@ -231,4 +245,21 @@ struct addr_64_st { *__ptr = cpu_to_be32 ( __value ); \ } while ( 0 ) +/* + * Extract value of named field + * + */ + +#define MLX_EXTRACT( _base, _structure, _field ) \ + ( { \ + unsigned int __index = \ + MLX_DWORD_OFFSET ( _structure, _field ); \ + uint32_t *__ptr = ( ( (uint32_t *) (_base) ) + __index ); \ + uint32_t __value = be32_to_cpu ( *__ptr ); \ + __value >>= MLX_DWORD_BIT_OFFSET ( _structure, __index, \ + _field ); \ + __value &= MLX_BIT_MASK ( _structure, _field ); \ + __value; \ + } ) + #endif /* __bit_ops_h__ */ diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index e8fcbb40..601a1f40 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -27,12 +27,13 @@ struct arbel_send_work_queue { /** Doorbell number */ unsigned int doorbell_idx; /** Work queue entries */ - struct ud_send_wqe_st *wqe; + // struct ud_send_wqe_st *wqe; + union ud_send_wqe_u *wqe_u; }; struct arbel { /** User Access Region */ - unsigned long uar; + void *uar; /** Doorbell records */ union db_record_st *db_rec; }; @@ -88,7 +89,6 @@ static uint8_t ib_broadcast[IB_ALEN] = { 0xff, }; static int mlx_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) { struct mlx_nic *mlx = netdev->priv; - ud_av_t av = iobuf->data; ud_send_wqe_t snd_wqe; int rc; @@ -110,6 +110,58 @@ static int mlx_transmit ( struct net_device *netdev, return 0; } +static int arbel_post_send ( struct ib_device *ibdev, struct io_buffer *iobuf, + struct ib_address_vector *av, + struct ib_queue_pair *qp ); + +static struct io_buffer *tx_ring[NUM_IPOIB_SND_WQES]; +static int tx_posted = 0; + +static int mlx_transmit_direct ( struct net_device *netdev, + struct io_buffer *iobuf ) { + struct mlx_nic *mlx = netdev->priv; + int rc; + + struct arbel arbel = { + .uar = memfree_pci_dev.uar, + .db_rec = dev_ib_data.uar_context_base, + }; + struct arbel_send_work_queue arbel_send_queue = { + .doorbell_idx = IPOIB_SND_QP_DB_IDX, + .wqe_u = ( (struct udqp_st *) ipoib_data.ipoib_qph )->snd_wq, + }; + struct ib_device ibdev = { + .priv = &arbel, + }; + struct ib_queue_pair qp = { + .qpn = ib_get_qpn ( mlx->ipoib_qph ), + .send = { + .num_wqes = NUM_IPOIB_SND_WQES, + .posted = tx_posted, + .iobufs = tx_ring, + .priv = &arbel_send_queue, + }, + }; + struct ud_av_st *bcast_av = mlx->bcast_av; + struct address_vector_st *bav = &bcast_av->av; + struct ib_address_vector av = { + .dest_qp = bcast_av->dest_qp, + .qkey = bcast_av->qkey, + .dlid = MLX_EXTRACT ( bav, arbelprm_ud_address_vector_st, rlid ), + .rate = ( MLX_EXTRACT ( bav, arbelprm_ud_address_vector_st, max_stat_rate ) ? 1 : 4 ), + .sl = MLX_EXTRACT ( bav, arbelprm_ud_address_vector_st, sl ), + .gid_present = 1, + }; + memcpy ( &av.gid, ( ( void * ) bav ) + 16, 16 ); + + rc = arbel_post_send ( &ibdev, iobuf, &av, &qp ); + + tx_posted = qp.send.posted; + + return rc; +} + + /** * Handle TX completion * @@ -234,7 +286,7 @@ static void mlx_irq ( struct net_device *netdev, int enable ) { static struct net_device_operations mlx_operations = { .open = mlx_open, .close = mlx_close, - .transmit = mlx_transmit, + .transmit = mlx_transmit_direct, .poll = mlx_poll, .irq = mlx_irq, }; @@ -274,12 +326,13 @@ static int arbel_post_send ( struct ib_device *ibdev, struct io_buffer *iobuf, /* Allocate work queue entry */ prev_wqe_idx = wq->posted; wqe_idx = ( prev_wqe_idx + 1 ); - if ( wq->iobuf[wqe_idx & wqe_idx_mask] ) { + if ( wq->iobufs[wqe_idx & wqe_idx_mask] ) { DBGC ( arbel, "ARBEL %p send queue full", arbel ); return -ENOBUFS; } - prev_wqe = &arbel_wq->wqe[prev_wqe_idx & wqe_idx_mask]; - wqe = &arbel_wq->wqe[wqe_idx & wqe_idx_mask]; + wq->iobufs[wqe_idx & wqe_idx_mask] = iobuf; + prev_wqe = &arbel_wq->wqe_u[prev_wqe_idx & wqe_idx_mask].wqe_cont.wqe; + wqe = &arbel_wq->wqe_u[wqe_idx & wqe_idx_mask].wqe_cont.wqe; /* Construct work queue entry */ memset ( &wqe->next.control, 0, diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index ccb6e49e..8b3a2f7c 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -73,10 +73,10 @@ struct ib_work_queue { * This is the index of the most recently posted entry. */ unsigned int posted; + /** I/O buffers assigned to work queue */ + struct io_buffer **iobufs; /** Driver private data */ void *priv; - /** I/O buffers assigned to work queue */ - struct io_buffer *iobuf[0]; }; /** An Infiniband Queue Pair */