diff --git a/src/drivers/net/mlx_ipoib/bit_ops.h b/src/drivers/net/mlx_ipoib/bit_ops.h index 969de642..2bc7684d 100644 --- a/src/drivers/net/mlx_ipoib/bit_ops.h +++ b/src/drivers/net/mlx_ipoib/bit_ops.h @@ -137,112 +137,139 @@ struct addr_64_st { /* Remaining code Copyright Fen Systems Ltd. 2007 */ +/** + * Wrapper structure for pseudo_bit_t structures + * + * This structure provides a wrapper around the autogenerated + * pseudo_bit_t structures. It has the correct size, and also + * encapsulates type information about the underlying pseudo_bit_t + * structure, which allows the MLX_POPULATE etc. macros to work + * without requiring explicit type information. + */ +#define MLX_DECLARE_STRUCT( _structure ) \ + _structure { \ + union { \ + uint8_t bytes[ sizeof ( struct _structure ## _st ) / 8 ]; \ + uint32_t dwords[ sizeof ( struct _structure ## _st ) / 32 ]; \ + struct _structure ## _st *dummy[0]; \ + } u; \ + } + +/** Get pseudo_bit_t structure type from wrapper structure pointer */ +#define MLX_PSEUDO_STRUCT( _ptr ) \ + typeof ( *((_ptr)->u.dummy[0]) ) + /** Bit offset of a field within a pseudo_bit_t structure */ -#define MLX_BIT_OFFSET( _structure, _field ) \ - offsetof ( struct _structure, _field ) +#define MLX_BIT_OFFSET( _structure_st, _field ) \ + offsetof ( _structure_st, _field ) /** Dword offset of a field within a pseudo_bit_t structure */ -#define MLX_DWORD_OFFSET( _structure, _field ) \ - ( MLX_BIT_OFFSET ( _structure, _field ) / 32 ) +#define MLX_DWORD_OFFSET( _structure_st, _field ) \ + ( MLX_BIT_OFFSET ( _structure_st, _field ) / 32 ) /** Dword bit offset of a field within a pseudo_bit_t structure * * Yes, using mod-32 would work, but would lose the check for the * error of specifying a mismatched field name and dword index. */ -#define MLX_DWORD_BIT_OFFSET( _structure, _index, _field ) \ - ( MLX_BIT_OFFSET ( _structure, _field ) - ( 32 * (_index) ) ) +#define MLX_DWORD_BIT_OFFSET( _structure_st, _index, _field ) \ + ( MLX_BIT_OFFSET ( _structure_st, _field ) - ( 32 * (_index) ) ) /** Bit width of a field within a pseudo_bit_t structure */ -#define MLX_BIT_WIDTH( _structure, _field ) \ - sizeof ( ( ( struct _structure * ) NULL )->_field ) +#define MLX_BIT_WIDTH( _structure_st, _field ) \ + sizeof ( ( ( _structure_st * ) NULL )->_field ) /** Bit mask for a field within a pseudo_bit_t structure */ -#define MLX_BIT_MASK( _structure, _field ) \ - ( ( 1 << MLX_BIT_WIDTH ( _structure, _field ) ) - 1 ) +#define MLX_BIT_MASK( _structure_st, _field ) \ + ( ( 1 << MLX_BIT_WIDTH ( _structure_st, _field ) ) - 1 ) /* * Assemble native-endian dword from named fields and values * */ -#define MLX_ASSEMBLE_1( _structure, _index, _field, _value ) \ - ( (_value) << MLX_DWORD_BIT_OFFSET ( _structure, _index, _field ) ) +#define MLX_ASSEMBLE_1( _structure_st, _index, _field, _value ) \ + ( (_value) << MLX_DWORD_BIT_OFFSET ( _structure_st, _index, _field ) ) -#define MLX_ASSEMBLE_2( _structure, _index, _field, _value, ... ) \ - ( MLX_ASSEMBLE_1 ( _structure, _index, _field, _value ) | \ - MLX_ASSEMBLE_1 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_ASSEMBLE_2( _structure_st, _index, _field, _value, ... ) \ + ( MLX_ASSEMBLE_1 ( _structure_st, _index, _field, _value ) | \ + MLX_ASSEMBLE_1 ( _structure_st, _index, __VA_ARGS__ ) ) -#define MLX_ASSEMBLE_3( _structure, _index, _field, _value, ... ) \ - ( MLX_ASSEMBLE_1 ( _structure, _index, _field, _value ) | \ - MLX_ASSEMBLE_2 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_ASSEMBLE_3( _structure_st, _index, _field, _value, ... ) \ + ( MLX_ASSEMBLE_1 ( _structure_st, _index, _field, _value ) | \ + MLX_ASSEMBLE_2 ( _structure_st, _index, __VA_ARGS__ ) ) -#define MLX_ASSEMBLE_4( _structure, _index, _field, _value, ... ) \ - ( MLX_ASSEMBLE_1 ( _structure, _index, _field, _value ) | \ - MLX_ASSEMBLE_3 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_ASSEMBLE_4( _structure_st, _index, _field, _value, ... ) \ + ( MLX_ASSEMBLE_1 ( _structure_st, _index, _field, _value ) | \ + MLX_ASSEMBLE_3 ( _structure_st, _index, __VA_ARGS__ ) ) /* * Build native-endian (positive) dword bitmasks from named fields * */ -#define MLX_MASK_1( _structure, _index, _field ) \ - ( MLX_BIT_MASK ( _structure, _field ) << \ - MLX_DWORD_BIT_OFFSET ( _structure, _index, _field ) ) +#define MLX_MASK_1( _structure_st, _index, _field ) \ + ( MLX_BIT_MASK ( _structure_st, _field ) << \ + MLX_DWORD_BIT_OFFSET ( _structure_st, _index, _field ) ) -#define MLX_MASK_2( _structure, _index, _field, ... ) \ - ( MLX_MASK_1 ( _structure, _index, _field ) | \ - MLX_MASK_1 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_MASK_2( _structure_st, _index, _field, ... ) \ + ( MLX_MASK_1 ( _structure_st, _index, _field ) | \ + MLX_MASK_1 ( _structure_st, _index, __VA_ARGS__ ) ) -#define MLX_MASK_3( _structure, _index, _field, ... ) \ - ( MLX_MASK_1 ( _structure, _index, _field ) | \ - MLX_MASK_2 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_MASK_3( _structure_st, _index, _field, ... ) \ + ( MLX_MASK_1 ( _structure_st, _index, _field ) | \ + MLX_MASK_2 ( _structure_st, _index, __VA_ARGS__ ) ) -#define MLX_MASK_4( _structure, _index, _field, ... ) \ - ( MLX_MASK_1 ( _structure, _index, _field ) | \ - MLX_MASK_3 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_MASK_4( _structure_st, _index, _field, ... ) \ + ( MLX_MASK_1 ( _structure_st, _index, _field ) | \ + MLX_MASK_3 ( _structure_st, _index, __VA_ARGS__ ) ) /* * Populate big-endian dwords from named fields and values * */ -#define MLX_POPULATE( _base, _index, _assembled ) \ - do { \ - uint32_t *__ptr = ( ( (uint32_t *) (_base) ) + (_index) ); \ - uint32_t __assembled = (_assembled); \ - *__ptr = cpu_to_be32 ( __assembled ); \ +#define MLX_POPULATE( _ptr, _index, _assembled ) \ + do { \ + uint32_t *__ptr = &(_ptr)->u.dwords[(_index)]; \ + uint32_t __assembled = (_assembled); \ + *__ptr = cpu_to_be32 ( __assembled ); \ } while ( 0 ) -#define MLX_POPULATE_1( _base, _structure, _index, ... ) \ - MLX_POPULATE ( _base, _index, \ - MLX_ASSEMBLE_1 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_POPULATE_1( _ptr, _index, ... ) \ + MLX_POPULATE ( _ptr, _index, \ + MLX_ASSEMBLE_1 ( MLX_PSEUDO_STRUCT ( _ptr ), \ + _index, __VA_ARGS__ ) ) -#define MLX_POPULATE_2( _base, _structure, _index, ... ) \ - MLX_POPULATE ( _base, _index, \ - MLX_ASSEMBLE_2 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_POPULATE_2( _ptr, _index, ... ) \ + MLX_POPULATE ( _ptr, _index, \ + MLX_ASSEMBLE_2 ( MLX_PSEUDO_STRUCT ( _ptr ), \ + _index, __VA_ARGS__ ) ) -#define MLX_POPULATE_3( _base, _structure, _index, ... ) \ - MLX_POPULATE ( _base, _index, \ - MLX_ASSEMBLE_3 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_POPULATE_3( _ptr, _index, ... ) \ + MLX_POPULATE ( _ptr, _index, \ + MLX_ASSEMBLE_3 ( MLX_PSEUDO_STRUCT ( _ptr ), \ + _index, __VA_ARGS__ ) ) -#define MLX_POPULATE_4( _base, _structure, _index, ... ) \ - MLX_POPULATE ( _base, _index, \ - MLX_ASSEMBLE_4 ( _structure, _index, __VA_ARGS__ ) ) +#define MLX_POPULATE_4( _ptr, _index, ... ) \ + MLX_POPULATE ( _ptr, _index, \ + MLX_ASSEMBLE_4 ( MLX_PSEUDO_STRUCT ( _ptr ), \ + _index, __VA_ARGS__ ) ) /* * Modify big-endian dword using named field and value * */ -#define MLX_MODIFY( _base, _structure, _index, _field, _value ) \ - do { \ - uint32_t *__ptr = ( ( (uint32_t *) (_base) ) + (_index) ); \ - uint32_t __value = be32_to_cpu ( *__ptr ); \ - __value &= ~( MLX_MASK_1 ( _structure, _index, _field ) ); \ - __value |= MLX_ASSEMBLE_1 ( _structure, _index, \ - _field, _value ); \ - *__ptr = cpu_to_be32 ( __value ); \ +#define MLX_MODIFY( _ptr, _index, _field, _value ) \ + do { \ + uint32_t *__ptr = &(_ptr)->u.dwords[(_index)]; \ + uint32_t __value = be32_to_cpu ( *__ptr ); \ + __value &= ~( MLX_MASK_1 ( MLX_PSEUDO_STRUCT ( _ptr ), \ + _index, _field ) ); \ + __value |= MLX_ASSEMBLE_1 ( MLX_PSEUDO_STRUCT ( _ptr ), \ + _index, _field, _value ); \ + *__ptr = cpu_to_be32 ( __value ); \ } while ( 0 ) /* @@ -250,16 +277,18 @@ struct addr_64_st { * */ -#define MLX_EXTRACT( _base, _structure, _field ) \ - ( { \ - unsigned int __index = \ - MLX_DWORD_OFFSET ( _structure, _field ); \ - uint32_t *__ptr = ( ( (uint32_t *) (_base) ) + __index ); \ - uint32_t __value = be32_to_cpu ( *__ptr ); \ - __value >>= MLX_DWORD_BIT_OFFSET ( _structure, __index, \ - _field ); \ - __value &= MLX_BIT_MASK ( _structure, _field ); \ - __value; \ +#define MLX_EXTRACT( _ptr, _field ) \ + ( { \ + unsigned int __index = \ + MLX_DWORD_OFFSET ( MLX_PSEUDO_STRUCT ( _ptr ), _field ); \ + uint32_t *__ptr = &(_ptr)->u.dwords[__index]; \ + uint32_t __value = be32_to_cpu ( *__ptr ); \ + __value >>= \ + MLX_DWORD_BIT_OFFSET ( MLX_PSEUDO_STRUCT ( _ptr ), \ + __index, _field ); \ + __value &= \ + MLX_BIT_MASK ( MLX_PSEUDO_STRUCT ( _ptr ), _field ); \ + __value; \ } ) #endif /* __bit_ops_h__ */ diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index ecf873bb..42e5465c 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -23,14 +23,23 @@ Skeleton NIC driver for Etherboot #include "mt25218_imp.c" +#include "arbel.h" + struct arbel_send_work_queue { - /** Doorbell number */ + /** Doorbell record number */ unsigned int doorbell_idx; /** Work queue entries */ // struct ud_send_wqe_st *wqe; union ud_send_wqe_u *wqe_u; }; +struct arbel_completion_queue { + /** Doorbell record number */ + unsigned int doorbell_idx; + /** Completion queue entries */ + union cqe_st *cqe; +}; + struct arbel { /** User Access Region */ void *uar; @@ -143,13 +152,14 @@ static int mlx_transmit_direct ( struct net_device *netdev, }, }; struct ud_av_st *bcast_av = mlx->bcast_av; - struct address_vector_st *bav = &bcast_av->av; + struct arbelprm_ud_address_vector *bav = + ( struct arbelprm_ud_address_vector * ) &bcast_av->av; struct ib_address_vector av = { .dest_qp = bcast_av->dest_qp, .qkey = bcast_av->qkey, - .dlid = MLX_EXTRACT ( bav, arbelprm_ud_address_vector_st, rlid ), - .rate = ( MLX_EXTRACT ( bav, arbelprm_ud_address_vector_st, max_stat_rate ) ? 1 : 4 ), - .sl = MLX_EXTRACT ( bav, arbelprm_ud_address_vector_st, sl ), + .dlid = MLX_EXTRACT ( bav, rlid ), + .rate = ( MLX_EXTRACT ( bav, max_stat_rate ) ? 1 : 4 ), + .sl = MLX_EXTRACT ( bav, sl ), .gid_present = 1, }; memcpy ( &av.gid, ( ( void * ) bav ) + 16, 16 ); @@ -301,6 +311,13 @@ static struct ib_gid arbel_no_gid = { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 } }; +/** + * Ring doorbell register in UAR + * + * @v arbel Arbel device + * @v db_reg Doorbell register structure + * @v offset Address of doorbell + */ static void arbel_ring_doorbell ( struct arbel *arbel, void *db_reg, unsigned int offset ) { uint32_t *db_reg_dword = db_reg; @@ -315,6 +332,15 @@ static void arbel_ring_doorbell ( struct arbel *arbel, void *db_reg, writel ( db_reg_dword[1], ( arbel->uar + offset + 4 ) ); } +/** + * Post send work queue entry + * + * @v ibdev Infiniband device + * @v iobuf I/O buffer + * @v av Address vector + * @v qp Queue pair + * @ret rc Return status code + */ static int arbel_post_send ( struct ib_device *ibdev, struct io_buffer *iobuf, struct ib_address_vector *av, struct ib_queue_pair *qp ) { @@ -365,14 +391,8 @@ static int arbel_post_send ( struct ib_device *ibdev, struct io_buffer *iobuf, destination_qp, av->dest_qp ); MLX_POPULATE_1 ( &wqe->udseg, arbelprm_wqe_segment_ud_st, 9, q_key, av->qkey ); - wqe->mpointer[0].local_addr_l = cpu_to_be32 ( virt_to_bus ( iobuf->data ) ); - - // memcpy ( bus_to_virt ( be32_to_cpu ( wqe->mpointer[0].local_addr_l ) ), - // iobuf->data, iob_len ( iobuf ) ); - - wqe->mpointer[0].byte_count = cpu_to_be32 ( iob_len ( iobuf ) ); DBG ( "Work queue entry:\n" ); @@ -416,8 +436,69 @@ static int arbel_post_send ( struct ib_device *ibdev, struct io_buffer *iobuf, return 0; } +static void arbel_parse_completion ( struct arbel *arbel, + union cqe_st *cqe, + struct ib_completion *completion ) { + memset ( completion, 0, sizeof ( *completion ) ); + is_send = MLX_EXTRACT ( cqe, arbelprm_completion_queue_entry_st, s ); + completion->len = + MLX_EXTRACT ( cqe, arbelprm_completion_queue_entry_st, + byte_cnt );} + +/** + * Poll completion queue + * + * @v ibdev Infiniband device + * @v cq Completion queue + * @v complete Completion handler + */ +static void arbel_poll_cq ( struct ib_device *ibdev, + struct ib_completion_queue *cq, + ib_completer_t complete_send, + ib_completer_t complete_recv ) { + struct arbel *arbel = ibdev->priv; + struct arbel_completion_queue *arbel_cq = cq->priv; + unsigned int cqe_idx_mask = ( cq->num_cqes - 1 ); + union db_record_st *db_rec = &arbel->db_rec[arbel_cq->doorbell_idx]; + union cqe_st *cqe; + struct ib_completion completion; + struct io_buffer *iobuf; + int is_send; + + while ( 1 ) { + /* Look for completion entry */ + cqe = &arbel_cq->cqe[cq->next_idx & cqe_idx_mask]; + if ( MLX_EXTRACT ( cqe, arbelprm_completion_queue_entry_st, + owner ) != 0 ) { + /* Entry still owned by hardware; end of poll */ + break; + } + + /* Parse completion */ + + + + /* Handle completion */ + ( is_send ? complete_send : complete_recv ) ( ibdev, + &completion, + iobuf ); + + /* Return ownership to hardware */ + MLX_POPULATE_1 ( cqe, arbelprm_completion_queue_entry_st, 7, + owner, 1 ); + barrier(); + /* Update completion queue's index */ + cq->next_idx++; + /* Update doorbell record */ + MLX_POPULATE_1 ( db_rec, arbelprm_cq_ci_db_record_st, 0, + counter, ( cq->next_idx & 0xffffffffUL ) ); + } +} + +/** Arbel Infiniband operations */ static struct ib_device_operations arbel_ib_operations = { .post_send = arbel_post_send, + .poll_cq = arbel_poll_cq, }; /** diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index 9337af35..c0819158 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -63,6 +63,7 @@ struct ibhdr { +struct ib_device; /** An Infiniband Work Queue */ struct ib_work_queue { @@ -71,9 +72,11 @@ struct ib_work_queue { /** Next work queue entry index * * This is the index of the next entry to be filled (i.e. the - * first empty entry). + * first empty entry). This value is not bounded by num_wqes; + * users must logical-AND with (num_wqes-1) to generate an + * array index. */ - unsigned int next_idx; + unsigned long next_idx; /** I/O buffers assigned to work queue */ struct io_buffer **iobufs; /** Driver private data */ @@ -92,6 +95,38 @@ struct ib_queue_pair { void *priv; }; +/** An Infiniband Completion Queue */ +struct ib_completion_queue { + /** Number of completion queue entries */ + unsigned int num_cqes; + /** Next completion queue entry index + * + * This is the index of the next entry to be filled (i.e. the + * first empty entry). This value is not bounded by num_wqes; + * users must logical-AND with (num_wqes-1) to generate an + * array index. + */ + unsigned long next_idx; + /** Driver private data */ + void *priv; +}; + +/** An Infiniband completion */ +struct ib_completion { + /** Length */ + size_t len; +}; + +/** An Infiniband completion handler + * + * @v ibdev Infiniband device + * @v completion Completion + * @v iobuf I/O buffer + */ +typedef void ( * ib_completer_t ) ( struct ib_device *ibdev, + struct ib_completion *completion, + struct io_buffer *iobuf ); + /** An Infiniband Address Vector */ struct ib_address_vector { /** Destination Queue Pair */ @@ -110,15 +145,13 @@ struct ib_address_vector { struct ib_gid gid; }; -struct ib_device; - /** * Infiniband device operations * * These represent a subset of the Infiniband Verbs. */ struct ib_device_operations { - /** Post Send work queue entry + /** Post send work queue entry * * @v ibdev Infiniband device * @v iobuf I/O buffer @@ -135,6 +168,19 @@ struct ib_device_operations { struct io_buffer *iobuf, struct ib_address_vector *av, struct ib_queue_pair *qp ); + /** Poll completion queue + * + * @v ibdev Infiniband device + * @v cq Completion queue + * @v complete_send Send completion handler + * @v complete_recv Receive completion handler + * + * The completion handler takes ownership of the I/O buffer. + */ + void ( * poll_cq ) ( struct ib_device *ibdev, + struct ib_completion_queue *cq, + ib_completer_t complete_send, + ib_completer_t complete_recv ); }; /** An Infiniband device */