diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c index 81011405..c4eea5a7 100644 --- a/src/drivers/net/ipoib.c +++ b/src/drivers/net/ipoib.c @@ -84,6 +84,8 @@ struct ipoib_device { struct ipoib_queue_set data; /** Data queue set */ struct ipoib_queue_set meta; + /** Broadcast GID */ + struct ib_gid broadcast_gid; }; /** @@ -112,6 +114,15 @@ static struct ipoib_cached_path ipoib_path_cache[IPOIB_NUM_CACHED_PATHS]; /** Oldest IPoIB path cache entry index */ static unsigned int ipoib_path_cache_idx = 0; +/** IPoIB metadata TID */ +static uint32_t ipoib_meta_tid = 0; + +/** IPv4 broadcast GID */ +static const struct ib_gid ipv4_broadcast_gid = { + { { 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff } } +}; + /**************************************************************************** * * IPoIB link layer @@ -319,7 +330,6 @@ static int ipoib_get_path_record ( struct ipoib_device *ipoib, struct io_buffer *iobuf; struct ib_mad_path_record *path_record; struct ib_address_vector av; - static uint32_t tid = 0; int rc; /* Allocate I/O buffer */ @@ -336,7 +346,7 @@ static int ipoib_get_path_record ( struct ipoib_device *ipoib, path_record->mad_hdr.class_version = 2; path_record->mad_hdr.method = IB_MGMT_METHOD_GET; path_record->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC ); - path_record->mad_hdr.tid = tid++; + path_record->mad_hdr.tid = ipoib_meta_tid++; path_record->sa_hdr.comp_mask[1] = htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID ); memcpy ( &path_record->dgid, gid, sizeof ( path_record->dgid ) ); @@ -361,6 +371,65 @@ static int ipoib_get_path_record ( struct ipoib_device *ipoib, return 0; } +/** + * Transmit multicast group membership request + * + * @v ipoib IPoIB device + * @v gid Multicast GID + * @v join Join (rather than leave) group + * @ret rc Return status code + */ +static int ipoib_mc_member_record ( struct ipoib_device *ipoib, + struct ib_gid *gid, int join ) { + struct ib_device *ibdev = ipoib->ibdev; + struct io_buffer *iobuf; + struct ib_mad_mc_member_record *mc_member_record; + struct ib_address_vector av; + int rc; + + /* Allocate I/O buffer */ + iobuf = alloc_iob ( sizeof ( *mc_member_record ) ); + if ( ! iobuf ) + return -ENOMEM; + iob_put ( iobuf, sizeof ( *mc_member_record ) ); + mc_member_record = iobuf->data; + memset ( mc_member_record, 0, sizeof ( *mc_member_record ) ); + + /* Construct path record request */ + mc_member_record->mad_hdr.base_version = IB_MGMT_BASE_VERSION; + mc_member_record->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; + mc_member_record->mad_hdr.class_version = 2; + mc_member_record->mad_hdr.method = + ( join ? IB_MGMT_METHOD_SET : IB_MGMT_METHOD_DELETE ); + mc_member_record->mad_hdr.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC ); + mc_member_record->mad_hdr.tid = ipoib_meta_tid++; + mc_member_record->sa_hdr.comp_mask[1] = + htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | + IB_SA_MCMEMBER_REC_JOIN_STATE ); + mc_member_record->scope__join_state = 1; + memcpy ( &mc_member_record->mgid, gid, + sizeof ( mc_member_record->mgid ) ); + memcpy ( &mc_member_record->port_gid, &ibdev->port_gid, + sizeof ( mc_member_record->port_gid ) ); + + /* Construct address vector */ + memset ( &av, 0, sizeof ( av ) ); + av.dlid = ibdev->sm_lid; + av.dest_qp = IB_SA_QPN; + av.qkey = IB_GLOBAL_QKEY; + + /* Post send request */ + if ( ( rc = ib_post_send ( ibdev, ipoib->meta.qp, &av, + iobuf ) ) != 0 ) { + DBGC ( ipoib, "IPoIB %p could not send get path record: %s\n", + ipoib, strerror ( rc ) ); + free_iob ( iobuf ); + return rc; + } + + return 0; +} + /** * Transmit packet via IPoIB network device * @@ -591,7 +660,7 @@ static int ipoib_open ( struct net_device *netdev ) { /* Attach to broadcast multicast GID */ if ( ( rc = ib_mcast_attach ( ibdev, ipoib->data.qp, - &ibdev->broadcast_gid ) ) != 0 ) { + &ipoib->broadcast_gid ) ) != 0 ) { DBG ( "Could not attach to broadcast GID: %s\n", strerror ( rc ) ); return rc; @@ -628,6 +697,27 @@ static struct net_device_operations ipoib_operations = { .irq = ipoib_irq, }; +/** + * Join IPoIB broadcast group + * + * @v ipoib IPoIB device + * @ret rc Return status code + */ +int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) { + int rc; + + /* Send join request */ + if ( ( rc = ipoib_mc_member_record ( ipoib, &ipoib->broadcast_gid, + 1 ) ) != 0 ) { + DBGC ( ipoib, "IPoIB %p could not send broadcast join: %s\n", + ipoib, strerror ( rc ) ); + return rc; + } + + + return 0; +} + /** * Probe IPoIB device * @@ -652,6 +742,11 @@ int ipoib_probe ( struct ib_device *ibdev ) { ipoib->netdev = netdev; ipoib->ibdev = ibdev; + /* Calculate broadcast GID */ + memcpy ( &ipoib->broadcast_gid, &ipv4_broadcast_gid, + sizeof ( ipoib->broadcast_gid ) ); + ipoib->broadcast_gid.u.words[2] = htons ( ibdev->pkey ); + /* Allocate metadata queue set */ if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta, IPOIB_META_NUM_CQES, @@ -663,6 +758,8 @@ int ipoib_probe ( struct ib_device *ibdev ) { goto err_create_meta_qset; } + + /* Allocate data queue set */ if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->data, IPOIB_DATA_NUM_CQES, diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index 5f3c3c4d..920874cc 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -1251,25 +1251,13 @@ static int arbel_get_sm_lid ( struct arbel *arbel, return 0; } -static int arbel_get_broadcast_gid ( struct arbel *arbel, - struct ib_gid *broadcast_gid ) { - static const struct ib_gid ipv4_broadcast_gid = { - { { 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff } } - }; +static int arbel_get_pkey ( struct arbel *arbel, unsigned long *pkey ) { struct ib_mad_pkey_table pkey_table; int rc; - /* Start with the IPv4 broadcast GID */ - memcpy ( broadcast_gid, &ipv4_broadcast_gid, - sizeof ( *broadcast_gid ) ); - - /* Add partition key */ if ( ( rc = arbel_get_pkey_table ( arbel, &pkey_table ) ) != 0 ) return rc; - memcpy ( &broadcast_gid->u.bytes[4], &pkey_table.pkey[0][0], - sizeof ( pkey_table.pkey[0][0] ) ); - + *pkey = ntohs ( pkey_table.pkey[0][0] ); return 0; } @@ -1340,12 +1328,11 @@ static int arbel_probe ( struct pci_device *pci, goto err_get_port_gid; } - /* Get broadcast GID */ - if ( ( rc = arbel_get_broadcast_gid ( arbel, - &ibdev->broadcast_gid ) ) != 0 ){ - DBGC ( arbel, "Arbel %p could not determine broadcast GID: " + /* Get partition key */ + if ( ( rc = arbel_get_pkey ( arbel, &ibdev->pkey ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not determine partition key: " "%s\n", arbel, strerror ( rc ) ); - goto err_get_broadcast_gid; + goto err_get_pkey; } struct ud_av_st *bcast_av = ib_data.bcast_av; @@ -1370,7 +1357,7 @@ static int arbel_probe ( struct pci_device *pci, return 0; err_ipoib_probe: - err_get_broadcast_gid: + err_get_pkey: err_get_port_gid: err_get_sm_lid: err_query_dev_lim: diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index 845c4c22..8e358bce 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -20,6 +20,7 @@ struct ib_gid { union { uint8_t bytes[16]; + uint16_t words[8]; uint32_t dwords[4]; } u; }; @@ -258,10 +259,10 @@ struct ib_device_operations { struct ib_device { /** Port GID */ struct ib_gid port_gid; - /** Broadcast GID */ - struct ib_gid broadcast_gid; /** Subnet manager LID */ unsigned long sm_lid; + /** Partition key */ + unsigned int pkey; /** Underlying device */ struct device *dev; /** Infiniband operations */ @@ -541,6 +542,26 @@ struct ib_mad_path_record { uint32_t reserved2[35]; } __attribute__ (( packed )); +struct ib_mad_mc_member_record { + struct ib_mad_hdr mad_hdr; + struct ib_rmpp_hdr rmpp_hdr; + struct ib_sa_hdr sa_hdr; + struct ib_gid mgid; + struct ib_gid port_gid; + uint32_t qkey; + uint16_t mlid; + uint8_t mtu_selector__mtu; + uint8_t tclass; + uint16_t pkey; + uint8_t rate_selector__rate; + uint8_t packet_lifetime_selector__packet_lifetime; + uint32_t sl__flow_label__hop_limit; + uint8_t scope__join_state; + uint8_t proxy_join__reserved; + uint16_t reserved0; + uint32_t reserved1[37]; +} __attribute__ (( packed )); + union ib_mad { struct ib_mad_hdr mad_hdr; struct ib_mad_data data;