diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h index c4b536a5..28893f51 100644 --- a/src/drivers/net/mlx_ipoib/arbel.h +++ b/src/drivers/net/mlx_ipoib/arbel.h @@ -33,6 +33,7 @@ #define ARBEL_HCR_INIT2RTR_QPEE 0x001a #define ARBEL_HCR_RTR2RTS_QPEE 0x001b #define ARBEL_HCR_2RST_QPEE 0x0021 +#define ARBEL_HCR_MAD_IFC 0x0024 #define ARBEL_HCR_READ_MGM 0x0025 #define ARBEL_HCR_WRITE_MGM 0x0026 #define ARBEL_HCR_MGID_HASH 0x0027 @@ -67,6 +68,7 @@ struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error ); struct MLX_DECLARE_STRUCT ( arbelprm_cq_arm_db_record ); struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record ); struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register ); +struct MLX_DECLARE_STRUCT ( arbelprm_mad_ifc ); struct MLX_DECLARE_STRUCT ( arbelprm_mgm_entry ); struct MLX_DECLARE_STRUCT ( arbelprm_mgm_hash ); struct MLX_DECLARE_STRUCT ( arbelprm_qp_db_record ); @@ -126,6 +128,11 @@ union arbelprm_doorbell_register { uint32_t dword[2]; } __attribute__ (( packed )); +union arbelprm_mad { + struct arbelprm_mad_ifc ifc; + union ib_mad mad; +} __attribute__ (( packed )); + /* * gPXE-specific definitions * diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index 13b7d78b..aed6d208 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -549,6 +549,15 @@ arbel_cmd_2rst_qpee ( struct arbel *arbel, unsigned long qpn ) { 0x03, NULL, qpn, NULL ); } +static inline int +arbel_cmd_mad_ifc ( struct arbel *arbel, union arbelprm_mad *mad ) { + return arbel_cmd ( arbel, + ARBEL_HCR_INOUT_CMD ( ARBEL_HCR_MAD_IFC, + 1, sizeof ( *mad ), + 1, sizeof ( *mad ) ), + 0x03, mad, PXE_IB_PORT, mad ); +} + static inline int arbel_cmd_read_mgm ( struct arbel *arbel, unsigned int index, struct arbelprm_mgm_entry *mgm ) { @@ -1233,6 +1242,15 @@ static int arbel_complete ( struct ib_device *ibdev, return rc; } +/** + * Drain event queue + * + * @v arbel Arbel device + */ +static void arbel_drain_eq ( struct arbel *arbel ) { +#warning "drain the event queue" +} + /** * Poll completion queue * @@ -1252,6 +1270,9 @@ static void arbel_poll_cq ( struct ib_device *ibdev, unsigned int cqe_idx_mask; int rc; + /* Drain the event queue */ + arbel_drain_eq ( arbel ); + while ( 1 ) { /* Look for completion entry */ cqe_idx_mask = ( cq->num_cqes - 1 ); @@ -1377,8 +1398,6 @@ static void arbel_mcast_detach ( struct ib_device *ibdev, } } - - /** Arbel Infiniband operations */ static struct ib_device_operations arbel_ib_operations = { .create_cq = arbel_create_cq, @@ -1392,20 +1411,83 @@ static struct ib_device_operations arbel_ib_operations = { .mcast_detach = arbel_mcast_detach, }; -/** - * Remove PCI device - * - * @v pci PCI device - */ -static void arbel_remove ( struct pci_device *pci ) { - struct net_device *netdev = pci_get_drvdata ( pci ); - unregister_netdev ( netdev ); - ib_driver_close ( 0 ); - netdev_nullify ( netdev ); - netdev_put ( netdev ); +static int arbel_mad_ifc ( struct arbel *arbel, + union arbelprm_mad *mad ) { + struct ib_mad_hdr *hdr = &mad->mad.mad_hdr; + int rc; + + hdr->base_version = IB_MGMT_BASE_VERSION; + if ( ( rc = arbel_cmd_mad_ifc ( arbel, mad ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n", + arbel, strerror ( rc ) ); + return rc; + } + if ( hdr->status != 0 ) { + DBGC ( arbel, "Arbel %p MAD IFC status %04x\n", + arbel, ntohs ( hdr->status ) ); + return -EIO; + } + return 0; } +static int arbel_get_port_info ( struct arbel *arbel, + struct ib_mad_port_info *port_info ) { + union arbelprm_mad mad; + struct ib_mad_hdr *hdr = &mad.mad.mad_hdr; + int rc; + + memset ( &mad, 0, sizeof ( mad ) ); + hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + hdr->class_version = 1; + hdr->method = IB_MGMT_METHOD_GET; + hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO ); + hdr->attr_mod = htonl ( PXE_IB_PORT ); + if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not get port info: %s\n", + arbel, strerror ( rc ) ); + return rc; + } + memcpy ( port_info, &mad.mad.port_info, sizeof ( *port_info ) ); + return 0; +} + +static int arbel_get_guid_info ( struct arbel *arbel, + struct ib_mad_guid_info *guid_info ) { + union arbelprm_mad mad; + struct ib_mad_hdr *hdr = &mad.mad.mad_hdr; + int rc; + + memset ( &mad, 0, sizeof ( mad ) ); + hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + hdr->class_version = 1; + hdr->method = IB_MGMT_METHOD_GET; + hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO ); + if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not get GUID info: %s\n", + arbel, strerror ( rc ) ); + return rc; + } + memcpy ( guid_info, &mad.mad.guid_info, sizeof ( *guid_info ) ); + return 0; +} + +static int arbel_get_port_gid ( struct arbel *arbel, struct ib_gid *gid ) { + struct ib_mad_port_info port_info; + struct ib_mad_guid_info guid_info; + int rc; + + if ( ( rc = arbel_get_port_info ( arbel, &port_info ) ) != 0 ) + return rc; + if ( ( rc = arbel_get_guid_info ( arbel, &guid_info ) ) != 0 ) + return rc; + memcpy ( &gid->bytes[0], port_info.gid_prefix, 8 ); + memcpy ( &gid->bytes[8], guid_info.gid_local, 8 ); + return 0; +} + + + /** * Probe PCI device * @@ -1514,11 +1596,20 @@ static int arbel_probe ( struct pci_device *pci, strerror ( rc ) ); return rc; } - + + if ( ( rc = arbel_get_port_gid ( arbel, &ibdev->port_gid ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not determine port GID: %s\n", + arbel, strerror ( rc ) ); + return rc; + } + + DBG ( "Port GID:\n" ); + DBG_HD ( &ibdev->port_gid, sizeof ( ibdev->port_gid ) ); + mac = ( ( struct ib_mac * ) netdev->ll_addr ); mac->qpn = htonl ( mlx->own_qp->qpn ); - memcpy ( &mac->gid, ib_data.port_gid.raw, sizeof ( mac->gid ) ); + memcpy ( &mac->gid, &ibdev->port_gid, sizeof ( mac->gid ) ); #endif #if 0 @@ -1545,6 +1636,20 @@ static int arbel_probe ( struct pci_device *pci, return rc; } +/** + * Remove PCI device + * + * @v pci PCI device + */ +static void arbel_remove ( struct pci_device *pci ) { + struct net_device *netdev = pci_get_drvdata ( pci ); + + unregister_netdev ( netdev ); + ib_driver_close ( 0 ); + netdev_nullify ( netdev ); + netdev_put ( netdev ); +} + static struct pci_device_id arbel_nics[] = { PCI_ROM ( 0x15b3, 0x6282, "MT25218", "MT25218 HCA driver" ), PCI_ROM ( 0x15b3, 0x6274, "MT25204", "MT25204 HCA driver" ), diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index 6a38a1b8..3f09808c 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -277,6 +277,8 @@ struct ib_device_operations { /** An Infiniband device */ struct ib_device { + /** Port GID */ + struct ib_gid port_gid; /** Infiniband operations */ struct ib_device_operations *op; /** Device private data */ @@ -323,6 +325,119 @@ ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp, ibdev->op->mcast_detach ( ibdev, qp, gid ); } +/***************************************************************************** + * + * Management datagrams + * + * Portions Copyright (c) 2004 Mellanox Technologies Ltd. All rights + * reserved. + * + */ + +/* Management base version */ +#define IB_MGMT_BASE_VERSION 1 + +/* Management classes */ +#define IB_MGMT_CLASS_SUBN_LID_ROUTED 0x01 +#define IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE 0x81 +#define IB_MGMT_CLASS_SUBN_ADM 0x03 +#define IB_MGMT_CLASS_PERF_MGMT 0x04 +#define IB_MGMT_CLASS_BM 0x05 +#define IB_MGMT_CLASS_DEVICE_MGMT 0x06 +#define IB_MGMT_CLASS_CM 0x07 +#define IB_MGMT_CLASS_SNMP 0x08 +#define IB_MGMT_CLASS_VENDOR_RANGE2_START 0x30 +#define IB_MGMT_CLASS_VENDOR_RANGE2_END 0x4F + +/* Management methods */ +#define IB_MGMT_METHOD_GET 0x01 +#define IB_MGMT_METHOD_SET 0x02 +#define IB_MGMT_METHOD_GET_RESP 0x81 +#define IB_MGMT_METHOD_SEND 0x03 +#define IB_MGMT_METHOD_TRAP 0x05 +#define IB_MGMT_METHOD_REPORT 0x06 +#define IB_MGMT_METHOD_REPORT_RESP 0x86 +#define IB_MGMT_METHOD_TRAP_REPRESS 0x07 +#define IB_MGMT_METHOD_DELETE 0x15 +#define IB_MGMT_METHOD_RESP 0x80 + +/* Subnet management attributes */ +#define IB_SMP_ATTR_NOTICE 0x0002 +#define IB_SMP_ATTR_NODE_DESC 0x0010 +#define IB_SMP_ATTR_NODE_INFO 0x0011 +#define IB_SMP_ATTR_SWITCH_INFO 0x0012 +#define IB_SMP_ATTR_GUID_INFO 0x0014 +#define IB_SMP_ATTR_PORT_INFO 0x0015 +#define IB_SMP_ATTR_PKEY_TABLE 0x0016 +#define IB_SMP_ATTR_SL_TO_VL_TABLE 0x0017 +#define IB_SMP_ATTR_VL_ARB_TABLE 0x0018 +#define IB_SMP_ATTR_LINEAR_FORWARD_TABLE 0x0019 +#define IB_SMP_ATTR_RANDOM_FORWARD_TABLE 0x001A +#define IB_SMP_ATTR_MCAST_FORWARD_TABLE 0x001B +#define IB_SMP_ATTR_SM_INFO 0x0020 +#define IB_SMP_ATTR_VENDOR_DIAG 0x0030 +#define IB_SMP_ATTR_LED_INFO 0x0031 +#define IB_SMP_ATTR_VENDOR_MASK 0xFF00 + +struct ib_mad_hdr { + uint8_t base_version; + uint8_t mgmt_class; + uint8_t class_version; + uint8_t method; + uint16_t status; + uint16_t class_specific; + uint64_t tid; + uint16_t attr_id; + uint16_t resv; + uint32_t attr_mod; +} __attribute__ (( packed )); + +struct ib_mad_data { + struct ib_mad_hdr mad_hdr; + uint8_t data[232]; +} __attribute__ (( packed )); + +struct ib_mad_guid_info { + struct ib_mad_hdr mad_hdr; + uint32_t mkey[2]; + uint32_t reserved[8]; + uint8_t gid_local[8]; +} __attribute__ (( packed )); + +struct ib_mad_port_info { + struct ib_mad_hdr mad_hdr; + uint32_t mkey[2]; + uint32_t reserved[8]; + uint32_t mkey2[2]; + uint8_t gid_prefix[8]; + uint16_t lid; + uint16_t mastersm_lid; + uint32_t cap_mask; + uint16_t diag_code; + uint16_t mkey_lease_period; + uint8_t local_port_num; + uint8_t link_width_enabled; + uint8_t link_width_supported; + uint8_t link_width_active; + uint8_t port_state__link_speed_supported; + uint8_t link_down_def_state__port_phys_state; + uint8_t lmc__r1__mkey_prot_bits; + uint8_t link_speed_enabled__link_speed_active; +} __attribute__ (( packed )); + +union ib_mad { + struct ib_mad_hdr mad_hdr; + struct ib_mad_data data; + struct ib_mad_guid_info guid_info; + struct ib_mad_port_info port_info; +} __attribute__ (( packed )); + + + + + + + extern struct ll_protocol infiniband_protocol;