david/ipxe
david
/
ipxe
Archived
1
0
Fork 0
Code

[infiniband] Split subnet management agent client out into ib_smc.c 

Not all Infiniband cards have embedded subnet management agents.
Split out the code that communicates with such an embedded SMA into a
separate ib_smc.c file, and have drivers call ib_smc_update()
explicitly when they suspect that the answers given by the embedded
SMA may have changed.
This commit is contained in:
Michael Brown 2008-11-06 22:31:19 +00:00
parent 830e19eb54
commit 663904a7bc
9 changed files with 805 additions and 520 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
src/drivers/infiniband/arbel.c
View File

@ -33,6 +33,7 @@
#include <gpxe/iobuf.h>
#include <gpxe/netdevice.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_smc.h>
#include "arbel.h"
/**
@ -482,6 +483,50 @@ arbel_cmd_map_fa ( struct arbel *arbel,
0, map, 1, NULL );
}
/***************************************************************************
*
* MAD operations
*
***************************************************************************
*/
/**
* Issue management datagram
*
* @v ibdev Infiniband device
* @v mad Management datagram
* @ret rc Return status code
*/
static int arbel_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
struct arbel *arbel = ib_get_drvdata ( ibdev );
union arbelprm_mad mad_ifc;
int rc;
linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
mad_size_mismatch );
/* Copy in request packet */
memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
/* Issue MAD */
if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port,
&mad_ifc ) ) != 0 ) {
DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n",
arbel, strerror ( rc ) );
return rc;
}
/* Copy out reply packet */
memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) );
if ( mad->hdr.status != 0 ) {
DBGC ( arbel, "Arbel %p MAD IFC status %04x\n",
arbel, ntohs ( mad->hdr.status ) );
return -EIO;
}
return 0;
}
/***************************************************************************
*
* Completion queue operations
@ -1394,6 +1439,9 @@ static void arbel_event_port_state_change ( struct arbel *arbel,
return;
}
/* Update MAD parameters */
ib_smc_update ( arbel->ibdev[port], arbel_mad );
/* Notify Infiniband core of link state change */
ib_link_state_changed ( arbel->ibdev[port] );
}
@ -1483,6 +1531,9 @@ static int arbel_open ( struct ib_device *ibdev ) {
return rc;
}
/* Update MAD parameters */
ib_smc_update ( ibdev, arbel_mad );
return 0;
}
@ -1598,51 +1649,6 @@ static void arbel_mcast_detach ( struct ib_device *ibdev,
}
}
/***************************************************************************
*
* MAD operations
*
***************************************************************************
*/
/**
* Issue management datagram
*
* @v ibdev Infiniband device
* @v mad Management datagram
* @v len Length of management datagram
* @ret rc Return status code
*/
static int arbel_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
size_t len ) {
struct arbel *arbel = ib_get_drvdata ( ibdev );
union arbelprm_mad mad_ifc;
int rc;
/* Copy in request packet */
memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
assert ( len <= sizeof ( mad_ifc.mad ) );
memcpy ( &mad_ifc.mad, mad, len );
/* Issue MAD */
if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port,
&mad_ifc ) ) != 0 ) {
DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n",
arbel, strerror ( rc ) );
return rc;
}
/* Copy out reply packet */
memcpy ( mad, &mad_ifc.mad, len );
if ( mad->status != 0 ) {
DBGC ( arbel, "Arbel %p MAD IFC status %04x\n",
arbel, ntohs ( mad->status ) );
return -EIO;
}
return 0;
}
/** Arbel Infiniband operations */
static struct ib_device_operations arbel_ib_operations = {
.create_cq = arbel_create_cq,
@ -1658,7 +1664,6 @@ static struct ib_device_operations arbel_ib_operations = {
.close = arbel_close,
.mcast_attach = arbel_mcast_attach,
.mcast_detach = arbel_mcast_detach,
.mad = arbel_mad,
};
/***************************************************************************

97
src/drivers/infiniband/hermon.c
View File

@ -31,6 +31,7 @@
#include <gpxe/iobuf.h>
#include <gpxe/netdevice.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_smc.h>
#include "hermon.h"
/**
@ -608,6 +609,50 @@ static void hermon_free_mtt ( struct hermon *hermon,
mtt->num_pages );
}
/***************************************************************************
*
* MAD operations
*
***************************************************************************
*/
/**
* Issue management datagram
*
* @v ibdev Infiniband device
* @v mad Management datagram
* @ret rc Return status code
*/
static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
union hermonprm_mad mad_ifc;
int rc;
linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
mad_size_mismatch );
/* Copy in request packet */
memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
/* Issue MAD */
if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
&mad_ifc ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
hermon, strerror ( rc ) );
return rc;
}
/* Copy out reply packet */
memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) );
if ( mad->hdr.status != 0 ) {
DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
hermon, ntohs ( mad->hdr.status ) );
return -EIO;
}
return 0;
}
/***************************************************************************
*
* Completion queue operations
@ -1377,6 +1422,9 @@ static void hermon_event_port_state_change ( struct hermon *hermon,
return;
}
/* Update MAD parameters */
ib_smc_update ( hermon->ibdev[port], hermon_mad );
/* Notify Infiniband core of link state change */
ib_link_state_changed ( hermon->ibdev[port] );
}
@ -1465,6 +1513,9 @@ static int hermon_open ( struct ib_device *ibdev ) {
return rc;
}
/* Update MAD parameters */
ib_smc_update ( ibdev, hermon_mad );
return 0;
}
@ -1579,51 +1630,6 @@ static void hermon_mcast_detach ( struct ib_device *ibdev,
}
}
/***************************************************************************
*
* MAD operations
*
***************************************************************************
*/
/**
* Issue management datagram
*
* @v ibdev Infiniband device
* @v mad Management datagram
* @v len Length of management datagram
* @ret rc Return status code
*/
static int hermon_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
size_t len ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
union hermonprm_mad mad_ifc;
int rc;
/* Copy in request packet */
memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
assert ( len <= sizeof ( mad_ifc.mad ) );
memcpy ( &mad_ifc.mad, mad, len );
/* Issue MAD */
if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
&mad_ifc ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
hermon, strerror ( rc ) );
return rc;
}
/* Copy out reply packet */
memcpy ( mad, &mad_ifc.mad, len );
if ( mad->status != 0 ) {
DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
hermon, ntohs ( mad->status ) );
return -EIO;
}
return 0;
}
/** Hermon Infiniband operations */
static struct ib_device_operations hermon_ib_operations = {
.create_cq = hermon_create_cq,
@ -1639,7 +1645,6 @@ static struct ib_device_operations hermon_ib_operations = {
.close = hermon_close,
.mcast_attach = hermon_mcast_attach,
.mcast_detach = hermon_mcast_detach,
.mad = hermon_mad,
};
/***************************************************************************

166
src/drivers/infiniband/ib_smc.c Normal file
View File

@ -0,0 +1,166 @@
/*
* Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <byteswap.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_smc.h>
/**
* @file
*
* Infiniband Subnet Management Client
*
*/
/**
* Get port information
*
* @v ibdev Infiniband device
* @v local_mad Method for issuing local MADs
* @v mad Management datagram to fill in
* @ret rc Return status code
*/
static int ib_smc_get_port_info ( struct ib_device *ibdev,
ib_local_mad_t local_mad,
union ib_mad *mad ) {
int rc;
/* Construct MAD */
memset ( mad, 0, sizeof ( *mad ) );
mad->hdr.base_version = IB_MGMT_BASE_VERSION;
mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
mad->hdr.class_version = 1;
mad->hdr.method = IB_MGMT_METHOD_GET;
mad->hdr.attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
mad->hdr.attr_mod = htonl ( ibdev->port );
if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not get port info: %s\n",
ibdev, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Get GUID information
*
* @v ibdev Infiniband device
* @v local_mad Method for issuing local MADs
* @v mad Management datagram to fill in
* @ret rc Return status code
*/
static int ib_smc_get_guid_info ( struct ib_device *ibdev,
ib_local_mad_t local_mad,
union ib_mad *mad ) {
int rc;
/* Construct MAD */
memset ( mad, 0, sizeof ( *mad ) );
mad->hdr.base_version = IB_MGMT_BASE_VERSION;
mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
mad->hdr.class_version = 1;
mad->hdr.method = IB_MGMT_METHOD_GET;
mad->hdr.attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n",
ibdev, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Get partition key table
*
* @v ibdev Infiniband device
* @v local_mad Method for issuing local MADs
* @v mad Management datagram to fill in
* @ret rc Return status code
*/
static int ib_smc_get_pkey_table ( struct ib_device *ibdev,
ib_local_mad_t local_mad,
union ib_mad *mad ) {
int rc;
/* Construct MAD */
memset ( mad, 0, sizeof ( *mad ) );
mad->hdr.base_version = IB_MGMT_BASE_VERSION;
mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
mad->hdr.class_version = 1;
mad->hdr.method = IB_MGMT_METHOD_GET;
mad->hdr.attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n",
ibdev, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Get MAD parameters
*
* @v ibdev Infiniband device
* @v local_mad Method for issuing local MADs
* @ret rc Return status code
*/
int ib_smc_update ( struct ib_device *ibdev, ib_local_mad_t local_mad ) {
union ib_mad mad;
union ib_smp_data *smp = &mad.smp.smp_data;
int rc;
/* Port info gives us the link state, the first half of the
* port GID and the SM LID.
*/
if ( ( rc = ib_smc_get_port_info ( ibdev, local_mad, &mad ) ) != 0 )
return rc;
ibdev->port_state =
( smp->port_info.link_speed_supported__port_state & 0x0f );
memcpy ( &ibdev->gid.u.half[0], smp->port_info.gid_prefix,
sizeof ( ibdev->gid.u.half[0] ) );
ibdev->lid = ntohs ( smp->port_info.lid );
ibdev->sm_lid = ntohs ( smp->port_info.mastersm_lid );
ibdev->sm_sl = ( smp->port_info.neighbour_mtu__mastersm_sl & 0xf );
/* GUID info gives us the second half of the port GID */
if ( ( rc = ib_smc_get_guid_info ( ibdev, local_mad, &mad ) ) != 0 )
return rc;
memcpy ( &ibdev->gid.u.half[1], smp->guid_info.guid[0],
sizeof ( ibdev->gid.u.half[1] ) );
/* Get partition key */
if ( ( rc = ib_smc_get_pkey_table ( ibdev, local_mad, &mad ) ) != 0 )
return rc;
ibdev->pkey = ntohs ( smp->pkey_table.pkey[0] );
DBGC ( ibdev, "IBDEV %p port GID is %08lx:%08lx:%08lx:%08lx\n", ibdev,
htonl ( ibdev->gid.u.dwords[0] ),
htonl ( ibdev->gid.u.dwords[1] ),
htonl ( ibdev->gid.u.dwords[2] ),
htonl ( ibdev->gid.u.dwords[3] ) );
return 0;
}

18
src/drivers/net/ipoib.c
View File

@ -383,12 +383,13 @@ static int ipoib_get_path_record ( struct ipoib_device *ipoib,
path_record->sa_hdr.comp_mask[1] =
htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
memcpy ( &path_record->dgid, gid, sizeof ( path_record->dgid ) );
memcpy ( &path_record->sgid, &ibdev->port_gid,
memcpy ( &path_record->sgid, &ibdev->gid,
sizeof ( path_record->sgid ) );
/* Construct address vector */
memset ( &av, 0, sizeof ( av ) );
av.lid = ibdev->sm_lid;
av.sl = ibdev->sm_sl;
av.qpn = IB_SA_QPN;
av.qkey = IB_GLOBAL_QKEY;
@ -443,12 +444,13 @@ static int ipoib_mc_member_record ( struct ipoib_device *ipoib,
mc_member_record->scope__join_state = 1;
memcpy ( &mc_member_record->mgid, gid,
sizeof ( mc_member_record->mgid ) );
memcpy ( &mc_member_record->port_gid, &ibdev->port_gid,
memcpy ( &mc_member_record->port_gid, &ibdev->gid,
sizeof ( mc_member_record->port_gid ) );
/* Construct address vector */
memset ( &av, 0, sizeof ( av ) );
av.lid = ibdev->sm_lid;
av.sl = ibdev->sm_sl;
av.qpn = IB_SA_QPN;
av.qkey = IB_GLOBAL_QKEY;
@ -491,7 +493,7 @@ static int ipoib_transmit ( struct net_device *netdev,
/* Attempting transmission while link is down will put the
* queue pair into an error state, so don't try it.
*/
if ( ! ibdev->link_up )
if ( ! ib_link_ok ( ibdev ) )
return -ENETUNREACH;
/* Construct address vector */
@ -691,13 +693,13 @@ ipoib_meta_complete_recv ( struct ib_device *ibdev __unused,
}
mad = iobuf->data;
if ( mad->mad_hdr.status != 0 ) {
if ( mad->hdr.status != 0 ) {
DBGC ( ipoib, "IPoIB %p metadata RX err status %04x\n",
ipoib, ntohs ( mad->mad_hdr.status ) );
ipoib, ntohs ( mad->hdr.status ) );
goto done;
}
switch ( mad->mad_hdr.tid[0] ) {
switch ( mad->hdr.tid[0] ) {
case IPOIB_TID_GET_PATH_REC:
ipoib_recv_path_record ( ipoib, &mad->path_record );
break;
@ -928,7 +930,7 @@ static void ipoib_set_ib_params ( struct ipoib_device *ipoib ) {
/* Calculate GID portion of MAC address based on port GID */
mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
memcpy ( &mac->gid, &ibdev->port_gid, sizeof ( mac->gid ) );
memcpy ( &mac->gid, &ibdev->gid, sizeof ( mac->gid ) );
/* Calculate broadcast GID based on partition key */
memcpy ( &ipoib->broadcast_gid, &ipv4_broadcast_gid,
@ -936,7 +938,7 @@ static void ipoib_set_ib_params ( struct ipoib_device *ipoib ) {
ipoib->broadcast_gid.u.words[2] = htons ( ibdev->pkey );
/* Set net device link state to reflect Infiniband link state */
if ( ibdev->link_up ) {
if ( ib_link_ok ( ibdev ) ) {
netdev_link_up ( netdev );
} else {
netdev_link_down ( netdev );

341
src/include/gpxe/ib_mad.h Normal file
View File

@ -0,0 +1,341 @@
#ifndef _GPXE_IB_MAD_H
#define _GPXE_IB_MAD_H
/** @file
*
* Infiniband management datagrams
*
*/
#include <stdint.h>
#include <gpxe/ib_packet.h>
/** A management datagram common header
*
* Defined in section 13.4.2 of the IBA.
*/
struct ib_mad_hdr {
uint8_t base_version;
uint8_t mgmt_class;
uint8_t class_version;
uint8_t method;
uint16_t status;
uint16_t class_specific;
uint32_t tid[2];
uint16_t attr_id;
uint8_t reserved[2];
uint32_t attr_mod;
} __attribute__ (( packed ));
/* Management base version */
#define IB_MGMT_BASE_VERSION 1
/* Management classes */
#define IB_MGMT_CLASS_SUBN_LID_ROUTED 0x01
#define IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE 0x81
#define IB_MGMT_CLASS_SUBN_ADM 0x03
#define IB_MGMT_CLASS_PERF_MGMT 0x04
#define IB_MGMT_CLASS_BM 0x05
#define IB_MGMT_CLASS_DEVICE_MGMT 0x06
#define IB_MGMT_CLASS_CM 0x07
#define IB_MGMT_CLASS_SNMP 0x08
#define IB_MGMT_CLASS_VENDOR_RANGE2_START 0x30
#define IB_MGMT_CLASS_VENDOR_RANGE2_END 0x4F
/* Management methods */
#define IB_MGMT_METHOD_GET 0x01
#define IB_MGMT_METHOD_SET 0x02
#define IB_MGMT_METHOD_GET_RESP 0x81
#define IB_MGMT_METHOD_SEND 0x03
#define IB_MGMT_METHOD_TRAP 0x05
#define IB_MGMT_METHOD_REPORT 0x06
#define IB_MGMT_METHOD_REPORT_RESP 0x86
#define IB_MGMT_METHOD_TRAP_REPRESS 0x07
#define IB_MGMT_METHOD_DELETE 0x15
/* Status codes */
#define IB_MGMT_STATUS_OK 0x0000
#define IB_MGMT_STATUS_BAD_VERSION 0x0001
#define IB_MGMT_STATUS_UNSUPPORTED_METHOD 0x0002
#define IB_MGMT_STATUS_UNSUPPORTED_METHOD_ATTR 0x0003
#define IB_MGMT_STATUS_INVALID_VALUE 0x0004
/** A LID routed SMP header
*
* Defined in section 14.2.1.1 of the IBA.
*/
struct ib_smp_lr_hdr {
uint64_t mkey;
uint8_t reserved[32];
} __attribute__ (( packed ));
/** A directed route SMP header
*
* Defined in section 14.2.1.2 of the IBA.
*/
struct ib_smp_dr_hdr {
uint64_t mkey;
uint16_t slid;
uint16_t dlid;
uint8_t reserved[28];
} __attribute__ (( packed ));
/** A subnet management header */
union ib_smp_hdr {
uint64_t mkey;
struct ib_smp_lr_hdr lr;
struct ib_smp_dr_hdr dr;
} __attribute__ (( packed ));
/** Subnet management class version */
#define IB_SMP_CLASS_VERSION 1
/** Subnet management direction bit
*
* This bit resides in the "status" field in the MAD header.
*/
#define IB_SMP_STATUS_D_INBOUND 0x8000
/* Subnet management attributes */
#define IB_SMP_ATTR_NOTICE 0x0002
#define IB_SMP_ATTR_NODE_DESC 0x0010
#define IB_SMP_ATTR_NODE_INFO 0x0011
#define IB_SMP_ATTR_SWITCH_INFO 0x0012
#define IB_SMP_ATTR_GUID_INFO 0x0014
#define IB_SMP_ATTR_PORT_INFO 0x0015
#define IB_SMP_ATTR_PKEY_TABLE 0x0016
#define IB_SMP_ATTR_SL_TO_VL_TABLE 0x0017
#define IB_SMP_ATTR_VL_ARB_TABLE 0x0018
#define IB_SMP_ATTR_LINEAR_FORWARD_TABLE 0x0019
#define IB_SMP_ATTR_RANDOM_FORWARD_TABLE 0x001A
#define IB_SMP_ATTR_MCAST_FORWARD_TABLE 0x001B
#define IB_SMP_ATTR_SM_INFO 0x0020
#define IB_SMP_ATTR_VENDOR_DIAG 0x0030
#define IB_SMP_ATTR_LED_INFO 0x0031
#define IB_SMP_ATTR_VENDOR_MASK 0xFF00
/**
* A Node Description attribute
*
* Defined in section 14.2.5.2 of the IBA
*/
struct ib_node_desc {
char node_string[64];
} __attribute__ (( packed ));
/** A Node Information attribute
*
* Defined in section 14.2.5.3 of the IBA.
*/
struct ib_node_info {
uint8_t base_version;
uint8_t class_version;
uint8_t node_type;
uint8_t num_ports;
uint8_t sys_guid[8];
uint8_t node_guid[8];
uint8_t port_guid[8];
uint16_t partition_cap;
uint16_t device_id;
uint32_t revision;
uint8_t local_port_num;
uint8_t vendor_id[3];
} __attribute__ ((packed));
#define IB_NODE_TYPE_HCA 0x01
#define IB_NODE_TYPE_SWITCH 0x02
#define IB_NODE_TYPE_ROUTER 0x03
/** A GUID Information attribute
*
* Defined in section 14.2.5.5 of the IBA.
*/
struct ib_guid_info {
uint8_t guid[8][8];
} __attribute__ (( packed ));
/** A Port Information attribute
*
* Defined in section 14.2.5.6 of the IBA.
*/
struct ib_port_info {
uint64_t mkey;
uint8_t gid_prefix[8];
uint16_t lid;
uint16_t mastersm_lid;
uint32_t cap_mask;
uint16_t diag_code;
uint16_t mkey_lease_period;
uint8_t local_port_num;
uint8_t link_width_enabled;
uint8_t link_width_supported;
uint8_t link_width_active;
uint8_t link_speed_supported__port_state;
uint8_t port_phys_state__link_down_def_state;
uint8_t mkey_prot_bits__lmc;
uint8_t link_speed_active__link_speed_enabled;
uint8_t neighbour_mtu__mastersm_sl;
uint8_t vl_cap__init_type;
uint8_t vl_high_limit;
uint8_t vl_arbitration_high_cap;
uint8_t vl_arbitration_low_cap;
uint8_t init_type_reply__mtu_cap;
uint8_t vl_stall_count__hoq_life;
uint8_t operational_vls__enforcement;
uint16_t mkey_violations;
uint16_t pkey_violations;
uint16_t qkey_violations;
uint8_t guid_cap;
uint8_t client_reregister__subnet_timeout;
uint8_t resp_time_value;
uint8_t local_phy_errors__overrun_errors;
uint16_t max_credit_hint;
uint32_t link_round_trip_latency;
} __attribute__ (( packed ));
#define IB_LINK_WIDTH_1X 0x01
#define IB_LINK_WIDTH_4X 0x02
#define IB_LINK_WIDTH_8X 0x04
#define IB_LINK_WIDTH_12X 0x08
#define IB_LINK_SPEED_SDR 0x01
#define IB_LINK_SPEED_DDR 0x02
#define IB_LINK_SPEED_QDR 0x04
#define IB_PORT_STATE_DOWN 0x01
#define IB_PORT_STATE_INIT 0x02
#define IB_PORT_STATE_ARMED 0x03
#define IB_PORT_STATE_ACTIVE 0x04
#define IB_PORT_PHYS_STATE_SLEEP 0x01
#define IB_PORT_PHYS_STATE_POLLING 0x02
#define IB_MTU_256 0x01
#define IB_MTU_512 0x02
#define IB_MTU_1024 0x03
#define IB_MTU_2048 0x04
#define IB_MTU_4096 0x05
#define IB_VL_0 0x01
#define IB_VL_0_1 0x02
#define IB_VL_0_3 0x03
#define IB_VL_0_7 0x04
#define IB_VL_0_14 0x05
/** A Partition Key Table attribute
*
* Defined in section 14.2.5.7 of the IBA.
*/
struct ib_pkey_table {
uint16_t pkey[32];
} __attribute__ (( packed ));
/** A subnet management attribute */
union ib_smp_data {
struct ib_node_desc node_desc;
struct ib_node_info node_info;
struct ib_guid_info guid_info;
struct ib_port_info port_info;
struct ib_pkey_table pkey_table;
uint8_t bytes[64];
} __attribute__ (( packed ));
/** A subnet management directed route path */
struct ib_smp_dr_path {
uint8_t reserved;
uint8_t hops[63];
} __attribute__ (( packed ));
/** A subnet management MAD */
struct ib_mad_smp {
struct ib_mad_hdr mad_hdr;
union ib_smp_hdr smp_hdr;
union ib_smp_data smp_data;
struct ib_smp_dr_path initial_path;
struct ib_smp_dr_path return_path;
} __attribute__ (( packed ));
struct ib_sa_hdr {
uint32_t sm_key[2];
uint16_t reserved;
uint16_t attrib_offset;
uint32_t comp_mask[2];
} __attribute__ (( packed ));
struct ib_rmpp_hdr {
uint32_t raw[3];
} __attribute__ (( packed ));
struct ib_mad_path_record {
struct ib_mad_hdr mad_hdr;
struct ib_rmpp_hdr rmpp_hdr;
struct ib_sa_hdr sa_hdr;
uint32_t reserved0[2];
struct ib_gid dgid;
struct ib_gid sgid;
uint16_t dlid;
uint16_t slid;
uint32_t hop_limit__flow_label__raw_traffic;
uint32_t pkey__numb_path__reversible__tclass;
uint8_t reserved1;
uint8_t reserved__sl;
uint8_t mtu_selector__mtu;
uint8_t rate_selector__rate;
uint32_t preference__packet_lifetime__packet_lifetime_selector;
uint32_t reserved2[35];
} __attribute__ (( packed ));
struct ib_mad_mc_member_record {
struct ib_mad_hdr mad_hdr;
struct ib_rmpp_hdr rmpp_hdr;
struct ib_sa_hdr sa_hdr;
struct ib_gid mgid;
struct ib_gid port_gid;
uint32_t qkey;
uint16_t mlid;
uint8_t mtu_selector__mtu;
uint8_t tclass;
uint16_t pkey;
uint8_t rate_selector__rate;
uint8_t packet_lifetime_selector__packet_lifetime;
uint32_t sl__flow_label__hop_limit;
uint8_t scope__join_state;
uint8_t proxy_join__reserved;
uint16_t reserved0;
uint32_t reserved1[37];
} __attribute__ (( packed ));
#define IB_SA_ATTR_MC_MEMBER_REC 0x38
#define IB_SA_ATTR_PATH_REC 0x35
#define IB_SA_MCMEMBER_REC_MGID (1<<0)
#define IB_SA_MCMEMBER_REC_PORT_GID (1<<1)
#define IB_SA_MCMEMBER_REC_QKEY (1<<2)
#define IB_SA_MCMEMBER_REC_MLID (1<<3)
#define IB_SA_MCMEMBER_REC_MTU_SELECTOR (1<<4)
#define IB_SA_MCMEMBER_REC_MTU (1<<5)
#define IB_SA_MCMEMBER_REC_TRAFFIC_CLASS (1<<6)
#define IB_SA_MCMEMBER_REC_PKEY (1<<7)
#define IB_SA_MCMEMBER_REC_RATE_SELECTOR (1<<8)
#define IB_SA_MCMEMBER_REC_RATE (1<<9)
#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR (1<<10)
#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME (1<<11)
#define IB_SA_MCMEMBER_REC_SL (1<<12)
#define IB_SA_MCMEMBER_REC_FLOW_LABEL (1<<13)
#define IB_SA_MCMEMBER_REC_HOP_LIMIT (1<<14)
#define IB_SA_MCMEMBER_REC_SCOPE (1<<15)
#define IB_SA_MCMEMBER_REC_JOIN_STATE (1<<16)
#define IB_SA_MCMEMBER_REC_PROXY_JOIN (1<<17)
#define IB_SA_PATH_REC_DGID (1<<2)
#define IB_SA_PATH_REC_SGID (1<<3)
union ib_mad {
struct ib_mad_hdr hdr;
struct ib_mad_smp smp;
struct ib_mad_path_record path_record;
struct ib_mad_mc_member_record mc_member_record;
uint8_t bytes[256];
} __attribute__ (( packed ));
#endif /* _GPXE_IB_MAD_H */

114
src/include/gpxe/ib_packet.h Normal file
View File

@ -0,0 +1,114 @@
#ifndef _GPXE_IB_PACKET_H
#define _GPXE_IB_PACKET_H
/** @file
*
* Infiniband packet format
*
*/
/** Half of an Infiniband Global Identifier */
struct ib_gid_half {
uint8_t bytes[8];
};
/** An Infiniband Global Identifier */
struct ib_gid {
union {
uint8_t bytes[16];
uint16_t words[8];
uint32_t dwords[4];
struct ib_gid_half half[2];
} u;
};
/** An Infiniband Local Route Header */
struct ib_local_route_header {
/** Virtual lane and link version */
uint8_t vl__lver;
/** Service level and next link header */
uint8_t sl__lnh;
/** Destination LID */
uint16_t dlid;
/** Packet length */
uint16_t length;
/** Source LID */
uint16_t slid;
} __attribute__ (( packed ));
/** Infiniband virtual lanes */
enum ib_vl {
IB_VL_DEFAULT = 0,
IB_VL_SMP = 15,
};
/** An Infiniband Link Next Header value */
enum ib_lnh {
IB_LNH_RAW = 0,
IB_LNH_IPv6 = 1,
IB_LNH_BTH = 2,
IB_LNH_GRH = 3
};
/** Default Infiniband LID */
#define IB_LID_NONE 0xffff
/** An Infiniband Global Route Header */
struct ib_global_route_header {
/** IP version, traffic class, and flow label
*
* 4 bits : Version of the GRH
* 8 bits : Traffic class
* 20 bits : Flow label
*/
uint32_t ipver__tclass__flowlabel;
/** Payload length */
uint16_t paylen;
/** Next header */
uint8_t nxthdr;
/** Hop limit */
uint8_t hoplmt;
/** Source GID */
struct ib_gid sgid;
/** Destiniation GID */
struct ib_gid dgid;
} __attribute__ (( packed ));
#define IB_GRH_IPVER_IPv6 0x06
#define IB_GRH_NXTHDR_IBA 0x1b
#define IB_GRH_HOPLMT_MAX 0xff
/** An Infiniband Base Transport Header */
struct ib_base_transport_header {
/** Opcode */
uint8_t opcode;
/** Transport header version, pad count, migration and solicitation */
uint8_t se__m__padcnt__tver;
/** Partition key */
uint16_t pkey;
/** Destination queue pair */
uint32_t dest_qp;
/** Packet sequence number and acknowledge request */
uint32_t ack__psn;
} __attribute__ (( packed ));
/** An Infiniband BTH opcode */
enum ib_bth_opcode {
BTH_OPCODE_UD_SEND = 0x64,
};
/** Default Infiniband partition key */
#define IB_PKEY_NONE 0xffff
/** Subnet management queue pair number */
#define IB_QPN_SMP 0
/** An Infiniband Datagram Extended Transport Header */
struct ib_datagram_extended_transport_header {
/** Queue key */
uint32_t qkey;
/** Source queue pair */
uint32_t src_qp;
} __attribute__ (( packed ));
#endif /* _GPXE_IB_PACKET_H */

18
src/include/gpxe/ib_smc.h Normal file
View File

@ -0,0 +1,18 @@
#ifndef _GPXE_IB_SMC_H
#define _GPXE_IB_SMC_H
/** @file
*
* Infiniband Subnet Management Client
*
*/
#include <gpxe/infiniband.h>
typedef int ( * ib_local_mad_t ) ( struct ib_device *ibdev,
union ib_mad *mad );
extern int ib_smc_update ( struct ib_device *ibdev,
ib_local_mad_t local_mad );
#endif /* _GPXE_IB_SMC_H */

323
src/include/gpxe/infiniband.h
View File

@ -10,6 +10,8 @@
#include <stdint.h>
#include <gpxe/refcnt.h>
#include <gpxe/device.h>
#include <gpxe/ib_packet.h>
#include <gpxe/ib_mad.h>
/** Subnet administrator QPN */
#define IB_SA_QPN 1
@ -20,36 +22,6 @@
/** Subnet administrator queue key */
#define IB_GLOBAL_QKEY 0x80010000UL
/** An Infiniband Global Identifier */
struct ib_gid {
union {
uint8_t bytes[16];
uint16_t words[8];
uint32_t dwords[4];
} u;
};
/** An Infiniband Global Route Header */
struct ib_global_route_header {
/** IP version, traffic class, and flow label
*
* 4 bits : Version of the GRH
* 8 bits : Traffic class
* 20 bits : Flow label
*/
uint32_t ipver_tclass_flowlabel;
/** Payload length */
uint16_t paylen;
/** Next header */
uint8_t nxthdr;
/** Hop limit */
uint8_t hoplmt;
/** Source GID */
struct ib_gid sgid;
/** Destiniation GID */
struct ib_gid dgid;
} __attribute__ (( packed ));
struct ib_device;
struct ib_queue_pair;
struct ib_address_vector;
@ -178,8 +150,6 @@ struct ib_completion_queue {
void *drv_priv;
};
struct ib_mad_hdr;
/**
* Infiniband device operations
*
@ -306,16 +276,6 @@ struct ib_device_operations {
void ( * mcast_detach ) ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_gid *gid );
/**
* Issue management datagram
*
* @v ibdev Infiniband device
* @v mad Management datagram
* @v len Length of management datagram
* @ret rc Return status code
*/
int ( * mad ) ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
size_t len );
};
/** An Infiniband device */
@ -330,14 +290,24 @@ struct ib_device {
struct ib_device_operations *op;
/** Port number */
unsigned int port;
/** Link state */
int link_up;
/** Port state */
uint8_t port_state;
/** Link width */
uint8_t link_width;
/** Link speed */
uint8_t link_speed;
/** Port GID */
struct ib_gid port_gid;
struct ib_gid gid;
/** Port LID */
uint16_t lid;
/** Subnet manager LID */
unsigned long sm_lid;
uint16_t sm_lid;
/** Subnet manager SL */
uint8_t sm_sl;
/** Partition key */
unsigned int pkey;
uint16_t pkey;
/** Driver private data */
void *drv_priv;
/** Owner private data */
@ -375,6 +345,11 @@ extern struct ib_device * alloc_ibdev ( size_t priv_size );
extern int register_ibdev ( struct ib_device *ibdev );
extern void unregister_ibdev ( struct ib_device *ibdev );
extern void ib_link_state_changed ( struct ib_device *ibdev );
extern struct list_head ib_devices;
/** Iterate over all network devices */
#define for_each_ibdev( ibdev ) \
list_for_each_entry ( (ibdev), &ib_devices, list )
/**
* Poll completion queue
@ -382,7 +357,7 @@ extern void ib_link_state_changed ( struct ib_device *ibdev );
* @v ibdev Infiniband device
* @v cq Completion queue
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) {
ibdev->op->poll_cq ( ibdev, cq );
}
@ -393,7 +368,7 @@ ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) {
* @v ibdev Infiniband device
* @ret rc Return status code
*/
static inline __attribute__ (( always_inline )) int
static inline __always_inline int
ib_open ( struct ib_device *ibdev ) {
return ibdev->op->open ( ibdev );
}
@ -403,11 +378,22 @@ ib_open ( struct ib_device *ibdev ) {
*
* @v ibdev Infiniband device
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ib_close ( struct ib_device *ibdev ) {
ibdev->op->close ( ibdev );
}
/**
* Check link state
*
* @v ibdev Infiniband device
* @ret link_up Link is up
*/
static inline __always_inline int
ib_link_ok ( struct ib_device *ibdev ) {
return ( ibdev->port_state == IB_PORT_STATE_ACTIVE );
}
/**
* Attach to multicast group
*
@ -416,7 +402,7 @@ ib_close ( struct ib_device *ibdev ) {
* @v gid Multicast GID
* @ret rc Return status code
*/
static inline __attribute__ (( always_inline )) int
static inline __always_inline int
ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *gid ) {
return ibdev->op->mcast_attach ( ibdev, qp, gid );
@ -429,32 +415,19 @@ ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
* @v qp Queue pair
* @v gid Multicast GID
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *gid ) {
ibdev->op->mcast_detach ( ibdev, qp, gid );
}
/**
* Issue management datagram
*
* @v ibdev Infiniband device
* @v mad Management datagram
* @v len Length of management datagram
* @ret rc Return status code
*/
static inline __attribute__ (( always_inline )) int
ib_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad, size_t len ) {
return ibdev->op->mad ( ibdev, mad, len );
}
/**
* Get reference to Infiniband device
*
* @v ibdev Infiniband device
* @ret ibdev Infiniband device
*/
static inline __attribute__ (( always_inline )) struct ib_device *
static inline __always_inline struct ib_device *
ibdev_get ( struct ib_device *ibdev ) {
ref_get ( &ibdev->refcnt );
return ibdev;
@ -465,7 +438,7 @@ ibdev_get ( struct ib_device *ibdev ) {
*
* @v ibdev Infiniband device
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ibdev_put ( struct ib_device *ibdev ) {
ref_put ( &ibdev->refcnt );
}
@ -476,7 +449,7 @@ ibdev_put ( struct ib_device *ibdev ) {
* @v wq Work queue
* @v priv Private data
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ib_wq_set_drvdata ( struct ib_work_queue *wq, void *priv ) {
wq->drv_priv = priv;
}
@ -487,7 +460,7 @@ ib_wq_set_drvdata ( struct ib_work_queue *wq, void *priv ) {
* @v wq Work queue
* @ret priv Private data
*/
static inline __attribute__ (( always_inline )) void *
static inline __always_inline void *
ib_wq_get_drvdata ( struct ib_work_queue *wq ) {
return wq->drv_priv;
}
@ -498,7 +471,7 @@ ib_wq_get_drvdata ( struct ib_work_queue *wq ) {
* @v qp Queue pair
* @v priv Private data
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ib_qp_set_drvdata ( struct ib_queue_pair *qp, void *priv ) {
qp->drv_priv = priv;
}
@ -509,7 +482,7 @@ ib_qp_set_drvdata ( struct ib_queue_pair *qp, void *priv ) {
* @v qp Queue pair
* @ret priv Private data
*/
static inline __attribute__ (( always_inline )) void *
static inline __always_inline void *
ib_qp_get_drvdata ( struct ib_queue_pair *qp ) {
return qp->drv_priv;
}
@ -520,7 +493,7 @@ ib_qp_get_drvdata ( struct ib_queue_pair *qp ) {
* @v qp Queue pair
* @v priv Private data
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ib_qp_set_ownerdata ( struct ib_queue_pair *qp, void *priv ) {
qp->owner_priv = priv;
}
@ -531,7 +504,7 @@ ib_qp_set_ownerdata ( struct ib_queue_pair *qp, void *priv ) {
* @v qp Queue pair
* @ret priv Private data
*/
static inline __attribute__ (( always_inline )) void *
static inline __always_inline void *
ib_qp_get_ownerdata ( struct ib_queue_pair *qp ) {
return qp->owner_priv;
}
@ -542,7 +515,7 @@ ib_qp_get_ownerdata ( struct ib_queue_pair *qp ) {
* @v cq Completion queue
* @v priv Private data
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ib_cq_set_drvdata ( struct ib_completion_queue *cq, void *priv ) {
cq->drv_priv = priv;
}
@ -553,7 +526,7 @@ ib_cq_set_drvdata ( struct ib_completion_queue *cq, void *priv ) {
* @v cq Completion queue
* @ret priv Private data
*/
static inline __attribute__ (( always_inline )) void *
static inline __always_inline void *
ib_cq_get_drvdata ( struct ib_completion_queue *cq ) {
return cq->drv_priv;
}
@ -564,7 +537,7 @@ ib_cq_get_drvdata ( struct ib_completion_queue *cq ) {
* @v ibdev Infiniband device
* @v priv Private data
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ib_set_drvdata ( struct ib_device *ibdev, void *priv ) {
ibdev->drv_priv = priv;
}
@ -575,7 +548,7 @@ ib_set_drvdata ( struct ib_device *ibdev, void *priv ) {
* @v ibdev Infiniband device
* @ret priv Private data
*/
static inline __attribute__ (( always_inline )) void *
static inline __always_inline void *
ib_get_drvdata ( struct ib_device *ibdev ) {
return ibdev->drv_priv;
}
@ -586,7 +559,7 @@ ib_get_drvdata ( struct ib_device *ibdev ) {
* @v ibdev Infiniband device
* @v priv Private data
*/
static inline __attribute__ (( always_inline )) void
static inline __always_inline void
ib_set_ownerdata ( struct ib_device *ibdev, void *priv ) {
ibdev->owner_priv = priv;
}
@ -597,201 +570,9 @@ ib_set_ownerdata ( struct ib_device *ibdev, void *priv ) {
* @v ibdev Infiniband device
* @ret priv Private data
*/
static inline __attribute__ (( always_inline )) void *
static inline __always_inline void *
ib_get_ownerdata ( struct ib_device *ibdev ) {
return ibdev->owner_priv;
}
/*****************************************************************************
*
* Management datagrams
*
* Portions Copyright (c) 2004 Mellanox Technologies Ltd. All rights
* reserved.
*
*/
/* Management base version */
#define IB_MGMT_BASE_VERSION 1
/* Management classes */
#define IB_MGMT_CLASS_SUBN_LID_ROUTED 0x01
#define IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE 0x81
#define IB_MGMT_CLASS_SUBN_ADM 0x03
#define IB_MGMT_CLASS_PERF_MGMT 0x04
#define IB_MGMT_CLASS_BM 0x05
#define IB_MGMT_CLASS_DEVICE_MGMT 0x06
#define IB_MGMT_CLASS_CM 0x07
#define IB_MGMT_CLASS_SNMP 0x08
#define IB_MGMT_CLASS_VENDOR_RANGE2_START 0x30
#define IB_MGMT_CLASS_VENDOR_RANGE2_END 0x4F
/* Management methods */
#define IB_MGMT_METHOD_GET 0x01
#define IB_MGMT_METHOD_SET 0x02
#define IB_MGMT_METHOD_GET_RESP 0x81
#define IB_MGMT_METHOD_SEND 0x03
#define IB_MGMT_METHOD_TRAP 0x05
#define IB_MGMT_METHOD_REPORT 0x06
#define IB_MGMT_METHOD_REPORT_RESP 0x86
#define IB_MGMT_METHOD_TRAP_REPRESS 0x07
#define IB_MGMT_METHOD_DELETE 0x15
#define IB_MGMT_METHOD_RESP 0x80
/* Subnet management attributes */
#define IB_SMP_ATTR_NOTICE 0x0002
#define IB_SMP_ATTR_NODE_DESC 0x0010
#define IB_SMP_ATTR_NODE_INFO 0x0011
#define IB_SMP_ATTR_SWITCH_INFO 0x0012
#define IB_SMP_ATTR_GUID_INFO 0x0014
#define IB_SMP_ATTR_PORT_INFO 0x0015
#define IB_SMP_ATTR_PKEY_TABLE 0x0016
#define IB_SMP_ATTR_SL_TO_VL_TABLE 0x0017
#define IB_SMP_ATTR_VL_ARB_TABLE 0x0018
#define IB_SMP_ATTR_LINEAR_FORWARD_TABLE 0x0019
#define IB_SMP_ATTR_RANDOM_FORWARD_TABLE 0x001A
#define IB_SMP_ATTR_MCAST_FORWARD_TABLE 0x001B
#define IB_SMP_ATTR_SM_INFO 0x0020
#define IB_SMP_ATTR_VENDOR_DIAG 0x0030
#define IB_SMP_ATTR_LED_INFO 0x0031
#define IB_SMP_ATTR_VENDOR_MASK 0xFF00
#define IB_SA_ATTR_MC_MEMBER_REC 0x38
#define IB_SA_ATTR_PATH_REC 0x35
#define IB_SA_MCMEMBER_REC_MGID (1<<0)
#define IB_SA_MCMEMBER_REC_PORT_GID (1<<1)
#define IB_SA_MCMEMBER_REC_QKEY (1<<2)
#define IB_SA_MCMEMBER_REC_MLID (1<<3)
#define IB_SA_MCMEMBER_REC_MTU_SELECTOR (1<<4)
#define IB_SA_MCMEMBER_REC_MTU (1<<5)
#define IB_SA_MCMEMBER_REC_TRAFFIC_CLASS (1<<6)
#define IB_SA_MCMEMBER_REC_PKEY (1<<7)
#define IB_SA_MCMEMBER_REC_RATE_SELECTOR (1<<8)
#define IB_SA_MCMEMBER_REC_RATE (1<<9)
#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR (1<<10)
#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME (1<<11)
#define IB_SA_MCMEMBER_REC_SL (1<<12)
#define IB_SA_MCMEMBER_REC_FLOW_LABEL (1<<13)
#define IB_SA_MCMEMBER_REC_HOP_LIMIT (1<<14)
#define IB_SA_MCMEMBER_REC_SCOPE (1<<15)
#define IB_SA_MCMEMBER_REC_JOIN_STATE (1<<16)
#define IB_SA_MCMEMBER_REC_PROXY_JOIN (1<<17)
#define IB_SA_PATH_REC_DGID (1<<2)
#define IB_SA_PATH_REC_SGID (1<<3)
struct ib_mad_hdr {
uint8_t base_version;
uint8_t mgmt_class;
uint8_t class_version;
uint8_t method;
uint16_t status;
uint16_t class_specific;
uint32_t tid[2];
uint16_t attr_id;
uint16_t resv;
uint32_t attr_mod;
} __attribute__ (( packed ));
struct ib_sa_hdr {
uint32_t sm_key[2];
uint16_t reserved;
uint16_t attrib_offset;
uint32_t comp_mask[2];
} __attribute__ (( packed ));
struct ib_rmpp_hdr {
uint32_t raw[3];
} __attribute__ (( packed ));
struct ib_mad_data {
struct ib_mad_hdr mad_hdr;
uint8_t data[232];
} __attribute__ (( packed ));
struct ib_mad_guid_info {
struct ib_mad_hdr mad_hdr;
uint32_t mkey[2];
uint32_t reserved[8];
uint8_t gid_local[8];
} __attribute__ (( packed ));
struct ib_mad_port_info {
struct ib_mad_hdr mad_hdr;
uint32_t mkey[2];
uint32_t reserved[8];
uint32_t mkey2[2];
uint8_t gid_prefix[8];
uint16_t lid;
uint16_t mastersm_lid;
uint32_t cap_mask;
uint16_t diag_code;
uint16_t mkey_lease_period;
uint8_t local_port_num;
uint8_t link_width_enabled;
uint8_t link_width_supported;
uint8_t link_width_active;
uint8_t port_state__link_speed_supported;
uint8_t link_down_def_state__port_phys_state;
uint8_t lmc__r1__mkey_prot_bits;
uint8_t link_speed_enabled__link_speed_active;
} __attribute__ (( packed ));
struct ib_mad_pkey_table {
struct ib_mad_hdr mad_hdr;
uint32_t mkey[2];
uint32_t reserved[8];
uint16_t pkey[16][2];
} __attribute__ (( packed ));
struct ib_mad_path_record {
struct ib_mad_hdr mad_hdr;
struct ib_rmpp_hdr rmpp_hdr;
struct ib_sa_hdr sa_hdr;
uint32_t reserved0[2];
struct ib_gid dgid;
struct ib_gid sgid;
uint16_t dlid;
uint16_t slid;
uint32_t hop_limit__flow_label__raw_traffic;
uint32_t pkey__numb_path__reversible__tclass;
uint8_t reserved1;
uint8_t reserved__sl;
uint8_t mtu_selector__mtu;
uint8_t rate_selector__rate;
uint32_t preference__packet_lifetime__packet_lifetime_selector;
uint32_t reserved2[35];
} __attribute__ (( packed ));
struct ib_mad_mc_member_record {
struct ib_mad_hdr mad_hdr;
struct ib_rmpp_hdr rmpp_hdr;
struct ib_sa_hdr sa_hdr;
struct ib_gid mgid;
struct ib_gid port_gid;
uint32_t qkey;
uint16_t mlid;
uint8_t mtu_selector__mtu;
uint8_t tclass;
uint16_t pkey;
uint8_t rate_selector__rate;
uint8_t packet_lifetime_selector__packet_lifetime;
uint32_t sl__flow_label__hop_limit;
uint8_t scope__join_state;
uint8_t proxy_join__reserved;
uint16_t reserved0;
uint32_t reserved1[37];
} __attribute__ (( packed ));
union ib_mad {
struct ib_mad_hdr mad_hdr;
struct ib_mad_data data;
struct ib_mad_guid_info guid_info;
struct ib_mad_port_info port_info;
struct ib_mad_pkey_table pkey_table;
struct ib_mad_path_record path_record;
struct ib_mad_mc_member_record mc_member_record;
} __attribute__ (( packed ));
#endif /* _GPXE_INFINIBAND_H */

151
src/net/infiniband.c
View File

@ -333,142 +333,6 @@ void ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
qp->recv.fill--;
}
/***************************************************************************
*
* Management datagram operations
*
***************************************************************************
*/
/**
* Get port information
*
* @v ibdev Infiniband device
* @v port_info Port information datagram to fill in
* @ret rc Return status code
*/
static int ib_get_port_info ( struct ib_device *ibdev,
struct ib_mad_port_info *port_info ) {
struct ib_mad_hdr *hdr = &port_info->mad_hdr;
int rc;
/* Construct MAD */
memset ( port_info, 0, sizeof ( *port_info ) );
hdr->base_version = IB_MGMT_BASE_VERSION;
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
hdr->class_version = 1;
hdr->method = IB_MGMT_METHOD_GET;
hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
hdr->attr_mod = htonl ( ibdev->port );
if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *port_info ) ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not get port info: %s\n",
ibdev, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Get GUID information
*
* @v ibdev Infiniband device
* @v guid_info GUID information datagram to fill in
* @ret rc Return status code
*/
static int ib_get_guid_info ( struct ib_device *ibdev,
struct ib_mad_guid_info *guid_info ) {
struct ib_mad_hdr *hdr = &guid_info->mad_hdr;
int rc;
/* Construct MAD */
memset ( guid_info, 0, sizeof ( *guid_info ) );
hdr->base_version = IB_MGMT_BASE_VERSION;
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
hdr->class_version = 1;
hdr->method = IB_MGMT_METHOD_GET;
hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *guid_info ) ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n",
ibdev, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Get partition key table
*
* @v ibdev Infiniband device
* @v guid_info Partition key table datagram to fill in
* @ret rc Return status code
*/
static int ib_get_pkey_table ( struct ib_device *ibdev,
struct ib_mad_pkey_table *pkey_table ) {
struct ib_mad_hdr *hdr = &pkey_table->mad_hdr;
int rc;
/* Construct MAD */
memset ( pkey_table, 0, sizeof ( *pkey_table ) );
hdr->base_version = IB_MGMT_BASE_VERSION;
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
hdr->class_version = 1;
hdr->method = IB_MGMT_METHOD_GET;
hdr->attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *pkey_table ) ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n",
ibdev, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Get MAD parameters
*
* @v ibdev Infiniband device
* @ret rc Return status code
*/
static int ib_get_mad_params ( struct ib_device *ibdev ) {
union {
/* This union exists just to save stack space */
struct ib_mad_port_info port_info;
struct ib_mad_guid_info guid_info;
struct ib_mad_pkey_table pkey_table;
} u;
int rc;
/* Port info gives us the link state, the first half of the
* port GID and the SM LID.
*/
if ( ( rc = ib_get_port_info ( ibdev, &u.port_info ) ) != 0 )
return rc;
ibdev->link_up = ( ( u.port_info.port_state__link_speed_supported
& 0xf ) == 4 );
memcpy ( &ibdev->port_gid.u.bytes[0], u.port_info.gid_prefix, 8 );
ibdev->sm_lid = ntohs ( u.port_info.mastersm_lid );
/* GUID info gives us the second half of the port GID */
if ( ( rc = ib_get_guid_info ( ibdev, &u.guid_info ) ) != 0 )
return rc;
memcpy ( &ibdev->port_gid.u.bytes[8], u.guid_info.gid_local, 8 );
/* Get partition key */
if ( ( rc = ib_get_pkey_table ( ibdev, &u.pkey_table ) ) != 0 )
return rc;
ibdev->pkey = ntohs ( u.pkey_table.pkey[0][0] );
DBGC ( ibdev, "IBDEV %p port GID is %08lx:%08lx:%08lx:%08lx\n",
ibdev, htonl ( ibdev->port_gid.u.dwords[0] ),
htonl ( ibdev->port_gid.u.dwords[1] ),
htonl ( ibdev->port_gid.u.dwords[2] ),
htonl ( ibdev->port_gid.u.dwords[3] ) );
return 0;
}
/***************************************************************************
*
* Event queues
@ -482,14 +346,6 @@ static int ib_get_mad_params ( struct ib_device *ibdev ) {
* @v ibdev Infiniband device
*/
void ib_link_state_changed ( struct ib_device *ibdev ) {
int rc;
/* Update MAD parameters */
if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not update MAD parameters: %s\n",
ibdev, strerror ( rc ) );
return;
}
/* Notify IPoIB of link state change */
ipoib_link_state_changed ( ibdev );
@ -536,6 +392,8 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
if ( ibdev ) {
drv_priv = ( ( ( void * ) ibdev ) + sizeof ( *ibdev ) );
ib_set_drvdata ( ibdev, drv_priv );
ibdev->lid = IB_LID_NONE;
ibdev->pkey = IB_PKEY_NONE;
}
return ibdev;
}
@ -557,10 +415,6 @@ int register_ibdev ( struct ib_device *ibdev ) {
if ( ( rc = ib_open ( ibdev ) ) != 0 )
goto err_open;
/* Get MAD parameters */
if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 )
goto err_get_mad_params;
/* Add IPoIB device */
if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n",
@ -573,7 +427,6 @@ int register_ibdev ( struct ib_device *ibdev ) {
return 0;
err_ipoib_probe:
err_get_mad_params:
ib_close ( ibdev );
err_open:
list_del ( &ibdev->list );