david/ipxe
david
/
ipxe
Archived
1
0
Fork 0
Code

[infiniband] Add the concept of an Infiniband upper-layer driver 

Replace the explicit calls from the Infiniband core to the IPoIB layer
with the general concept of an Infiniband upper-layer driver
(analogous to a PCI driver) which can create arbitrary devices on top
of Infiniband devices.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
This commit is contained in:
Michael Brown 2010-09-04 23:35:09 +01:00
parent ca4df90a63
commit 35b19d8848
4 changed files with 170 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
src/drivers/net/ipoib.c
View File

@ -597,6 +597,42 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
}
}
/**
* Handle link status change
*
* @v ibdev Infiniband device
*/
static void ipoib_link_state_changed ( struct ib_device *ibdev ) {
struct net_device *netdev = ib_get_ownerdata ( ibdev );
struct ipoib_device *ipoib = netdev->priv;
struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
int rc;
/* Leave existing broadcast group */
ipoib_leave_broadcast_group ( ipoib );
/* Update MAC address based on potentially-new GID prefix */
memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
sizeof ( mac->gid.u.half[0] ) );
/* Update broadcast GID based on potentially-new partition key */
ipoib->broadcast.gid.u.words[2] =
htons ( ibdev->pkey | IB_PKEY_FULL );
/* Set net device link state to reflect Infiniband link state */
rc = ib_link_rc ( ibdev );
netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
/* Join new broadcast group */
if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) &&
( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
"%s\n", ipoib, strerror ( rc ) );
netdev_link_err ( netdev, rc );
return;
}
}
/**
* Open IPoIB network device
*
@ -690,49 +726,13 @@ static struct net_device_operations ipoib_operations = {
.irq = ipoib_irq,
};
/**
* Handle link status change
*
* @v ibdev Infiniband device
*/
void ipoib_link_state_changed ( struct ib_device *ibdev ) {
struct net_device *netdev = ib_get_ownerdata ( ibdev );
struct ipoib_device *ipoib = netdev->priv;
struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
int rc;
/* Leave existing broadcast group */
ipoib_leave_broadcast_group ( ipoib );
/* Update MAC address based on potentially-new GID prefix */
memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
sizeof ( mac->gid.u.half[0] ) );
/* Update broadcast GID based on potentially-new partition key */
ipoib->broadcast.gid.u.words[2] =
htons ( ibdev->pkey | IB_PKEY_FULL );
/* Set net device link state to reflect Infiniband link state */
rc = ib_link_rc ( ibdev );
netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
/* Join new broadcast group */
if ( ib_link_ok ( ibdev ) &&
( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
"%s\n", ipoib, strerror ( rc ) );
netdev_link_err ( netdev, rc );
return;
}
}
/**
* Probe IPoIB device
*
* @v ibdev Infiniband device
* @ret rc Return status code
*/
int ipoib_probe ( struct ib_device *ibdev ) {
static int ipoib_probe ( struct ib_device *ibdev ) {
struct net_device *netdev;
struct ipoib_device *ipoib;
int rc;
@ -775,10 +775,18 @@ int ipoib_probe ( struct ib_device *ibdev ) {
*
* @v ibdev Infiniband device
*/
void ipoib_remove ( struct ib_device *ibdev ) {
static void ipoib_remove ( struct ib_device *ibdev ) {
struct net_device *netdev = ib_get_ownerdata ( ibdev );
unregister_netdev ( netdev );
netdev_nullify ( netdev );
netdev_put ( netdev );
}
/** IPoIB driver */
struct ib_driver ipoib_driver __ib_driver = {
.name = "IPoIB",
.probe = ipoib_probe,
.notify = ipoib_link_state_changed,
.remove = ipoib_remove,
};

42
src/include/ipxe/infiniband.h
View File

@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <stdint.h>
#include <ipxe/refcnt.h>
#include <ipxe/device.h>
#include <ipxe/tables.h>
#include <ipxe/ib_packet.h>
#include <ipxe/ib_mad.h>
@ -432,6 +433,34 @@ struct ib_device {
void *owner_priv;
};
/** An Infiniband upper-layer driver */
struct ib_driver {
/** Name */
const char *name;
/** Probe device
*
* @v ibdev Infiniband device
* @ret rc Return status code
*/
int ( * probe ) ( struct ib_device *ibdev );
/** Notify of device or link state change
*
* @v ibdev Infiniband device
*/
void ( * notify ) ( struct ib_device *ibdev );
/** Remove device
*
* @v ibdev Infiniband device
*/
void ( * remove ) ( struct ib_device *ibdev );
};
/** Infiniband driver table */
#define IB_DRIVERS __table ( struct ib_driver, "ib_drivers" )
/** Declare an Infiniband driver */
#define __ib_driver __table_entry ( IB_DRIVERS, 01 )
extern struct ib_completion_queue *
ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
struct ib_completion_queue_operations *op );
@ -492,7 +521,7 @@ extern struct list_head ib_devices;
list_for_each_entry ( (ibdev), &ib_devices, list )
/**
* Check link state
* Check link state of Infiniband device
*
* @v ibdev Infiniband device
* @ret link_up Link is up
@ -502,6 +531,17 @@ ib_link_ok ( struct ib_device *ibdev ) {
return ( ibdev->port_state == IB_PORT_STATE_ACTIVE );
}
/**
* Check whether or not Infiniband device is open
*
* @v ibdev Infiniband device
* @v is_open Infiniband device is open
*/
static inline __attribute__ (( always_inline )) int
ib_is_open ( struct ib_device *ibdev ) {
return ( ibdev->open_count > 0 );
}
/**
* Get reference to Infiniband device
*

3
src/include/ipxe/ipoib.h
View File

@ -53,9 +53,6 @@ struct ipoib_hdr {
} __attribute__ (( packed ));
extern const char * ipoib_ntoa ( const void *ll_addr );
extern void ipoib_link_state_changed ( struct ib_device *ibdev );
extern int ipoib_probe ( struct ib_device *ibdev );
extern void ipoib_remove ( struct ib_device *ibdev );
extern struct net_device * alloc_ipoibdev ( size_t priv_size );
#endif /* _IPXE_IPOIB_H */

122
src/net/infiniband.c
View File

@ -31,7 +31,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <ipxe/if_arp.h>
#include <ipxe/netdevice.h>
#include <ipxe/iobuf.h>
#include <ipxe/ipoib.h>
#include <ipxe/process.h>
#include <ipxe/infiniband.h>
#include <ipxe/ib_mi.h>
@ -538,6 +537,64 @@ void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
***************************************************************************
*/
/**
* Get link state
*
* @v ibdev Infiniband device
* @ret rc Link status code
*/
int ib_link_rc ( struct ib_device *ibdev ) {
switch ( ibdev->port_state ) {
case IB_PORT_STATE_DOWN: return -ENOTCONN;
case IB_PORT_STATE_INIT: return -EINPROGRESS_INIT;
case IB_PORT_STATE_ARMED: return -EINPROGRESS_ARMED;
case IB_PORT_STATE_ACTIVE: return 0;
default: return -EINVAL;
}
}
/**
* Textual representation of Infiniband link state
*
* @v ibdev Infiniband device
* @ret link_text Link state text
*/
static const char * ib_link_state_text ( struct ib_device *ibdev ) {
switch ( ibdev->port_state ) {
case IB_PORT_STATE_DOWN: return "DOWN";
case IB_PORT_STATE_INIT: return "INIT";
case IB_PORT_STATE_ARMED: return "ARMED";
case IB_PORT_STATE_ACTIVE: return "ACTIVE";
default: return "UNKNOWN";
}
}
/**
* Notify drivers of Infiniband device or link state change
*
* @v ibdev Infiniband device
*/
static void ib_notify ( struct ib_device *ibdev ) {
struct ib_driver *driver;
for_each_table_entry ( driver, IB_DRIVERS )
driver->notify ( ibdev );
}
/**
* Notify of Infiniband link state change
*
* @v ibdev Infiniband device
*/
void ib_link_state_changed ( struct ib_device *ibdev ) {
DBGC ( ibdev, "IBDEV %p link state is %s\n",
ibdev, ib_link_state_text ( ibdev ) );
/* Notify drivers of link state change */
ib_notify ( ibdev );
}
/**
* Open port
*
@ -586,6 +643,9 @@ int ib_open ( struct ib_device *ibdev ) {
/* Add to head of open devices list */
list_add ( &ibdev->open_list, &open_ib_devices );
/* Notify drivers of device state change */
ib_notify ( ibdev );
assert ( ibdev->open_count == 1 );
return 0;
@ -614,6 +674,7 @@ void ib_close ( struct ib_device *ibdev ) {
/* Close device if this was the last remaining requested opening */
if ( ibdev->open_count == 0 ) {
ib_notify ( ibdev );
list_del ( &ibdev->open_list );
ib_destroy_mi ( ibdev, ibdev->gsi );
ib_destroy_sma ( ibdev, ibdev->smi );
@ -622,22 +683,6 @@ void ib_close ( struct ib_device *ibdev ) {
}
}
/**
* Get link state
*
* @v ibdev Infiniband device
* @ret rc Link status code
*/
int ib_link_rc ( struct ib_device *ibdev ) {
switch ( ibdev->port_state ) {
case IB_PORT_STATE_DOWN: return -ENOTCONN;
case IB_PORT_STATE_INIT: return -EINPROGRESS_INIT;
case IB_PORT_STATE_ARMED: return -EINPROGRESS_ARMED;
case IB_PORT_STATE_ACTIVE: return 0;
default: return -EINVAL;
}
}
/***************************************************************************
*
* Multicast
@ -799,17 +844,6 @@ int ib_set_pkey_table ( struct ib_device *ibdev, union ib_mad *mad ) {
***************************************************************************
*/
/**
* Handle Infiniband link state change
*
* @v ibdev Infiniband device
*/
void ib_link_state_changed ( struct ib_device *ibdev ) {
/* Notify IPoIB of link state change */
ipoib_link_state_changed ( ibdev );
}
/**
* Poll event queue
*
@ -883,24 +917,29 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
* @ret rc Return status code
*/
int register_ibdev ( struct ib_device *ibdev ) {
struct ib_driver *driver;
int rc;
/* Add to device list */
ibdev_get ( ibdev );
list_add_tail ( &ibdev->list, &ib_devices );
/* Add IPoIB device */
if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n",
ibdev, strerror ( rc ) );
goto err_ipoib_probe;
}
DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev,
ibdev->dev->name );
/* Probe device */
for_each_table_entry ( driver, IB_DRIVERS ) {
if ( ( rc = driver->probe ( ibdev ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p could not add %s device: %s\n",
ibdev, driver->name, strerror ( rc ) );
goto err_probe;
}
}
return 0;
err_ipoib_probe:
err_probe:
for_each_table_entry_continue_reverse ( driver, IB_DRIVERS )
driver->remove ( ibdev );
list_del ( &ibdev->list );
ibdev_put ( ibdev );
return rc;
@ -912,9 +951,11 @@ int register_ibdev ( struct ib_device *ibdev ) {
* @v ibdev Infiniband device
*/
void unregister_ibdev ( struct ib_device *ibdev ) {
struct ib_driver *driver;
/* Close device */
ipoib_remove ( ibdev );
/* Remove device */
for_each_table_entry_reverse ( driver, IB_DRIVERS )
driver->remove ( ibdev );
/* Remove from device list */
list_del ( &ibdev->list );
@ -953,3 +994,6 @@ struct ib_device * last_opened_ibdev ( void ) {
return NULL;
}
/* Drag in IPoIB */
REQUIRE_OBJECT ( ipoib );