From 35b19d8848c5141aff8ef858f908e9f7a7cf0b1d Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Sat, 4 Sep 2010 23:35:09 +0100 Subject: [PATCH] [infiniband] Add the concept of an Infiniband upper-layer driver Replace the explicit calls from the Infiniband core to the IPoIB layer with the general concept of an Infiniband upper-layer driver (analogous to a PCI driver) which can create arbitrary devices on top of Infiniband devices. Signed-off-by: Michael Brown --- src/drivers/net/ipoib.c | 84 ++++++++++++----------- src/include/ipxe/infiniband.h | 42 +++++++++++- src/include/ipxe/ipoib.h | 3 - src/net/infiniband.c | 122 +++++++++++++++++++++++----------- 4 files changed, 170 insertions(+), 81 deletions(-) diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c index baa1c644..82d7ca99 100644 --- a/src/drivers/net/ipoib.c +++ b/src/drivers/net/ipoib.c @@ -597,6 +597,42 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) { } } +/** + * Handle link status change + * + * @v ibdev Infiniband device + */ +static void ipoib_link_state_changed ( struct ib_device *ibdev ) { + struct net_device *netdev = ib_get_ownerdata ( ibdev ); + struct ipoib_device *ipoib = netdev->priv; + struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr ); + int rc; + + /* Leave existing broadcast group */ + ipoib_leave_broadcast_group ( ipoib ); + + /* Update MAC address based on potentially-new GID prefix */ + memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0], + sizeof ( mac->gid.u.half[0] ) ); + + /* Update broadcast GID based on potentially-new partition key */ + ipoib->broadcast.gid.u.words[2] = + htons ( ibdev->pkey | IB_PKEY_FULL ); + + /* Set net device link state to reflect Infiniband link state */ + rc = ib_link_rc ( ibdev ); + netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) ); + + /* Join new broadcast group */ + if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) && + ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) { + DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: " + "%s\n", ipoib, strerror ( rc ) ); + netdev_link_err ( netdev, rc ); + return; + } +} + /** * Open IPoIB network device * @@ -690,49 +726,13 @@ static struct net_device_operations ipoib_operations = { .irq = ipoib_irq, }; -/** - * Handle link status change - * - * @v ibdev Infiniband device - */ -void ipoib_link_state_changed ( struct ib_device *ibdev ) { - struct net_device *netdev = ib_get_ownerdata ( ibdev ); - struct ipoib_device *ipoib = netdev->priv; - struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr ); - int rc; - - /* Leave existing broadcast group */ - ipoib_leave_broadcast_group ( ipoib ); - - /* Update MAC address based on potentially-new GID prefix */ - memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0], - sizeof ( mac->gid.u.half[0] ) ); - - /* Update broadcast GID based on potentially-new partition key */ - ipoib->broadcast.gid.u.words[2] = - htons ( ibdev->pkey | IB_PKEY_FULL ); - - /* Set net device link state to reflect Infiniband link state */ - rc = ib_link_rc ( ibdev ); - netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) ); - - /* Join new broadcast group */ - if ( ib_link_ok ( ibdev ) && - ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) { - DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: " - "%s\n", ipoib, strerror ( rc ) ); - netdev_link_err ( netdev, rc ); - return; - } -} - /** * Probe IPoIB device * * @v ibdev Infiniband device * @ret rc Return status code */ -int ipoib_probe ( struct ib_device *ibdev ) { +static int ipoib_probe ( struct ib_device *ibdev ) { struct net_device *netdev; struct ipoib_device *ipoib; int rc; @@ -775,10 +775,18 @@ int ipoib_probe ( struct ib_device *ibdev ) { * * @v ibdev Infiniband device */ -void ipoib_remove ( struct ib_device *ibdev ) { +static void ipoib_remove ( struct ib_device *ibdev ) { struct net_device *netdev = ib_get_ownerdata ( ibdev ); unregister_netdev ( netdev ); netdev_nullify ( netdev ); netdev_put ( netdev ); } + +/** IPoIB driver */ +struct ib_driver ipoib_driver __ib_driver = { + .name = "IPoIB", + .probe = ipoib_probe, + .notify = ipoib_link_state_changed, + .remove = ipoib_remove, +}; diff --git a/src/include/ipxe/infiniband.h b/src/include/ipxe/infiniband.h index f2eb57cd..edcce371 100644 --- a/src/include/ipxe/infiniband.h +++ b/src/include/ipxe/infiniband.h @@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include #include #include +#include #include #include @@ -432,6 +433,34 @@ struct ib_device { void *owner_priv; }; +/** An Infiniband upper-layer driver */ +struct ib_driver { + /** Name */ + const char *name; + /** Probe device + * + * @v ibdev Infiniband device + * @ret rc Return status code + */ + int ( * probe ) ( struct ib_device *ibdev ); + /** Notify of device or link state change + * + * @v ibdev Infiniband device + */ + void ( * notify ) ( struct ib_device *ibdev ); + /** Remove device + * + * @v ibdev Infiniband device + */ + void ( * remove ) ( struct ib_device *ibdev ); +}; + +/** Infiniband driver table */ +#define IB_DRIVERS __table ( struct ib_driver, "ib_drivers" ) + +/** Declare an Infiniband driver */ +#define __ib_driver __table_entry ( IB_DRIVERS, 01 ) + extern struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes, struct ib_completion_queue_operations *op ); @@ -492,7 +521,7 @@ extern struct list_head ib_devices; list_for_each_entry ( (ibdev), &ib_devices, list ) /** - * Check link state + * Check link state of Infiniband device * * @v ibdev Infiniband device * @ret link_up Link is up @@ -502,6 +531,17 @@ ib_link_ok ( struct ib_device *ibdev ) { return ( ibdev->port_state == IB_PORT_STATE_ACTIVE ); } +/** + * Check whether or not Infiniband device is open + * + * @v ibdev Infiniband device + * @v is_open Infiniband device is open + */ +static inline __attribute__ (( always_inline )) int +ib_is_open ( struct ib_device *ibdev ) { + return ( ibdev->open_count > 0 ); +} + /** * Get reference to Infiniband device * diff --git a/src/include/ipxe/ipoib.h b/src/include/ipxe/ipoib.h index 31b0c1b6..6a3fd607 100644 --- a/src/include/ipxe/ipoib.h +++ b/src/include/ipxe/ipoib.h @@ -53,9 +53,6 @@ struct ipoib_hdr { } __attribute__ (( packed )); extern const char * ipoib_ntoa ( const void *ll_addr ); -extern void ipoib_link_state_changed ( struct ib_device *ibdev ); -extern int ipoib_probe ( struct ib_device *ibdev ); -extern void ipoib_remove ( struct ib_device *ibdev ); extern struct net_device * alloc_ipoibdev ( size_t priv_size ); #endif /* _IPXE_IPOIB_H */ diff --git a/src/net/infiniband.c b/src/net/infiniband.c index 5daecd52..76fce3bd 100644 --- a/src/net/infiniband.c +++ b/src/net/infiniband.c @@ -31,7 +31,6 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include #include #include -#include #include #include #include @@ -538,6 +537,64 @@ void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { *************************************************************************** */ +/** + * Get link state + * + * @v ibdev Infiniband device + * @ret rc Link status code + */ +int ib_link_rc ( struct ib_device *ibdev ) { + switch ( ibdev->port_state ) { + case IB_PORT_STATE_DOWN: return -ENOTCONN; + case IB_PORT_STATE_INIT: return -EINPROGRESS_INIT; + case IB_PORT_STATE_ARMED: return -EINPROGRESS_ARMED; + case IB_PORT_STATE_ACTIVE: return 0; + default: return -EINVAL; + } +} + +/** + * Textual representation of Infiniband link state + * + * @v ibdev Infiniband device + * @ret link_text Link state text + */ +static const char * ib_link_state_text ( struct ib_device *ibdev ) { + switch ( ibdev->port_state ) { + case IB_PORT_STATE_DOWN: return "DOWN"; + case IB_PORT_STATE_INIT: return "INIT"; + case IB_PORT_STATE_ARMED: return "ARMED"; + case IB_PORT_STATE_ACTIVE: return "ACTIVE"; + default: return "UNKNOWN"; + } +} + +/** + * Notify drivers of Infiniband device or link state change + * + * @v ibdev Infiniband device + */ +static void ib_notify ( struct ib_device *ibdev ) { + struct ib_driver *driver; + + for_each_table_entry ( driver, IB_DRIVERS ) + driver->notify ( ibdev ); +} + +/** + * Notify of Infiniband link state change + * + * @v ibdev Infiniband device + */ +void ib_link_state_changed ( struct ib_device *ibdev ) { + + DBGC ( ibdev, "IBDEV %p link state is %s\n", + ibdev, ib_link_state_text ( ibdev ) ); + + /* Notify drivers of link state change */ + ib_notify ( ibdev ); +} + /** * Open port * @@ -586,6 +643,9 @@ int ib_open ( struct ib_device *ibdev ) { /* Add to head of open devices list */ list_add ( &ibdev->open_list, &open_ib_devices ); + /* Notify drivers of device state change */ + ib_notify ( ibdev ); + assert ( ibdev->open_count == 1 ); return 0; @@ -614,6 +674,7 @@ void ib_close ( struct ib_device *ibdev ) { /* Close device if this was the last remaining requested opening */ if ( ibdev->open_count == 0 ) { + ib_notify ( ibdev ); list_del ( &ibdev->open_list ); ib_destroy_mi ( ibdev, ibdev->gsi ); ib_destroy_sma ( ibdev, ibdev->smi ); @@ -622,22 +683,6 @@ void ib_close ( struct ib_device *ibdev ) { } } -/** - * Get link state - * - * @v ibdev Infiniband device - * @ret rc Link status code - */ -int ib_link_rc ( struct ib_device *ibdev ) { - switch ( ibdev->port_state ) { - case IB_PORT_STATE_DOWN: return -ENOTCONN; - case IB_PORT_STATE_INIT: return -EINPROGRESS_INIT; - case IB_PORT_STATE_ARMED: return -EINPROGRESS_ARMED; - case IB_PORT_STATE_ACTIVE: return 0; - default: return -EINVAL; - } -} - /*************************************************************************** * * Multicast @@ -799,17 +844,6 @@ int ib_set_pkey_table ( struct ib_device *ibdev, union ib_mad *mad ) { *************************************************************************** */ -/** - * Handle Infiniband link state change - * - * @v ibdev Infiniband device - */ -void ib_link_state_changed ( struct ib_device *ibdev ) { - - /* Notify IPoIB of link state change */ - ipoib_link_state_changed ( ibdev ); -} - /** * Poll event queue * @@ -883,24 +917,29 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) { * @ret rc Return status code */ int register_ibdev ( struct ib_device *ibdev ) { + struct ib_driver *driver; int rc; /* Add to device list */ ibdev_get ( ibdev ); list_add_tail ( &ibdev->list, &ib_devices ); - - /* Add IPoIB device */ - if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) { - DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n", - ibdev, strerror ( rc ) ); - goto err_ipoib_probe; - } - DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev, ibdev->dev->name ); + + /* Probe device */ + for_each_table_entry ( driver, IB_DRIVERS ) { + if ( ( rc = driver->probe ( ibdev ) ) != 0 ) { + DBGC ( ibdev, "IBDEV %p could not add %s device: %s\n", + ibdev, driver->name, strerror ( rc ) ); + goto err_probe; + } + } + return 0; - err_ipoib_probe: + err_probe: + for_each_table_entry_continue_reverse ( driver, IB_DRIVERS ) + driver->remove ( ibdev ); list_del ( &ibdev->list ); ibdev_put ( ibdev ); return rc; @@ -912,9 +951,11 @@ int register_ibdev ( struct ib_device *ibdev ) { * @v ibdev Infiniband device */ void unregister_ibdev ( struct ib_device *ibdev ) { + struct ib_driver *driver; - /* Close device */ - ipoib_remove ( ibdev ); + /* Remove device */ + for_each_table_entry_reverse ( driver, IB_DRIVERS ) + driver->remove ( ibdev ); /* Remove from device list */ list_del ( &ibdev->list ); @@ -953,3 +994,6 @@ struct ib_device * last_opened_ibdev ( void ) { return NULL; } + +/* Drag in IPoIB */ +REQUIRE_OBJECT ( ipoib );