[infiniband] Add the concept of an Infiniband upper-layer driver
Replace the explicit calls from the Infiniband core to the IPoIB layer with the general concept of an Infiniband upper-layer driver (analogous to a PCI driver) which can create arbitrary devices on top of Infiniband devices. Signed-off-by: Michael Brown <mcb30@ipxe.org>
This commit is contained in:
parent
ca4df90a63
commit
35b19d8848
@ -597,6 +597,42 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle link status change
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
*/
|
||||||
|
static void ipoib_link_state_changed ( struct ib_device *ibdev ) {
|
||||||
|
struct net_device *netdev = ib_get_ownerdata ( ibdev );
|
||||||
|
struct ipoib_device *ipoib = netdev->priv;
|
||||||
|
struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
|
||||||
|
int rc;
|
||||||
|
|
||||||
|
/* Leave existing broadcast group */
|
||||||
|
ipoib_leave_broadcast_group ( ipoib );
|
||||||
|
|
||||||
|
/* Update MAC address based on potentially-new GID prefix */
|
||||||
|
memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
|
||||||
|
sizeof ( mac->gid.u.half[0] ) );
|
||||||
|
|
||||||
|
/* Update broadcast GID based on potentially-new partition key */
|
||||||
|
ipoib->broadcast.gid.u.words[2] =
|
||||||
|
htons ( ibdev->pkey | IB_PKEY_FULL );
|
||||||
|
|
||||||
|
/* Set net device link state to reflect Infiniband link state */
|
||||||
|
rc = ib_link_rc ( ibdev );
|
||||||
|
netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
|
||||||
|
|
||||||
|
/* Join new broadcast group */
|
||||||
|
if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) &&
|
||||||
|
( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
|
||||||
|
DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
|
||||||
|
"%s\n", ipoib, strerror ( rc ) );
|
||||||
|
netdev_link_err ( netdev, rc );
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Open IPoIB network device
|
* Open IPoIB network device
|
||||||
*
|
*
|
||||||
@ -690,49 +726,13 @@ static struct net_device_operations ipoib_operations = {
|
|||||||
.irq = ipoib_irq,
|
.irq = ipoib_irq,
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle link status change
|
|
||||||
*
|
|
||||||
* @v ibdev Infiniband device
|
|
||||||
*/
|
|
||||||
void ipoib_link_state_changed ( struct ib_device *ibdev ) {
|
|
||||||
struct net_device *netdev = ib_get_ownerdata ( ibdev );
|
|
||||||
struct ipoib_device *ipoib = netdev->priv;
|
|
||||||
struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
|
|
||||||
int rc;
|
|
||||||
|
|
||||||
/* Leave existing broadcast group */
|
|
||||||
ipoib_leave_broadcast_group ( ipoib );
|
|
||||||
|
|
||||||
/* Update MAC address based on potentially-new GID prefix */
|
|
||||||
memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
|
|
||||||
sizeof ( mac->gid.u.half[0] ) );
|
|
||||||
|
|
||||||
/* Update broadcast GID based on potentially-new partition key */
|
|
||||||
ipoib->broadcast.gid.u.words[2] =
|
|
||||||
htons ( ibdev->pkey | IB_PKEY_FULL );
|
|
||||||
|
|
||||||
/* Set net device link state to reflect Infiniband link state */
|
|
||||||
rc = ib_link_rc ( ibdev );
|
|
||||||
netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
|
|
||||||
|
|
||||||
/* Join new broadcast group */
|
|
||||||
if ( ib_link_ok ( ibdev ) &&
|
|
||||||
( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
|
|
||||||
DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
|
|
||||||
"%s\n", ipoib, strerror ( rc ) );
|
|
||||||
netdev_link_err ( netdev, rc );
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Probe IPoIB device
|
* Probe IPoIB device
|
||||||
*
|
*
|
||||||
* @v ibdev Infiniband device
|
* @v ibdev Infiniband device
|
||||||
* @ret rc Return status code
|
* @ret rc Return status code
|
||||||
*/
|
*/
|
||||||
int ipoib_probe ( struct ib_device *ibdev ) {
|
static int ipoib_probe ( struct ib_device *ibdev ) {
|
||||||
struct net_device *netdev;
|
struct net_device *netdev;
|
||||||
struct ipoib_device *ipoib;
|
struct ipoib_device *ipoib;
|
||||||
int rc;
|
int rc;
|
||||||
@ -775,10 +775,18 @@ int ipoib_probe ( struct ib_device *ibdev ) {
|
|||||||
*
|
*
|
||||||
* @v ibdev Infiniband device
|
* @v ibdev Infiniband device
|
||||||
*/
|
*/
|
||||||
void ipoib_remove ( struct ib_device *ibdev ) {
|
static void ipoib_remove ( struct ib_device *ibdev ) {
|
||||||
struct net_device *netdev = ib_get_ownerdata ( ibdev );
|
struct net_device *netdev = ib_get_ownerdata ( ibdev );
|
||||||
|
|
||||||
unregister_netdev ( netdev );
|
unregister_netdev ( netdev );
|
||||||
netdev_nullify ( netdev );
|
netdev_nullify ( netdev );
|
||||||
netdev_put ( netdev );
|
netdev_put ( netdev );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** IPoIB driver */
|
||||||
|
struct ib_driver ipoib_driver __ib_driver = {
|
||||||
|
.name = "IPoIB",
|
||||||
|
.probe = ipoib_probe,
|
||||||
|
.notify = ipoib_link_state_changed,
|
||||||
|
.remove = ipoib_remove,
|
||||||
|
};
|
||||||
|
@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
|||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <ipxe/refcnt.h>
|
#include <ipxe/refcnt.h>
|
||||||
#include <ipxe/device.h>
|
#include <ipxe/device.h>
|
||||||
|
#include <ipxe/tables.h>
|
||||||
#include <ipxe/ib_packet.h>
|
#include <ipxe/ib_packet.h>
|
||||||
#include <ipxe/ib_mad.h>
|
#include <ipxe/ib_mad.h>
|
||||||
|
|
||||||
@ -432,6 +433,34 @@ struct ib_device {
|
|||||||
void *owner_priv;
|
void *owner_priv;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** An Infiniband upper-layer driver */
|
||||||
|
struct ib_driver {
|
||||||
|
/** Name */
|
||||||
|
const char *name;
|
||||||
|
/** Probe device
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @ret rc Return status code
|
||||||
|
*/
|
||||||
|
int ( * probe ) ( struct ib_device *ibdev );
|
||||||
|
/** Notify of device or link state change
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
*/
|
||||||
|
void ( * notify ) ( struct ib_device *ibdev );
|
||||||
|
/** Remove device
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
*/
|
||||||
|
void ( * remove ) ( struct ib_device *ibdev );
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Infiniband driver table */
|
||||||
|
#define IB_DRIVERS __table ( struct ib_driver, "ib_drivers" )
|
||||||
|
|
||||||
|
/** Declare an Infiniband driver */
|
||||||
|
#define __ib_driver __table_entry ( IB_DRIVERS, 01 )
|
||||||
|
|
||||||
extern struct ib_completion_queue *
|
extern struct ib_completion_queue *
|
||||||
ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
|
ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
|
||||||
struct ib_completion_queue_operations *op );
|
struct ib_completion_queue_operations *op );
|
||||||
@ -492,7 +521,7 @@ extern struct list_head ib_devices;
|
|||||||
list_for_each_entry ( (ibdev), &ib_devices, list )
|
list_for_each_entry ( (ibdev), &ib_devices, list )
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check link state
|
* Check link state of Infiniband device
|
||||||
*
|
*
|
||||||
* @v ibdev Infiniband device
|
* @v ibdev Infiniband device
|
||||||
* @ret link_up Link is up
|
* @ret link_up Link is up
|
||||||
@ -502,6 +531,17 @@ ib_link_ok ( struct ib_device *ibdev ) {
|
|||||||
return ( ibdev->port_state == IB_PORT_STATE_ACTIVE );
|
return ( ibdev->port_state == IB_PORT_STATE_ACTIVE );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check whether or not Infiniband device is open
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v is_open Infiniband device is open
|
||||||
|
*/
|
||||||
|
static inline __attribute__ (( always_inline )) int
|
||||||
|
ib_is_open ( struct ib_device *ibdev ) {
|
||||||
|
return ( ibdev->open_count > 0 );
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get reference to Infiniband device
|
* Get reference to Infiniband device
|
||||||
*
|
*
|
||||||
|
@ -53,9 +53,6 @@ struct ipoib_hdr {
|
|||||||
} __attribute__ (( packed ));
|
} __attribute__ (( packed ));
|
||||||
|
|
||||||
extern const char * ipoib_ntoa ( const void *ll_addr );
|
extern const char * ipoib_ntoa ( const void *ll_addr );
|
||||||
extern void ipoib_link_state_changed ( struct ib_device *ibdev );
|
|
||||||
extern int ipoib_probe ( struct ib_device *ibdev );
|
|
||||||
extern void ipoib_remove ( struct ib_device *ibdev );
|
|
||||||
extern struct net_device * alloc_ipoibdev ( size_t priv_size );
|
extern struct net_device * alloc_ipoibdev ( size_t priv_size );
|
||||||
|
|
||||||
#endif /* _IPXE_IPOIB_H */
|
#endif /* _IPXE_IPOIB_H */
|
||||||
|
@ -31,7 +31,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
|||||||
#include <ipxe/if_arp.h>
|
#include <ipxe/if_arp.h>
|
||||||
#include <ipxe/netdevice.h>
|
#include <ipxe/netdevice.h>
|
||||||
#include <ipxe/iobuf.h>
|
#include <ipxe/iobuf.h>
|
||||||
#include <ipxe/ipoib.h>
|
|
||||||
#include <ipxe/process.h>
|
#include <ipxe/process.h>
|
||||||
#include <ipxe/infiniband.h>
|
#include <ipxe/infiniband.h>
|
||||||
#include <ipxe/ib_mi.h>
|
#include <ipxe/ib_mi.h>
|
||||||
@ -538,6 +537,64 @@ void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
|
|||||||
***************************************************************************
|
***************************************************************************
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get link state
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @ret rc Link status code
|
||||||
|
*/
|
||||||
|
int ib_link_rc ( struct ib_device *ibdev ) {
|
||||||
|
switch ( ibdev->port_state ) {
|
||||||
|
case IB_PORT_STATE_DOWN: return -ENOTCONN;
|
||||||
|
case IB_PORT_STATE_INIT: return -EINPROGRESS_INIT;
|
||||||
|
case IB_PORT_STATE_ARMED: return -EINPROGRESS_ARMED;
|
||||||
|
case IB_PORT_STATE_ACTIVE: return 0;
|
||||||
|
default: return -EINVAL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Textual representation of Infiniband link state
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @ret link_text Link state text
|
||||||
|
*/
|
||||||
|
static const char * ib_link_state_text ( struct ib_device *ibdev ) {
|
||||||
|
switch ( ibdev->port_state ) {
|
||||||
|
case IB_PORT_STATE_DOWN: return "DOWN";
|
||||||
|
case IB_PORT_STATE_INIT: return "INIT";
|
||||||
|
case IB_PORT_STATE_ARMED: return "ARMED";
|
||||||
|
case IB_PORT_STATE_ACTIVE: return "ACTIVE";
|
||||||
|
default: return "UNKNOWN";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Notify drivers of Infiniband device or link state change
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
*/
|
||||||
|
static void ib_notify ( struct ib_device *ibdev ) {
|
||||||
|
struct ib_driver *driver;
|
||||||
|
|
||||||
|
for_each_table_entry ( driver, IB_DRIVERS )
|
||||||
|
driver->notify ( ibdev );
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Notify of Infiniband link state change
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
*/
|
||||||
|
void ib_link_state_changed ( struct ib_device *ibdev ) {
|
||||||
|
|
||||||
|
DBGC ( ibdev, "IBDEV %p link state is %s\n",
|
||||||
|
ibdev, ib_link_state_text ( ibdev ) );
|
||||||
|
|
||||||
|
/* Notify drivers of link state change */
|
||||||
|
ib_notify ( ibdev );
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Open port
|
* Open port
|
||||||
*
|
*
|
||||||
@ -586,6 +643,9 @@ int ib_open ( struct ib_device *ibdev ) {
|
|||||||
/* Add to head of open devices list */
|
/* Add to head of open devices list */
|
||||||
list_add ( &ibdev->open_list, &open_ib_devices );
|
list_add ( &ibdev->open_list, &open_ib_devices );
|
||||||
|
|
||||||
|
/* Notify drivers of device state change */
|
||||||
|
ib_notify ( ibdev );
|
||||||
|
|
||||||
assert ( ibdev->open_count == 1 );
|
assert ( ibdev->open_count == 1 );
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
@ -614,6 +674,7 @@ void ib_close ( struct ib_device *ibdev ) {
|
|||||||
|
|
||||||
/* Close device if this was the last remaining requested opening */
|
/* Close device if this was the last remaining requested opening */
|
||||||
if ( ibdev->open_count == 0 ) {
|
if ( ibdev->open_count == 0 ) {
|
||||||
|
ib_notify ( ibdev );
|
||||||
list_del ( &ibdev->open_list );
|
list_del ( &ibdev->open_list );
|
||||||
ib_destroy_mi ( ibdev, ibdev->gsi );
|
ib_destroy_mi ( ibdev, ibdev->gsi );
|
||||||
ib_destroy_sma ( ibdev, ibdev->smi );
|
ib_destroy_sma ( ibdev, ibdev->smi );
|
||||||
@ -622,22 +683,6 @@ void ib_close ( struct ib_device *ibdev ) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get link state
|
|
||||||
*
|
|
||||||
* @v ibdev Infiniband device
|
|
||||||
* @ret rc Link status code
|
|
||||||
*/
|
|
||||||
int ib_link_rc ( struct ib_device *ibdev ) {
|
|
||||||
switch ( ibdev->port_state ) {
|
|
||||||
case IB_PORT_STATE_DOWN: return -ENOTCONN;
|
|
||||||
case IB_PORT_STATE_INIT: return -EINPROGRESS_INIT;
|
|
||||||
case IB_PORT_STATE_ARMED: return -EINPROGRESS_ARMED;
|
|
||||||
case IB_PORT_STATE_ACTIVE: return 0;
|
|
||||||
default: return -EINVAL;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/***************************************************************************
|
/***************************************************************************
|
||||||
*
|
*
|
||||||
* Multicast
|
* Multicast
|
||||||
@ -799,17 +844,6 @@ int ib_set_pkey_table ( struct ib_device *ibdev, union ib_mad *mad ) {
|
|||||||
***************************************************************************
|
***************************************************************************
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle Infiniband link state change
|
|
||||||
*
|
|
||||||
* @v ibdev Infiniband device
|
|
||||||
*/
|
|
||||||
void ib_link_state_changed ( struct ib_device *ibdev ) {
|
|
||||||
|
|
||||||
/* Notify IPoIB of link state change */
|
|
||||||
ipoib_link_state_changed ( ibdev );
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Poll event queue
|
* Poll event queue
|
||||||
*
|
*
|
||||||
@ -883,24 +917,29 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
|
|||||||
* @ret rc Return status code
|
* @ret rc Return status code
|
||||||
*/
|
*/
|
||||||
int register_ibdev ( struct ib_device *ibdev ) {
|
int register_ibdev ( struct ib_device *ibdev ) {
|
||||||
|
struct ib_driver *driver;
|
||||||
int rc;
|
int rc;
|
||||||
|
|
||||||
/* Add to device list */
|
/* Add to device list */
|
||||||
ibdev_get ( ibdev );
|
ibdev_get ( ibdev );
|
||||||
list_add_tail ( &ibdev->list, &ib_devices );
|
list_add_tail ( &ibdev->list, &ib_devices );
|
||||||
|
|
||||||
/* Add IPoIB device */
|
|
||||||
if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
|
|
||||||
DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n",
|
|
||||||
ibdev, strerror ( rc ) );
|
|
||||||
goto err_ipoib_probe;
|
|
||||||
}
|
|
||||||
|
|
||||||
DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev,
|
DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev,
|
||||||
ibdev->dev->name );
|
ibdev->dev->name );
|
||||||
|
|
||||||
|
/* Probe device */
|
||||||
|
for_each_table_entry ( driver, IB_DRIVERS ) {
|
||||||
|
if ( ( rc = driver->probe ( ibdev ) ) != 0 ) {
|
||||||
|
DBGC ( ibdev, "IBDEV %p could not add %s device: %s\n",
|
||||||
|
ibdev, driver->name, strerror ( rc ) );
|
||||||
|
goto err_probe;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
err_ipoib_probe:
|
err_probe:
|
||||||
|
for_each_table_entry_continue_reverse ( driver, IB_DRIVERS )
|
||||||
|
driver->remove ( ibdev );
|
||||||
list_del ( &ibdev->list );
|
list_del ( &ibdev->list );
|
||||||
ibdev_put ( ibdev );
|
ibdev_put ( ibdev );
|
||||||
return rc;
|
return rc;
|
||||||
@ -912,9 +951,11 @@ int register_ibdev ( struct ib_device *ibdev ) {
|
|||||||
* @v ibdev Infiniband device
|
* @v ibdev Infiniband device
|
||||||
*/
|
*/
|
||||||
void unregister_ibdev ( struct ib_device *ibdev ) {
|
void unregister_ibdev ( struct ib_device *ibdev ) {
|
||||||
|
struct ib_driver *driver;
|
||||||
|
|
||||||
/* Close device */
|
/* Remove device */
|
||||||
ipoib_remove ( ibdev );
|
for_each_table_entry_reverse ( driver, IB_DRIVERS )
|
||||||
|
driver->remove ( ibdev );
|
||||||
|
|
||||||
/* Remove from device list */
|
/* Remove from device list */
|
||||||
list_del ( &ibdev->list );
|
list_del ( &ibdev->list );
|
||||||
@ -953,3 +994,6 @@ struct ib_device * last_opened_ibdev ( void ) {
|
|||||||
|
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Drag in IPoIB */
|
||||||
|
REQUIRE_OBJECT ( ipoib );
|
||||||
|
Reference in New Issue
Block a user