[infiniband] Update all other MAD users to use a management interface
This commit is contained in:
parent
44251ebb9a
commit
34bfc04e4c
|
@ -58,12 +58,14 @@ struct ipoib_device {
|
||||||
struct ib_queue_pair *qp;
|
struct ib_queue_pair *qp;
|
||||||
/** Broadcast MAC */
|
/** Broadcast MAC */
|
||||||
struct ipoib_mac broadcast;
|
struct ipoib_mac broadcast;
|
||||||
/** Joined to multicast group
|
/** Joined to IPv4 broadcast multicast group
|
||||||
*
|
*
|
||||||
* This flag indicates whether or not we have initiated the
|
* This flag indicates whether or not we have initiated the
|
||||||
* join to the IPv4 multicast group.
|
* join to the IPv4 broadcast multicast group.
|
||||||
*/
|
*/
|
||||||
int broadcast_joined;
|
int broadcast_joined;
|
||||||
|
/** IPv4 broadcast multicast group membership */
|
||||||
|
struct ib_mc_membership broadcast_membership;
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Broadcast IPoIB address */
|
/** Broadcast IPoIB address */
|
||||||
|
@ -456,6 +458,26 @@ static void ipoib_irq ( struct net_device *netdev __unused,
|
||||||
/* No implementation */
|
/* No implementation */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle IPv4 broadcast multicast group join completion
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v qp Queue pair
|
||||||
|
* @v membership Multicast group membership
|
||||||
|
* @v rc Status code
|
||||||
|
* @v mad Response MAD (or NULL on error)
|
||||||
|
*/
|
||||||
|
void ipoib_join_complete ( struct ib_device *ibdev __unused,
|
||||||
|
struct ib_queue_pair *qp __unused,
|
||||||
|
struct ib_mc_membership *membership, int rc,
|
||||||
|
union ib_mad *mad __unused ) {
|
||||||
|
struct ipoib_device *ipoib = container_of ( membership,
|
||||||
|
struct ipoib_device, broadcast_membership );
|
||||||
|
|
||||||
|
/* Record join status as link status */
|
||||||
|
netdev_link_err ( ipoib->netdev, rc );
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Join IPv4 broadcast multicast group
|
* Join IPv4 broadcast multicast group
|
||||||
*
|
*
|
||||||
|
@ -466,7 +488,9 @@ static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
|
||||||
int rc;
|
int rc;
|
||||||
|
|
||||||
if ( ( rc = ib_mcast_join ( ipoib->ibdev, ipoib->qp,
|
if ( ( rc = ib_mcast_join ( ipoib->ibdev, ipoib->qp,
|
||||||
&ipoib->broadcast.gid ) ) != 0 ) {
|
&ipoib->broadcast_membership,
|
||||||
|
&ipoib->broadcast.gid,
|
||||||
|
ipoib_join_complete ) ) != 0 ) {
|
||||||
DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n",
|
DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n",
|
||||||
ipoib, strerror ( rc ) );
|
ipoib, strerror ( rc ) );
|
||||||
return rc;
|
return rc;
|
||||||
|
@ -485,7 +509,7 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
|
||||||
|
|
||||||
if ( ipoib->broadcast_joined ) {
|
if ( ipoib->broadcast_joined ) {
|
||||||
ib_mcast_leave ( ipoib->ibdev, ipoib->qp,
|
ib_mcast_leave ( ipoib->ibdev, ipoib->qp,
|
||||||
&ipoib->broadcast.gid );
|
&ipoib->broadcast_membership );
|
||||||
ipoib->broadcast_joined = 0;
|
ipoib->broadcast_joined = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,12 +10,60 @@
|
||||||
FILE_LICENCE ( GPL2_OR_LATER );
|
FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
|
|
||||||
#include <gpxe/infiniband.h>
|
#include <gpxe/infiniband.h>
|
||||||
|
#include <gpxe/retry.h>
|
||||||
|
|
||||||
extern int ib_cm_connect ( struct ib_queue_pair *qp, struct ib_gid *dgid,
|
struct ib_mad_transaction;
|
||||||
struct ib_gid_half *service_id,
|
struct ib_connection;
|
||||||
void *private_data, size_t private_data_len,
|
|
||||||
void ( * notify ) ( struct ib_queue_pair *qp,
|
/** Infiniband connection operations */
|
||||||
int rc, void *private_data,
|
struct ib_connection_operations {
|
||||||
size_t private_data_len ) );
|
/** Handle change of connection status
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v qp Queue pair
|
||||||
|
* @v conn Connection
|
||||||
|
* @v rc Connection status code
|
||||||
|
* @v private_data Private data, if available
|
||||||
|
* @v private_data_len Length of private data
|
||||||
|
*/
|
||||||
|
void ( * changed ) ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||||
|
struct ib_connection *conn, int rc,
|
||||||
|
void *private_data, size_t private_data_len );
|
||||||
|
};
|
||||||
|
|
||||||
|
/** An Infiniband connection */
|
||||||
|
struct ib_connection {
|
||||||
|
/** Infiniband device */
|
||||||
|
struct ib_device *ibdev;
|
||||||
|
/** Queue pair */
|
||||||
|
struct ib_queue_pair *qp;
|
||||||
|
/** Local communication ID */
|
||||||
|
uint32_t local_id;
|
||||||
|
/** Remote communication ID */
|
||||||
|
uint32_t remote_id;
|
||||||
|
/** Target service ID */
|
||||||
|
struct ib_gid_half service_id;
|
||||||
|
/** Connection operations */
|
||||||
|
struct ib_connection_operations *op;
|
||||||
|
|
||||||
|
/** Path to target */
|
||||||
|
struct ib_path *path;
|
||||||
|
/** Connection request management transaction */
|
||||||
|
struct ib_mad_transaction *madx;
|
||||||
|
|
||||||
|
/** Length of connection request private data */
|
||||||
|
size_t private_data_len;
|
||||||
|
/** Connection request private data */
|
||||||
|
uint8_t private_data[0];
|
||||||
|
};
|
||||||
|
|
||||||
|
extern struct ib_connection *
|
||||||
|
ib_create_conn ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||||
|
struct ib_gid *dgid, struct ib_gid_half *service_id,
|
||||||
|
void *req_private_data, size_t req_private_data_len,
|
||||||
|
struct ib_connection_operations *op );
|
||||||
|
extern void ib_destroy_conn ( struct ib_device *ibdev,
|
||||||
|
struct ib_queue_pair *qp,
|
||||||
|
struct ib_connection *conn );
|
||||||
|
|
||||||
#endif /* _GPXE_IB_CM_H */
|
#endif /* _GPXE_IB_CM_H */
|
||||||
|
|
|
@ -1,65 +0,0 @@
|
||||||
#ifndef _GPXE_IB_GMA_H
|
|
||||||
#define _GPXE_IB_GMA_H
|
|
||||||
|
|
||||||
/** @file
|
|
||||||
*
|
|
||||||
* Infiniband General Management Agent
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
FILE_LICENCE ( GPL2_OR_LATER );
|
|
||||||
|
|
||||||
#include <gpxe/list.h>
|
|
||||||
#include <gpxe/retry.h>
|
|
||||||
#include <gpxe/tables.h>
|
|
||||||
#include <gpxe/infiniband.h>
|
|
||||||
|
|
||||||
struct ib_gma;
|
|
||||||
|
|
||||||
/** A GMA attribute handler */
|
|
||||||
struct ib_gma_handler {
|
|
||||||
/** Management class */
|
|
||||||
uint8_t mgmt_class;
|
|
||||||
/** Management class don't-care bits */
|
|
||||||
uint8_t mgmt_class_ignore;
|
|
||||||
/** Class version */
|
|
||||||
uint8_t class_version;
|
|
||||||
/** Method */
|
|
||||||
uint8_t method;
|
|
||||||
/** Attribute (in network byte order) */
|
|
||||||
uint16_t attr_id;
|
|
||||||
/** Handle attribute
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
* @v mad MAD
|
|
||||||
* @ret response MAD response, or NULL to send no response
|
|
||||||
*/
|
|
||||||
union ib_mad * ( * handle ) ( struct ib_gma *gma, union ib_mad *mad );
|
|
||||||
};
|
|
||||||
|
|
||||||
/** GMA attribute handlers */
|
|
||||||
#define IB_GMA_HANDLERS __table ( struct ib_gma_handler, "ib_gma_handlers" )
|
|
||||||
|
|
||||||
/** Declare a GMA attribute handler */
|
|
||||||
#define __ib_gma_handler __table_entry ( IB_GMA_HANDLERS, 01 )
|
|
||||||
|
|
||||||
/** An Infiniband General Management Agent */
|
|
||||||
struct ib_gma {
|
|
||||||
/** Infiniband device */
|
|
||||||
struct ib_device *ibdev;
|
|
||||||
/** Completion queue */
|
|
||||||
struct ib_completion_queue *cq;
|
|
||||||
/** Queue pair */
|
|
||||||
struct ib_queue_pair *qp;
|
|
||||||
|
|
||||||
/** List of outstanding MAD requests */
|
|
||||||
struct list_head requests;
|
|
||||||
};
|
|
||||||
|
|
||||||
extern int ib_gma_request ( struct ib_gma *gma, union ib_mad *mad,
|
|
||||||
struct ib_address_vector *av, int retry );
|
|
||||||
extern struct ib_gma * ib_create_gma ( struct ib_device *ibdev,
|
|
||||||
enum ib_queue_pair_type type );
|
|
||||||
extern void ib_destroy_gma ( struct ib_gma *gma );
|
|
||||||
|
|
||||||
#endif /* _GPXE_IB_GMA_H */
|
|
|
@ -303,6 +303,16 @@ union ib_sa_data {
|
||||||
#define IB_CM_ATTR_LOAD_ALTERNATE_PATH 0x0019
|
#define IB_CM_ATTR_LOAD_ALTERNATE_PATH 0x0019
|
||||||
#define IB_CM_ATTR_ALTERNATE_PATH_RESPONSE 0x001a
|
#define IB_CM_ATTR_ALTERNATE_PATH_RESPONSE 0x001a
|
||||||
|
|
||||||
|
/** Communication management common fields */
|
||||||
|
struct ib_cm_common {
|
||||||
|
/** Local communication ID */
|
||||||
|
uint32_t local_id;
|
||||||
|
/** Remote communication ID */
|
||||||
|
uint32_t remote_id;
|
||||||
|
/** Reserved */
|
||||||
|
uint8_t reserved[224];
|
||||||
|
} __attribute__ (( packed ));
|
||||||
|
|
||||||
/** A communication management path */
|
/** A communication management path */
|
||||||
struct ib_cm_path {
|
struct ib_cm_path {
|
||||||
/** Local port LID */
|
/** Local port LID */
|
||||||
|
@ -438,6 +448,7 @@ struct ib_cm_ready_to_use {
|
||||||
|
|
||||||
/** A communication management attribute */
|
/** A communication management attribute */
|
||||||
union ib_cm_data {
|
union ib_cm_data {
|
||||||
|
struct ib_cm_common common;
|
||||||
struct ib_cm_connect_request connect_request;
|
struct ib_cm_connect_request connect_request;
|
||||||
struct ib_cm_connect_reject connect_reject;
|
struct ib_cm_connect_reject connect_reject;
|
||||||
struct ib_cm_connect_reply connect_reply;
|
struct ib_cm_connect_reply connect_reply;
|
||||||
|
|
|
@ -11,9 +11,38 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
|
|
||||||
#include <gpxe/infiniband.h>
|
#include <gpxe/infiniband.h>
|
||||||
|
|
||||||
|
struct ib_mad_transaction;
|
||||||
|
|
||||||
|
/** An Infiniband multicast group membership */
|
||||||
|
struct ib_mc_membership {
|
||||||
|
/** Queue pair */
|
||||||
|
struct ib_queue_pair *qp;
|
||||||
|
/** Multicast GID */
|
||||||
|
struct ib_gid gid;
|
||||||
|
/** Multicast group join transaction */
|
||||||
|
struct ib_mad_transaction *madx;
|
||||||
|
/** Handle join success/failure
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v qp Queue pair
|
||||||
|
* @v membership Multicast group membership
|
||||||
|
* @v rc Status code
|
||||||
|
* @v mad Response MAD (or NULL on error)
|
||||||
|
*/
|
||||||
|
void ( * complete ) ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||||
|
struct ib_mc_membership *membership, int rc,
|
||||||
|
union ib_mad *mad );
|
||||||
|
};
|
||||||
|
|
||||||
extern int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
extern int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||||
struct ib_gid *gid );
|
struct ib_mc_membership *membership,
|
||||||
|
struct ib_gid *gid,
|
||||||
|
void ( * joined ) ( struct ib_device *ibdev,
|
||||||
|
struct ib_queue_pair *qp,
|
||||||
|
struct ib_mc_membership *memb,
|
||||||
|
int rc, union ib_mad *mad ) );
|
||||||
|
|
||||||
extern void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
extern void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||||
struct ib_gid *gid );
|
struct ib_mc_membership *membership );
|
||||||
|
|
||||||
#endif /* _GPXE_IB_MCAST_H */
|
#endif /* _GPXE_IB_MCAST_H */
|
||||||
|
|
|
@ -11,6 +11,65 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
|
|
||||||
#include <gpxe/infiniband.h>
|
#include <gpxe/infiniband.h>
|
||||||
|
|
||||||
|
struct ib_mad_transaction;
|
||||||
|
struct ib_path;
|
||||||
|
|
||||||
|
/** Infiniband path operations */
|
||||||
|
struct ib_path_operations {
|
||||||
|
/** Handle path transaction completion
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v path Path
|
||||||
|
* @v rc Status code
|
||||||
|
* @v av Address vector, or NULL on error
|
||||||
|
*/
|
||||||
|
void ( * complete ) ( struct ib_device *ibdev,
|
||||||
|
struct ib_path *path, int rc,
|
||||||
|
struct ib_address_vector *av );
|
||||||
|
};
|
||||||
|
|
||||||
|
/** An Infiniband path */
|
||||||
|
struct ib_path {
|
||||||
|
/** Infiniband device */
|
||||||
|
struct ib_device *ibdev;
|
||||||
|
/** Address vector */
|
||||||
|
struct ib_address_vector av;
|
||||||
|
/** Management transaction */
|
||||||
|
struct ib_mad_transaction *madx;
|
||||||
|
/** Path operations */
|
||||||
|
struct ib_path_operations *op;
|
||||||
|
/** Owner private data */
|
||||||
|
void *owner_priv;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set Infiniband path owner-private data
|
||||||
|
*
|
||||||
|
* @v path Path
|
||||||
|
* @v priv Private data
|
||||||
|
*/
|
||||||
|
static inline __always_inline void
|
||||||
|
ib_path_set_ownerdata ( struct ib_path *path, void *priv ) {
|
||||||
|
path->owner_priv = priv;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get Infiniband path owner-private data
|
||||||
|
*
|
||||||
|
* @v path Path
|
||||||
|
* @ret priv Private data
|
||||||
|
*/
|
||||||
|
static inline __always_inline void *
|
||||||
|
ib_path_get_ownerdata ( struct ib_path *path ) {
|
||||||
|
return path->owner_priv;
|
||||||
|
}
|
||||||
|
|
||||||
|
extern struct ib_path *
|
||||||
|
ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av,
|
||||||
|
struct ib_path_operations *op );
|
||||||
|
extern void ib_destroy_path ( struct ib_device *ibdev,
|
||||||
|
struct ib_path *path );
|
||||||
|
|
||||||
extern int ib_resolve_path ( struct ib_device *ibdev,
|
extern int ib_resolve_path ( struct ib_device *ibdev,
|
||||||
struct ib_address_vector *av );
|
struct ib_address_vector *av );
|
||||||
|
|
||||||
|
|
|
@ -46,7 +46,6 @@ struct ib_queue_pair;
|
||||||
struct ib_address_vector;
|
struct ib_address_vector;
|
||||||
struct ib_completion_queue;
|
struct ib_completion_queue;
|
||||||
struct ib_mad_interface;
|
struct ib_mad_interface;
|
||||||
struct ib_gma;
|
|
||||||
|
|
||||||
/** Infiniband transmission rates */
|
/** Infiniband transmission rates */
|
||||||
enum ib_rate {
|
enum ib_rate {
|
||||||
|
@ -416,8 +415,8 @@ struct ib_device {
|
||||||
|
|
||||||
/** Subnet management interface */
|
/** Subnet management interface */
|
||||||
struct ib_mad_interface *smi;
|
struct ib_mad_interface *smi;
|
||||||
/** General management agent */
|
/** General services interface */
|
||||||
struct ib_gma *gma;
|
struct ib_mad_interface *gsi;
|
||||||
|
|
||||||
/** Driver private data */
|
/** Driver private data */
|
||||||
void *drv_priv;
|
void *drv_priv;
|
||||||
|
|
|
@ -35,7 +35,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
#include <gpxe/infiniband.h>
|
#include <gpxe/infiniband.h>
|
||||||
#include <gpxe/ib_mi.h>
|
#include <gpxe/ib_mi.h>
|
||||||
#include <gpxe/ib_sma.h>
|
#include <gpxe/ib_sma.h>
|
||||||
#include <gpxe/ib_gma.h>
|
|
||||||
|
|
||||||
/** @file
|
/** @file
|
||||||
*
|
*
|
||||||
|
@ -551,12 +550,12 @@ int ib_open ( struct ib_device *ibdev ) {
|
||||||
goto err_create_sma;
|
goto err_create_sma;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Create general management agent */
|
/* Create general services interface */
|
||||||
ibdev->gma = ib_create_gma ( ibdev, IB_QPT_GSI );
|
ibdev->gsi = ib_create_mi ( ibdev, IB_QPT_GSI );
|
||||||
if ( ! ibdev->gma ) {
|
if ( ! ibdev->gsi ) {
|
||||||
DBGC ( ibdev, "IBDEV %p could not create GMA\n", ibdev );
|
DBGC ( ibdev, "IBDEV %p could not create GSI\n", ibdev );
|
||||||
rc = -ENOMEM;
|
rc = -ENOMEM;
|
||||||
goto err_create_gma;
|
goto err_create_gsi;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Open device */
|
/* Open device */
|
||||||
|
@ -571,8 +570,8 @@ int ib_open ( struct ib_device *ibdev ) {
|
||||||
|
|
||||||
ibdev->op->close ( ibdev );
|
ibdev->op->close ( ibdev );
|
||||||
err_open:
|
err_open:
|
||||||
ib_destroy_gma ( ibdev->gma );
|
ib_destroy_mi ( ibdev, ibdev->gsi );
|
||||||
err_create_gma:
|
err_create_gsi:
|
||||||
ib_destroy_sma ( ibdev, ibdev->smi );
|
ib_destroy_sma ( ibdev, ibdev->smi );
|
||||||
err_create_sma:
|
err_create_sma:
|
||||||
ib_destroy_mi ( ibdev, ibdev->smi );
|
ib_destroy_mi ( ibdev, ibdev->smi );
|
||||||
|
@ -594,7 +593,7 @@ void ib_close ( struct ib_device *ibdev ) {
|
||||||
|
|
||||||
/* Close device if this was the last remaining requested opening */
|
/* Close device if this was the last remaining requested opening */
|
||||||
if ( ibdev->open_count == 0 ) {
|
if ( ibdev->open_count == 0 ) {
|
||||||
ib_destroy_gma ( ibdev->gma );
|
ib_destroy_mi ( ibdev, ibdev->gsi );
|
||||||
ib_destroy_sma ( ibdev, ibdev->smi );
|
ib_destroy_sma ( ibdev, ibdev->smi );
|
||||||
ib_destroy_mi ( ibdev, ibdev->smi );
|
ib_destroy_mi ( ibdev, ibdev->smi );
|
||||||
ibdev->op->close ( ibdev );
|
ibdev->op->close ( ibdev );
|
||||||
|
|
|
@ -24,10 +24,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
#include <byteswap.h>
|
#include <byteswap.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <assert.h>
|
#include <assert.h>
|
||||||
#include <gpxe/list.h>
|
|
||||||
#include <gpxe/process.h>
|
|
||||||
#include <gpxe/infiniband.h>
|
#include <gpxe/infiniband.h>
|
||||||
#include <gpxe/ib_gma.h>
|
#include <gpxe/ib_mi.h>
|
||||||
#include <gpxe/ib_pathrec.h>
|
#include <gpxe/ib_pathrec.h>
|
||||||
#include <gpxe/ib_cm.h>
|
#include <gpxe/ib_cm.h>
|
||||||
|
|
||||||
|
@ -38,64 +36,170 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/** An outstanding connection request */
|
|
||||||
struct ib_cm_request {
|
|
||||||
/** List of all outstanding requests */
|
|
||||||
struct list_head list;
|
|
||||||
/** Local communication ID */
|
|
||||||
uint32_t local_id;
|
|
||||||
/** Remote communication ID */
|
|
||||||
uint32_t remote_id;
|
|
||||||
/** Queue pair */
|
|
||||||
struct ib_queue_pair *qp;
|
|
||||||
/** Target service ID */
|
|
||||||
struct ib_gid_half service_id;
|
|
||||||
/** Connection process */
|
|
||||||
struct process process;
|
|
||||||
/** Notification handler
|
|
||||||
*
|
|
||||||
* @v qp Queue pair
|
|
||||||
* @v rc Connection status code
|
|
||||||
* @v private_data Private data
|
|
||||||
* @v private_data_len Length of private data
|
|
||||||
*/
|
|
||||||
void ( * notify ) ( struct ib_queue_pair *qp, int rc,
|
|
||||||
void *private_data, size_t private_data_len );
|
|
||||||
/** Private data length */
|
|
||||||
size_t private_data_len;
|
|
||||||
/** Private data */
|
|
||||||
uint8_t private_data[0];
|
|
||||||
};
|
|
||||||
|
|
||||||
/** List of all outstanding connection requests */
|
|
||||||
static LIST_HEAD ( ib_cm_requests );
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Send connection request
|
* Send "ready to use" response
|
||||||
*
|
*
|
||||||
* @v request Connection request
|
* @v ibdev Infiniband device
|
||||||
|
* @v mi Management interface
|
||||||
|
* @v conn Connection
|
||||||
|
* @v av Address vector
|
||||||
* @ret rc Return status code
|
* @ret rc Return status code
|
||||||
*/
|
*/
|
||||||
static int ib_cm_send_request ( struct ib_cm_request *request ) {
|
static int ib_cm_send_rtu ( struct ib_device *ibdev,
|
||||||
struct ib_queue_pair *qp = request->qp;
|
struct ib_mad_interface *mi,
|
||||||
struct ib_device *ibdev = qp->ibdev;
|
struct ib_connection *conn,
|
||||||
struct ib_gma *gma = ibdev->gma;
|
struct ib_address_vector *av ) {
|
||||||
union ib_mad mad;
|
union ib_mad mad;
|
||||||
struct ib_mad_cm *cm = &mad.cm;
|
struct ib_cm_ready_to_use *ready =
|
||||||
struct ib_cm_connect_request *connect_req =
|
&mad.cm.cm_data.ready_to_use;
|
||||||
&cm->cm_data.connect_request;
|
|
||||||
size_t private_data_len;
|
|
||||||
int rc;
|
int rc;
|
||||||
|
|
||||||
|
/* Construct "ready to use" response */
|
||||||
|
memset ( &mad, 0, sizeof ( mad ) );
|
||||||
|
mad.hdr.mgmt_class = IB_MGMT_CLASS_CM;
|
||||||
|
mad.hdr.class_version = IB_CM_CLASS_VERSION;
|
||||||
|
mad.hdr.method = IB_MGMT_METHOD_SEND;
|
||||||
|
mad.hdr.attr_id = htons ( IB_CM_ATTR_READY_TO_USE );
|
||||||
|
ready->local_id = htonl ( conn->local_id );
|
||||||
|
ready->remote_id = htonl ( conn->remote_id );
|
||||||
|
if ( ( rc = ib_mi_send ( ibdev, mi, &mad, av ) ) != 0 ){
|
||||||
|
DBGC ( conn, "CM %p could not send RTU: %s\n",
|
||||||
|
conn, strerror ( rc ) );
|
||||||
|
return rc;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle connection request transaction completion
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v mi Management interface
|
||||||
|
* @v madx Management transaction
|
||||||
|
* @v rc Status code
|
||||||
|
* @v mad Received MAD (or NULL on error)
|
||||||
|
* @v av Source address vector (or NULL on error)
|
||||||
|
*/
|
||||||
|
static void ib_cm_req_complete ( struct ib_device *ibdev,
|
||||||
|
struct ib_mad_interface *mi,
|
||||||
|
struct ib_mad_transaction *madx,
|
||||||
|
int rc, union ib_mad *mad,
|
||||||
|
struct ib_address_vector *av ) {
|
||||||
|
struct ib_connection *conn = ib_madx_get_ownerdata ( madx );
|
||||||
|
struct ib_queue_pair *qp = conn->qp;
|
||||||
|
struct ib_cm_common *common = &mad->cm.cm_data.common;
|
||||||
|
struct ib_cm_connect_reply *connect_rep =
|
||||||
|
&mad->cm.cm_data.connect_reply;
|
||||||
|
struct ib_cm_connect_reject *connect_rej =
|
||||||
|
&mad->cm.cm_data.connect_reject;
|
||||||
|
void *private_data = NULL;
|
||||||
|
size_t private_data_len = 0;
|
||||||
|
|
||||||
|
/* Report failures */
|
||||||
|
if ( rc != 0 ) {
|
||||||
|
DBGC ( conn, "CM %p connection request failed: %s\n",
|
||||||
|
conn, strerror ( rc ) );
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Record remote communication ID */
|
||||||
|
conn->remote_id = ntohl ( common->local_id );
|
||||||
|
|
||||||
|
/* Handle response */
|
||||||
|
switch ( mad->hdr.attr_id ) {
|
||||||
|
|
||||||
|
case htons ( IB_CM_ATTR_CONNECT_REPLY ) :
|
||||||
|
/* Extract fields */
|
||||||
|
qp->av.qpn = ( ntohl ( connect_rep->local_qpn ) >> 8 );
|
||||||
|
qp->send.psn = ( ntohl ( connect_rep->starting_psn ) >> 8 );
|
||||||
|
private_data = &connect_rep->private_data;
|
||||||
|
private_data_len = sizeof ( connect_rep->private_data );
|
||||||
|
DBGC ( conn, "CM %p connected to QPN %lx PSN %x\n",
|
||||||
|
conn, qp->av.qpn, qp->send.psn );
|
||||||
|
|
||||||
|
/* Modify queue pair */
|
||||||
|
if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
|
||||||
|
DBGC ( conn, "CM %p could not modify queue pair: %s\n",
|
||||||
|
conn, strerror ( rc ) );
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Send "ready to use" reply */
|
||||||
|
if ( ( rc = ib_cm_send_rtu ( ibdev, mi, conn, av ) ) != 0 ) {
|
||||||
|
/* Treat as non-fatal */
|
||||||
|
rc = 0;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case htons ( IB_CM_ATTR_CONNECT_REJECT ) :
|
||||||
|
/* Extract fields */
|
||||||
|
DBGC ( conn, "CM %p connection rejected (reason %d)\n",
|
||||||
|
conn, ntohs ( connect_rej->reason ) );
|
||||||
|
private_data = &connect_rej->private_data;
|
||||||
|
private_data_len = sizeof ( connect_rej->private_data );
|
||||||
|
rc = -ENOTCONN;
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
DBGC ( conn, "CM %p unexpected response (attribute %04x)\n",
|
||||||
|
conn, ntohs ( mad->hdr.attr_id ) );
|
||||||
|
rc = -EIO;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
out:
|
||||||
|
/* Destroy the completed transaction */
|
||||||
|
ib_destroy_madx ( ibdev, ibdev->gsi, madx );
|
||||||
|
conn->madx = NULL;
|
||||||
|
|
||||||
|
/* Hand off to the upper completion handler */
|
||||||
|
conn->op->changed ( ibdev, qp, conn, rc, private_data,
|
||||||
|
private_data_len );
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Connection request operations */
|
||||||
|
static struct ib_mad_transaction_operations ib_cm_req_op = {
|
||||||
|
.complete = ib_cm_req_complete,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle connection path transaction completion
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v path Path
|
||||||
|
* @v rc Status code
|
||||||
|
* @v av Address vector, or NULL on error
|
||||||
|
*/
|
||||||
|
static void ib_cm_path_complete ( struct ib_device *ibdev,
|
||||||
|
struct ib_path *path, int rc,
|
||||||
|
struct ib_address_vector *av ) {
|
||||||
|
struct ib_connection *conn = ib_path_get_ownerdata ( path );
|
||||||
|
struct ib_queue_pair *qp = conn->qp;
|
||||||
|
union ib_mad mad;
|
||||||
|
struct ib_cm_connect_request *connect_req =
|
||||||
|
&mad.cm.cm_data.connect_request;
|
||||||
|
size_t private_data_len;
|
||||||
|
|
||||||
|
/* Report failures */
|
||||||
|
if ( rc != 0 ) {
|
||||||
|
DBGC ( conn, "CM %p path lookup failed: %s\n",
|
||||||
|
conn, strerror ( rc ) );
|
||||||
|
conn->op->changed ( ibdev, qp, conn, rc, NULL, 0 );
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Update queue pair peer path */
|
||||||
|
memcpy ( &qp->av, av, sizeof ( qp->av ) );
|
||||||
|
|
||||||
/* Construct connection request */
|
/* Construct connection request */
|
||||||
memset ( cm, 0, sizeof ( *cm ) );
|
memset ( &mad, 0, sizeof ( mad ) );
|
||||||
cm->mad_hdr.base_version = IB_MGMT_BASE_VERSION;
|
mad.hdr.mgmt_class = IB_MGMT_CLASS_CM;
|
||||||
cm->mad_hdr.mgmt_class = IB_MGMT_CLASS_CM;
|
mad.hdr.class_version = IB_CM_CLASS_VERSION;
|
||||||
cm->mad_hdr.class_version = IB_CM_CLASS_VERSION;
|
mad.hdr.method = IB_MGMT_METHOD_SEND;
|
||||||
cm->mad_hdr.method = IB_MGMT_METHOD_SEND;
|
mad.hdr.attr_id = htons ( IB_CM_ATTR_CONNECT_REQUEST );
|
||||||
cm->mad_hdr.attr_id = htons ( IB_CM_ATTR_CONNECT_REQUEST );
|
connect_req->local_id = htonl ( conn->local_id );
|
||||||
connect_req->local_id = htonl ( request->local_id );
|
memcpy ( &connect_req->service_id, &conn->service_id,
|
||||||
memcpy ( &connect_req->service_id, &request->service_id,
|
|
||||||
sizeof ( connect_req->service_id ) );
|
sizeof ( connect_req->service_id ) );
|
||||||
ib_get_hca_info ( ibdev, &connect_req->local_ca );
|
ib_get_hca_info ( ibdev, &connect_req->local_ca );
|
||||||
connect_req->local_qpn__responder_resources =
|
connect_req->local_qpn__responder_resources =
|
||||||
|
@ -113,217 +217,116 @@ static int ib_cm_send_request ( struct ib_cm_request *request ) {
|
||||||
connect_req->max_cm_retries__srq =
|
connect_req->max_cm_retries__srq =
|
||||||
( ( 0x0f << 4 ) | ( 0 << 3 ) );
|
( ( 0x0f << 4 ) | ( 0 << 3 ) );
|
||||||
connect_req->primary.local_lid = htons ( ibdev->lid );
|
connect_req->primary.local_lid = htons ( ibdev->lid );
|
||||||
connect_req->primary.remote_lid = htons ( request->qp->av.lid );
|
connect_req->primary.remote_lid = htons ( conn->qp->av.lid );
|
||||||
memcpy ( &connect_req->primary.local_gid, &ibdev->gid,
|
memcpy ( &connect_req->primary.local_gid, &ibdev->gid,
|
||||||
sizeof ( connect_req->primary.local_gid ) );
|
sizeof ( connect_req->primary.local_gid ) );
|
||||||
memcpy ( &connect_req->primary.remote_gid, &request->qp->av.gid,
|
memcpy ( &connect_req->primary.remote_gid, &conn->qp->av.gid,
|
||||||
sizeof ( connect_req->primary.remote_gid ) );
|
sizeof ( connect_req->primary.remote_gid ) );
|
||||||
connect_req->primary.flow_label__rate =
|
connect_req->primary.flow_label__rate =
|
||||||
htonl ( ( 0 << 12 ) | ( request->qp->av.rate << 0 ) );
|
htonl ( ( 0 << 12 ) | ( conn->qp->av.rate << 0 ) );
|
||||||
connect_req->primary.hop_limit = 0;
|
connect_req->primary.hop_limit = 0;
|
||||||
connect_req->primary.sl__subnet_local =
|
connect_req->primary.sl__subnet_local =
|
||||||
( ( request->qp->av.sl << 4 ) | ( 1 << 3 ) );
|
( ( conn->qp->av.sl << 4 ) | ( 1 << 3 ) );
|
||||||
connect_req->primary.local_ack_timeout = ( 0x13 << 3 );
|
connect_req->primary.local_ack_timeout = ( 0x13 << 3 );
|
||||||
private_data_len = request->private_data_len;
|
private_data_len = conn->private_data_len;
|
||||||
if ( private_data_len > sizeof ( connect_req->private_data ) )
|
if ( private_data_len > sizeof ( connect_req->private_data ) )
|
||||||
private_data_len = sizeof ( connect_req->private_data );
|
private_data_len = sizeof ( connect_req->private_data );
|
||||||
memcpy ( &connect_req->private_data, &request->private_data,
|
memcpy ( &connect_req->private_data, &conn->private_data,
|
||||||
private_data_len );
|
private_data_len );
|
||||||
|
|
||||||
/* Send request */
|
/* Create connection request */
|
||||||
if ( ( rc = ib_gma_request ( gma, &mad, NULL, 1 ) ) != 0 ) {
|
conn->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
|
||||||
DBGC ( gma, "GMA %p could not send connection request: %s\n",
|
&ib_cm_req_op );
|
||||||
gma, strerror ( rc ) );
|
if ( ! conn->madx ) {
|
||||||
return rc;
|
DBGC ( conn, "CM %p could not create connection request\n",
|
||||||
|
conn );
|
||||||
|
conn->op->changed ( ibdev, qp, conn, rc, NULL, 0 );
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
ib_madx_set_ownerdata ( conn->madx, conn );
|
||||||
|
|
||||||
|
out:
|
||||||
|
/* Destroy the completed transaction */
|
||||||
|
ib_destroy_path ( ibdev, path );
|
||||||
|
conn->path = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
/** Connection path operations */
|
||||||
|
static struct ib_path_operations ib_cm_path_op = {
|
||||||
}
|
.complete = ib_cm_path_complete,
|
||||||
|
|
||||||
/**
|
|
||||||
* Connection request process step
|
|
||||||
*
|
|
||||||
* @v process Connection request process
|
|
||||||
*/
|
|
||||||
static void ib_cm_step ( struct process *process ) {
|
|
||||||
struct ib_cm_request *request =
|
|
||||||
container_of ( process, struct ib_cm_request, process );
|
|
||||||
struct ib_queue_pair *qp = request->qp;
|
|
||||||
struct ib_device *ibdev = qp->ibdev;
|
|
||||||
int rc;
|
|
||||||
|
|
||||||
/* Wait until path can be resolved */
|
|
||||||
if ( ( rc = ib_resolve_path ( ibdev, &request->qp->av ) ) != 0 )
|
|
||||||
return;
|
|
||||||
|
|
||||||
/* Wait until request can be sent */
|
|
||||||
if ( ( rc = ib_cm_send_request ( request ) ) != 0 )
|
|
||||||
return;
|
|
||||||
|
|
||||||
/* Stop process */
|
|
||||||
process_del ( process );
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Identify connection request by communication ID
|
|
||||||
*
|
|
||||||
* @v local_id Local communication ID
|
|
||||||
* @v remote_id Remote communication ID
|
|
||||||
* @ret request Connection request, or NULL
|
|
||||||
*/
|
|
||||||
static struct ib_cm_request * ib_cm_find_request ( uint32_t local_id,
|
|
||||||
uint32_t remote_id ) {
|
|
||||||
struct ib_cm_request *request;
|
|
||||||
|
|
||||||
list_for_each_entry ( request, &ib_cm_requests, list ) {
|
|
||||||
if ( request->local_id == local_id ) {
|
|
||||||
request->remote_id = remote_id;
|
|
||||||
return request;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle connection reply
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
* @v mad MAD
|
|
||||||
* @ret response MAD response
|
|
||||||
*/
|
|
||||||
static union ib_mad * ib_cm_connect_reply ( struct ib_gma *gma,
|
|
||||||
union ib_mad *mad ) {
|
|
||||||
struct ib_cm_connect_reply *connect_rep =
|
|
||||||
&mad->cm.cm_data.connect_reply;
|
|
||||||
struct ib_cm_ready_to_use *ready =
|
|
||||||
&mad->cm.cm_data.ready_to_use;
|
|
||||||
struct ib_cm_request *request;
|
|
||||||
int rc;
|
|
||||||
|
|
||||||
/* Identify request */
|
|
||||||
request = ib_cm_find_request ( ntohl ( connect_rep->remote_id ),
|
|
||||||
ntohl ( connect_rep->local_id ) );
|
|
||||||
if ( ! request ) {
|
|
||||||
DBGC ( gma, "GMA %p received connection reply with unknown "
|
|
||||||
"ID %08x\n", gma, ntohl ( connect_rep->remote_id ) );
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Extract fields */
|
|
||||||
request->qp->av.qpn = ( ntohl ( connect_rep->local_qpn ) >> 8 );
|
|
||||||
request->qp->send.psn = ( ntohl ( connect_rep->starting_psn ) >> 8 );
|
|
||||||
DBGC ( gma, "GMA %p QPN %lx connected to QPN %lx PSN %x\n", gma,
|
|
||||||
request->qp->qpn, request->qp->av.qpn, request->qp->send.psn );
|
|
||||||
|
|
||||||
/* Modify queue pair */
|
|
||||||
if ( ( rc = ib_modify_qp ( request->qp->ibdev, request->qp ) ) != 0 ) {
|
|
||||||
DBGC ( gma, "GMA %p QPN %lx could not modify queue pair: %s\n",
|
|
||||||
gma, request->qp->qpn, strerror ( rc ) );
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Inform recipient that we are now connected */
|
|
||||||
request->notify ( request->qp, 0, &connect_rep->private_data,
|
|
||||||
sizeof ( connect_rep->private_data ) );
|
|
||||||
|
|
||||||
/* Construct ready to use reply */
|
|
||||||
mad->hdr.attr_id = htons ( IB_CM_ATTR_READY_TO_USE );
|
|
||||||
memset ( ready, 0, sizeof ( *ready ) );
|
|
||||||
ready->local_id = htonl ( request->local_id );
|
|
||||||
ready->remote_id = htonl ( request->remote_id );
|
|
||||||
|
|
||||||
return mad;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle connection rejection
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
* @v mad MAD
|
|
||||||
* @ret response MAD response
|
|
||||||
*/
|
|
||||||
static union ib_mad * ib_cm_connect_reject ( struct ib_gma *gma,
|
|
||||||
union ib_mad *mad ) {
|
|
||||||
struct ib_cm_connect_reject *connect_rej =
|
|
||||||
&mad->cm.cm_data.connect_reject;
|
|
||||||
struct ib_cm_request *request;
|
|
||||||
uint16_t reason;
|
|
||||||
|
|
||||||
/* Identify request */
|
|
||||||
request = ib_cm_find_request ( ntohl ( connect_rej->remote_id ),
|
|
||||||
ntohl ( connect_rej->local_id ) );
|
|
||||||
if ( ! request ) {
|
|
||||||
DBGC ( gma, "GMA %p received connection rejection with "
|
|
||||||
"unknown ID %08x\n", gma,
|
|
||||||
ntohl ( connect_rej->remote_id ) );
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Extract fields */
|
|
||||||
reason = ntohs ( connect_rej->reason );
|
|
||||||
DBGC ( gma, "GMA %p QPN %lx connection rejected (reason %d)\n",
|
|
||||||
gma, request->qp->qpn, reason );
|
|
||||||
|
|
||||||
/* Inform recipient that we are now disconnected */
|
|
||||||
request->notify ( request->qp, -ENOTCONN, &connect_rej->private_data,
|
|
||||||
sizeof ( connect_rej->private_data ) );
|
|
||||||
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Communication management MAD handlers */
|
|
||||||
struct ib_gma_handler ib_cm_handlers[] __ib_gma_handler = {
|
|
||||||
{
|
|
||||||
.mgmt_class = IB_MGMT_CLASS_CM,
|
|
||||||
.class_version = IB_CM_CLASS_VERSION,
|
|
||||||
.method = IB_MGMT_METHOD_SEND,
|
|
||||||
.attr_id = htons ( IB_CM_ATTR_CONNECT_REPLY ),
|
|
||||||
.handle = ib_cm_connect_reply,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
.mgmt_class = IB_MGMT_CLASS_CM,
|
|
||||||
.class_version = IB_CM_CLASS_VERSION,
|
|
||||||
.method = IB_MGMT_METHOD_SEND,
|
|
||||||
.attr_id = htons ( IB_CM_ATTR_CONNECT_REJECT ),
|
|
||||||
.handle = ib_cm_connect_reject,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Connect to remote QP
|
* Create connection to remote QP
|
||||||
*
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
* @v qp Queue pair
|
* @v qp Queue pair
|
||||||
* @v dgid Target GID
|
* @v dgid Target GID
|
||||||
* @v service_id Target service ID
|
* @v service_id Target service ID
|
||||||
* @v private_data Private data
|
* @v private_data Connection request private data
|
||||||
* @v private_data_len Length of private data
|
* @v private_data_len Length of connection request private data
|
||||||
* @ret rc Return status code
|
* @v op Connection operations
|
||||||
|
* @ret conn Connection
|
||||||
*/
|
*/
|
||||||
int ib_cm_connect ( struct ib_queue_pair *qp, struct ib_gid *dgid,
|
struct ib_connection *
|
||||||
struct ib_gid_half *service_id,
|
ib_create_conn ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||||
|
struct ib_gid *dgid, struct ib_gid_half *service_id,
|
||||||
void *private_data, size_t private_data_len,
|
void *private_data, size_t private_data_len,
|
||||||
void ( * notify ) ( struct ib_queue_pair *qp, int rc,
|
struct ib_connection_operations *op ) {
|
||||||
void *private_data,
|
struct ib_connection *conn;
|
||||||
size_t private_data_len ) ) {
|
|
||||||
struct ib_cm_request *request;
|
|
||||||
|
|
||||||
/* Allocate and initialise request */
|
/* Allocate and initialise request */
|
||||||
request = zalloc ( sizeof ( *request ) + private_data_len );
|
conn = zalloc ( sizeof ( *conn ) + private_data_len );
|
||||||
if ( ! request )
|
if ( ! conn )
|
||||||
return -ENOMEM;
|
goto err_alloc_conn;
|
||||||
list_add ( &request->list, &ib_cm_requests );
|
conn->ibdev = ibdev;
|
||||||
request->local_id = random();
|
conn->qp = qp;
|
||||||
request->qp = qp;
|
|
||||||
memset ( &qp->av, 0, sizeof ( qp->av ) );
|
memset ( &qp->av, 0, sizeof ( qp->av ) );
|
||||||
qp->av.gid_present = 1;
|
qp->av.gid_present = 1;
|
||||||
memcpy ( &qp->av.gid, dgid, sizeof ( qp->av.gid ) );
|
memcpy ( &qp->av.gid, dgid, sizeof ( qp->av.gid ) );
|
||||||
memcpy ( &request->service_id, service_id,
|
conn->local_id = random();
|
||||||
sizeof ( request->service_id ) );
|
memcpy ( &conn->service_id, service_id, sizeof ( conn->service_id ) );
|
||||||
request->notify = notify;
|
conn->op = op;
|
||||||
request->private_data_len = private_data_len;
|
conn->private_data_len = private_data_len;
|
||||||
memcpy ( &request->private_data, private_data, private_data_len );
|
memcpy ( &conn->private_data, private_data, private_data_len );
|
||||||
process_init ( &request->process, ib_cm_step, NULL );
|
|
||||||
|
|
||||||
return 0;
|
/* Create path */
|
||||||
|
conn->path = ib_create_path ( ibdev, &qp->av, &ib_cm_path_op );
|
||||||
|
if ( ! conn->path )
|
||||||
|
goto err_create_path;
|
||||||
|
ib_path_set_ownerdata ( conn->path, conn );
|
||||||
|
|
||||||
|
DBGC ( conn, "CM %p created for IBDEV %p QPN %lx\n",
|
||||||
|
conn, ibdev, qp->qpn );
|
||||||
|
DBGC ( conn, "CM %p connecting to %08x:%08x:%08x:%08x %08x:%08x\n",
|
||||||
|
conn, ntohl ( dgid->u.dwords[0] ), ntohl ( dgid->u.dwords[1] ),
|
||||||
|
ntohl ( dgid->u.dwords[2] ), ntohl ( dgid->u.dwords[3] ),
|
||||||
|
ntohl ( service_id->u.dwords[0] ),
|
||||||
|
ntohl ( service_id->u.dwords[1] ) );
|
||||||
|
|
||||||
|
return conn;
|
||||||
|
|
||||||
|
ib_destroy_path ( ibdev, conn->path );
|
||||||
|
err_create_path:
|
||||||
|
free ( conn );
|
||||||
|
err_alloc_conn:
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Destroy connection to remote QP
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v qp Queue pair
|
||||||
|
* @v conn Connection
|
||||||
|
*/
|
||||||
|
void ib_destroy_conn ( struct ib_device *ibdev,
|
||||||
|
struct ib_queue_pair *qp __unused,
|
||||||
|
struct ib_connection *conn ) {
|
||||||
|
|
||||||
|
if ( conn->madx )
|
||||||
|
ib_destroy_madx ( ibdev, ibdev->gsi, conn->madx );
|
||||||
|
if ( conn->path )
|
||||||
|
ib_destroy_path ( ibdev, conn->path );
|
||||||
|
free ( conn );
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,403 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>.
|
|
||||||
*
|
|
||||||
* This program is free software; you can redistribute it and/or
|
|
||||||
* modify it under the terms of the GNU General Public License as
|
|
||||||
* published by the Free Software Foundation; either version 2 of the
|
|
||||||
* License, or any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful, but
|
|
||||||
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
||||||
* General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program; if not, write to the Free Software
|
|
||||||
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
|
||||||
*/
|
|
||||||
|
|
||||||
FILE_LICENCE ( GPL2_OR_LATER );
|
|
||||||
|
|
||||||
#include <stdint.h>
|
|
||||||
#include <stdlib.h>
|
|
||||||
#include <string.h>
|
|
||||||
#include <errno.h>
|
|
||||||
#include <stdio.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <byteswap.h>
|
|
||||||
#include <gpxe/infiniband.h>
|
|
||||||
#include <gpxe/iobuf.h>
|
|
||||||
#include <gpxe/ib_gma.h>
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @file
|
|
||||||
*
|
|
||||||
* Infiniband General Management Agent
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
/** A MAD request */
|
|
||||||
struct ib_mad_request {
|
|
||||||
/** Associated GMA */
|
|
||||||
struct ib_gma *gma;
|
|
||||||
/** List of outstanding MAD requests */
|
|
||||||
struct list_head list;
|
|
||||||
/** Retry timer */
|
|
||||||
struct retry_timer timer;
|
|
||||||
/** Destination address */
|
|
||||||
struct ib_address_vector av;
|
|
||||||
/** MAD request */
|
|
||||||
union ib_mad mad;
|
|
||||||
};
|
|
||||||
|
|
||||||
/** GMA number of send WQEs
|
|
||||||
*
|
|
||||||
* This is a policy decision.
|
|
||||||
*/
|
|
||||||
#define IB_GMA_NUM_SEND_WQES 4
|
|
||||||
|
|
||||||
/** GMA number of receive WQEs
|
|
||||||
*
|
|
||||||
* This is a policy decision.
|
|
||||||
*/
|
|
||||||
#define IB_GMA_NUM_RECV_WQES 2
|
|
||||||
|
|
||||||
/** GMA number of completion queue entries
|
|
||||||
*
|
|
||||||
* This is a policy decision
|
|
||||||
*/
|
|
||||||
#define IB_GMA_NUM_CQES 8
|
|
||||||
|
|
||||||
/** TID magic signature */
|
|
||||||
#define IB_GMA_TID_MAGIC ( ( 'g' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )
|
|
||||||
|
|
||||||
/** TID to use for next MAD request */
|
|
||||||
static unsigned int next_request_tid;
|
|
||||||
|
|
||||||
/*****************************************************************************
|
|
||||||
*
|
|
||||||
* General management agent
|
|
||||||
*
|
|
||||||
*****************************************************************************
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Call attribute handler
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
* @v mad MAD
|
|
||||||
* @ret mad MAD response
|
|
||||||
*/
|
|
||||||
static union ib_mad * ib_handle_mad ( struct ib_gma *gma, union ib_mad *mad ) {
|
|
||||||
struct ib_mad_hdr *hdr = &mad->hdr;
|
|
||||||
struct ib_gma_handler *handler;
|
|
||||||
|
|
||||||
for_each_table_entry ( handler, IB_GMA_HANDLERS ) {
|
|
||||||
if ( ( ( handler->mgmt_class & ~handler->mgmt_class_ignore ) ==
|
|
||||||
( hdr->mgmt_class & ~handler->mgmt_class_ignore ) ) &&
|
|
||||||
( handler->class_version == hdr->class_version ) &&
|
|
||||||
( handler->method == hdr->method ) &&
|
|
||||||
( handler->attr_id == hdr->attr_id ) ) {
|
|
||||||
return handler->handle ( gma, mad );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
hdr->method = IB_MGMT_METHOD_TRAP;
|
|
||||||
hdr->status = htons ( IB_MGMT_STATUS_UNSUPPORTED_METHOD_ATTR );
|
|
||||||
return mad;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Complete GMA receive
|
|
||||||
*
|
|
||||||
*
|
|
||||||
* @v ibdev Infiniband device
|
|
||||||
* @v qp Queue pair
|
|
||||||
* @v av Address vector
|
|
||||||
* @v iobuf I/O buffer
|
|
||||||
* @v rc Completion status code
|
|
||||||
*/
|
|
||||||
static void ib_gma_complete_recv ( struct ib_device *ibdev,
|
|
||||||
struct ib_queue_pair *qp,
|
|
||||||
struct ib_address_vector *av,
|
|
||||||
struct io_buffer *iobuf, int rc ) {
|
|
||||||
struct ib_gma *gma = ib_qp_get_ownerdata ( qp );
|
|
||||||
struct ib_mad_request *request;
|
|
||||||
union ib_mad *mad;
|
|
||||||
struct ib_mad_hdr *hdr;
|
|
||||||
union ib_mad *response;
|
|
||||||
|
|
||||||
/* Ignore errors */
|
|
||||||
if ( rc != 0 ) {
|
|
||||||
DBGC ( gma, "GMA %p RX error: %s\n", gma, strerror ( rc ) );
|
|
||||||
goto out;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Sanity checks */
|
|
||||||
if ( iob_len ( iobuf ) != sizeof ( *mad ) ) {
|
|
||||||
DBGC ( gma, "GMA %p RX bad size (%zd bytes)\n",
|
|
||||||
gma, iob_len ( iobuf ) );
|
|
||||||
DBGC_HDA ( gma, 0, iobuf->data, iob_len ( iobuf ) );
|
|
||||||
goto out;
|
|
||||||
}
|
|
||||||
mad = iobuf->data;
|
|
||||||
hdr = &mad->hdr;
|
|
||||||
if ( hdr->base_version != IB_MGMT_BASE_VERSION ) {
|
|
||||||
DBGC ( gma, "GMA %p unsupported base version %x\n",
|
|
||||||
gma, hdr->base_version );
|
|
||||||
DBGC_HDA ( gma, 0, mad, sizeof ( *mad ) );
|
|
||||||
goto out;
|
|
||||||
}
|
|
||||||
DBGC ( gma, "GMA %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status "
|
|
||||||
"%04x\n", gma, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
|
|
||||||
hdr->mgmt_class, hdr->class_version, hdr->method,
|
|
||||||
ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
|
|
||||||
DBGC2_HDA ( gma, 0, mad, sizeof ( *mad ) );
|
|
||||||
|
|
||||||
/* Dequeue request if applicable */
|
|
||||||
list_for_each_entry ( request, &gma->requests, list ) {
|
|
||||||
if ( memcmp ( &request->mad.hdr.tid, &hdr->tid,
|
|
||||||
sizeof ( request->mad.hdr.tid ) ) == 0 ) {
|
|
||||||
stop_timer ( &request->timer );
|
|
||||||
list_del ( &request->list );
|
|
||||||
free ( request );
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Handle MAD */
|
|
||||||
if ( ( response = ib_handle_mad ( gma, mad ) ) == NULL )
|
|
||||||
goto out;
|
|
||||||
|
|
||||||
/* Re-use I/O buffer for response */
|
|
||||||
memcpy ( mad, response, sizeof ( *mad ) );
|
|
||||||
DBGC ( gma, "GMA %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status "
|
|
||||||
"%04x\n", gma, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
|
|
||||||
hdr->mgmt_class, hdr->class_version, hdr->method,
|
|
||||||
ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
|
|
||||||
DBGC2_HDA ( gma, 0, mad, sizeof ( *mad ) );
|
|
||||||
|
|
||||||
/* Send MAD response, if applicable */
|
|
||||||
if ( ( rc = ib_post_send ( ibdev, qp, av,
|
|
||||||
iob_disown ( iobuf ) ) ) != 0 ) {
|
|
||||||
DBGC ( gma, "GMA %p could not send MAD response: %s\n",
|
|
||||||
gma, strerror ( rc ) );
|
|
||||||
goto out;
|
|
||||||
}
|
|
||||||
|
|
||||||
out:
|
|
||||||
free_iob ( iobuf );
|
|
||||||
}
|
|
||||||
|
|
||||||
/** GMA completion operations */
|
|
||||||
static struct ib_completion_queue_operations ib_gma_completion_ops = {
|
|
||||||
.complete_recv = ib_gma_complete_recv,
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Transmit MAD request
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
* @v request MAD request
|
|
||||||
* @ret rc Return status code
|
|
||||||
*/
|
|
||||||
static int ib_gma_send ( struct ib_gma *gma, struct ib_mad_request *request ) {
|
|
||||||
struct io_buffer *iobuf;
|
|
||||||
int rc;
|
|
||||||
|
|
||||||
DBGC ( gma, "GMA %p TX TID %08x%08x (%02x,%02x,%02x,%04x)\n",
|
|
||||||
gma, ntohl ( request->mad.hdr.tid[0] ),
|
|
||||||
ntohl ( request->mad.hdr.tid[1] ), request->mad.hdr.mgmt_class,
|
|
||||||
request->mad.hdr.class_version, request->mad.hdr.method,
|
|
||||||
ntohs ( request->mad.hdr.attr_id ) );
|
|
||||||
DBGC2_HDA ( gma, 0, &request->mad, sizeof ( request->mad ) );
|
|
||||||
|
|
||||||
/* Construct I/O buffer */
|
|
||||||
iobuf = alloc_iob ( sizeof ( request->mad ) );
|
|
||||||
if ( ! iobuf ) {
|
|
||||||
DBGC ( gma, "GMA %p could not allocate buffer for TID "
|
|
||||||
"%08x%08x\n", gma, ntohl ( request->mad.hdr.tid[0] ),
|
|
||||||
ntohl ( request->mad.hdr.tid[1] ) );
|
|
||||||
return -ENOMEM;
|
|
||||||
}
|
|
||||||
memcpy ( iob_put ( iobuf, sizeof ( request->mad ) ), &request->mad,
|
|
||||||
sizeof ( request->mad ) );
|
|
||||||
|
|
||||||
/* Send I/O buffer */
|
|
||||||
if ( ( rc = ib_post_send ( gma->ibdev, gma->qp, &request->av,
|
|
||||||
iobuf ) ) != 0 ) {
|
|
||||||
DBGC ( gma, "GMA %p could not send TID %08x%08x: %s\n",
|
|
||||||
gma, ntohl ( request->mad.hdr.tid[0] ),
|
|
||||||
ntohl ( request->mad.hdr.tid[1] ), strerror ( rc ) );
|
|
||||||
free_iob ( iobuf );
|
|
||||||
return rc;
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle MAD request timer expiry
|
|
||||||
*
|
|
||||||
* @v timer Retry timer
|
|
||||||
* @v expired Failure indicator
|
|
||||||
*/
|
|
||||||
static void ib_gma_timer_expired ( struct retry_timer *timer, int expired ) {
|
|
||||||
struct ib_mad_request *request =
|
|
||||||
container_of ( timer, struct ib_mad_request, timer );
|
|
||||||
struct ib_gma *gma = request->gma;
|
|
||||||
|
|
||||||
/* Abandon TID if we have tried too many times */
|
|
||||||
if ( expired ) {
|
|
||||||
DBGC ( gma, "GMA %p abandoning TID %08x%08x\n",
|
|
||||||
gma, ntohl ( request->mad.hdr.tid[0] ),
|
|
||||||
ntohl ( request->mad.hdr.tid[1] ) );
|
|
||||||
list_del ( &request->list );
|
|
||||||
free ( request );
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Restart retransmission timer */
|
|
||||||
start_timer ( timer );
|
|
||||||
|
|
||||||
/* Resend request */
|
|
||||||
ib_gma_send ( gma, request );
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Issue MAD request
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
* @v mad MAD request
|
|
||||||
* @v av Destination address, or NULL for SM
|
|
||||||
* @v retry Request should be retried until a response arrives
|
|
||||||
* @ret rc Return status code
|
|
||||||
*/
|
|
||||||
int ib_gma_request ( struct ib_gma *gma, union ib_mad *mad,
|
|
||||||
struct ib_address_vector *av, int retry ) {
|
|
||||||
struct ib_device *ibdev = gma->ibdev;
|
|
||||||
struct ib_mad_request *request;
|
|
||||||
|
|
||||||
/* Allocate and initialise structure */
|
|
||||||
request = zalloc ( sizeof ( *request ) );
|
|
||||||
if ( ! request ) {
|
|
||||||
DBGC ( gma, "GMA %p could not allocate MAD request\n", gma );
|
|
||||||
return -ENOMEM;
|
|
||||||
}
|
|
||||||
request->gma = gma;
|
|
||||||
request->timer.expired = ib_gma_timer_expired;
|
|
||||||
|
|
||||||
/* Determine address vector */
|
|
||||||
if ( av ) {
|
|
||||||
memcpy ( &request->av, av, sizeof ( request->av ) );
|
|
||||||
} else {
|
|
||||||
request->av.lid = ibdev->sm_lid;
|
|
||||||
request->av.sl = ibdev->sm_sl;
|
|
||||||
request->av.qpn = IB_QPN_GSI;
|
|
||||||
request->av.qkey = IB_QKEY_GSI;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Copy MAD body */
|
|
||||||
memcpy ( &request->mad, mad, sizeof ( request->mad ) );
|
|
||||||
|
|
||||||
/* Allocate TID */
|
|
||||||
request->mad.hdr.tid[0] = htonl ( IB_GMA_TID_MAGIC );
|
|
||||||
request->mad.hdr.tid[1] = htonl ( ++next_request_tid );
|
|
||||||
|
|
||||||
/* Send initial request. Ignore errors; the retry timer will
|
|
||||||
* take care of those we care about.
|
|
||||||
*/
|
|
||||||
ib_gma_send ( gma, request );
|
|
||||||
|
|
||||||
/* Add to list and start timer if applicable */
|
|
||||||
if ( retry ) {
|
|
||||||
list_add ( &request->list, &gma->requests );
|
|
||||||
start_timer ( &request->timer );
|
|
||||||
} else {
|
|
||||||
free ( request );
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create GMA
|
|
||||||
*
|
|
||||||
* @v ibdev Infiniband device
|
|
||||||
* @v type Queue pair type
|
|
||||||
* @ret gma General management agent, or NULL
|
|
||||||
*/
|
|
||||||
struct ib_gma * ib_create_gma ( struct ib_device *ibdev,
|
|
||||||
enum ib_queue_pair_type type ) {
|
|
||||||
struct ib_gma *gma;
|
|
||||||
int rc;
|
|
||||||
|
|
||||||
/* Allocate and initialise fields */
|
|
||||||
gma = zalloc ( sizeof ( *gma ) );
|
|
||||||
if ( ! gma )
|
|
||||||
goto err_alloc;
|
|
||||||
gma->ibdev = ibdev;
|
|
||||||
INIT_LIST_HEAD ( &gma->requests );
|
|
||||||
|
|
||||||
/* Create completion queue */
|
|
||||||
gma->cq = ib_create_cq ( ibdev, IB_GMA_NUM_CQES,
|
|
||||||
&ib_gma_completion_ops );
|
|
||||||
if ( ! gma->cq ) {
|
|
||||||
DBGC ( gma, "GMA %p could not allocate completion queue\n",
|
|
||||||
gma );
|
|
||||||
goto err_create_cq;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Create queue pair */
|
|
||||||
gma->qp = ib_create_qp ( ibdev, type, IB_GMA_NUM_SEND_WQES, gma->cq,
|
|
||||||
IB_GMA_NUM_RECV_WQES, gma->cq );
|
|
||||||
if ( ! gma->qp ) {
|
|
||||||
DBGC ( gma, "GMA %p could not allocate queue pair\n", gma );
|
|
||||||
goto err_create_qp;
|
|
||||||
}
|
|
||||||
ib_qp_set_ownerdata ( gma->qp, gma );
|
|
||||||
DBGC ( gma, "GMA %p running on QPN %#lx\n", gma, gma->qp->qpn );
|
|
||||||
|
|
||||||
/* Set queue key */
|
|
||||||
gma->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI );
|
|
||||||
if ( ( rc = ib_modify_qp ( ibdev, gma->qp ) ) != 0 ) {
|
|
||||||
DBGC ( gma, "GMA %p could not set queue key: %s\n",
|
|
||||||
gma, strerror ( rc ) );
|
|
||||||
goto err_modify_qp;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Fill receive ring */
|
|
||||||
ib_refill_recv ( ibdev, gma->qp );
|
|
||||||
return gma;
|
|
||||||
|
|
||||||
err_modify_qp:
|
|
||||||
ib_destroy_qp ( ibdev, gma->qp );
|
|
||||||
err_create_qp:
|
|
||||||
ib_destroy_cq ( ibdev, gma->cq );
|
|
||||||
err_create_cq:
|
|
||||||
free ( gma );
|
|
||||||
err_alloc:
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Destroy GMA
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
*/
|
|
||||||
void ib_destroy_gma ( struct ib_gma *gma ) {
|
|
||||||
struct ib_device *ibdev = gma->ibdev;
|
|
||||||
struct ib_mad_request *request;
|
|
||||||
struct ib_mad_request *tmp;
|
|
||||||
|
|
||||||
/* Flush any outstanding requests */
|
|
||||||
list_for_each_entry_safe ( request, tmp, &gma->requests, list ) {
|
|
||||||
stop_timer ( &request->timer );
|
|
||||||
list_del ( &request->list );
|
|
||||||
free ( request );
|
|
||||||
}
|
|
||||||
|
|
||||||
ib_destroy_qp ( ibdev, gma->qp );
|
|
||||||
ib_destroy_cq ( ibdev, gma->cq );
|
|
||||||
free ( gma );
|
|
||||||
}
|
|
|
@ -24,7 +24,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <gpxe/list.h>
|
#include <gpxe/list.h>
|
||||||
#include <gpxe/infiniband.h>
|
#include <gpxe/infiniband.h>
|
||||||
#include <gpxe/ib_gma.h>
|
#include <gpxe/ib_mi.h>
|
||||||
#include <gpxe/ib_mcast.h>
|
#include <gpxe/ib_mcast.h>
|
||||||
|
|
||||||
/** @file
|
/** @file
|
||||||
|
@ -34,22 +34,19 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Transmit multicast group membership request
|
* Generate multicast membership MAD
|
||||||
*
|
*
|
||||||
* @v gma General management agent
|
* @v ibdev Infiniband device
|
||||||
* @v gid Multicast GID
|
* @v gid Multicast GID
|
||||||
* @v join Join (rather than leave) group
|
* @v join Join (rather than leave) group
|
||||||
* @ret rc Return status code
|
* @v mad MAD to fill in
|
||||||
*/
|
*/
|
||||||
static int ib_mc_member_request ( struct ib_gma *gma, struct ib_gid *gid,
|
static void ib_mcast_mad ( struct ib_device *ibdev, struct ib_gid *gid,
|
||||||
int join ) {
|
int join, union ib_mad *mad ) {
|
||||||
union ib_mad mad;
|
struct ib_mad_sa *sa = &mad->sa;
|
||||||
struct ib_mad_sa *sa = &mad.sa;
|
|
||||||
int rc;
|
|
||||||
|
|
||||||
/* Construct multicast membership record request */
|
/* Construct multicast membership record request */
|
||||||
memset ( sa, 0, sizeof ( *sa ) );
|
memset ( sa, 0, sizeof ( *sa ) );
|
||||||
sa->mad_hdr.base_version = IB_MGMT_BASE_VERSION;
|
|
||||||
sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
|
sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
|
||||||
sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
|
sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
|
||||||
sa->mad_hdr.method =
|
sa->mad_hdr.method =
|
||||||
|
@ -61,51 +58,123 @@ static int ib_mc_member_request ( struct ib_gma *gma, struct ib_gid *gid,
|
||||||
sa->sa_data.mc_member_record.scope__join_state = 1;
|
sa->sa_data.mc_member_record.scope__join_state = 1;
|
||||||
memcpy ( &sa->sa_data.mc_member_record.mgid, gid,
|
memcpy ( &sa->sa_data.mc_member_record.mgid, gid,
|
||||||
sizeof ( sa->sa_data.mc_member_record.mgid ) );
|
sizeof ( sa->sa_data.mc_member_record.mgid ) );
|
||||||
memcpy ( &sa->sa_data.mc_member_record.port_gid, &gma->ibdev->gid,
|
memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid,
|
||||||
sizeof ( sa->sa_data.mc_member_record.port_gid ) );
|
sizeof ( sa->sa_data.mc_member_record.port_gid ) );
|
||||||
|
|
||||||
/* Issue multicast membership record request */
|
|
||||||
if ( ( rc = ib_gma_request ( gma, &mad, NULL, join ) ) != 0 ) {
|
|
||||||
DBGC ( gma, "GMA %p could not join group: %s\n",
|
|
||||||
gma, strerror ( rc ) );
|
|
||||||
return rc;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
/**
|
||||||
|
* Handle multicast membership record join response
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v mi Management interface
|
||||||
|
* @v madx Management transaction
|
||||||
|
* @v rc Status code
|
||||||
|
* @v mad Received MAD (or NULL on error)
|
||||||
|
* @v av Source address vector (or NULL on error)
|
||||||
|
*/
|
||||||
|
static void ib_mcast_complete ( struct ib_device *ibdev,
|
||||||
|
struct ib_mad_interface *mi __unused,
|
||||||
|
struct ib_mad_transaction *madx,
|
||||||
|
int rc, union ib_mad *mad,
|
||||||
|
struct ib_address_vector *av __unused ) {
|
||||||
|
struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx );
|
||||||
|
struct ib_queue_pair *qp = membership->qp;
|
||||||
|
struct ib_gid *gid = &membership->gid;
|
||||||
|
struct ib_mc_member_record *mc_member_record =
|
||||||
|
&mad->sa.sa_data.mc_member_record;
|
||||||
|
int joined;
|
||||||
|
unsigned long qkey;
|
||||||
|
|
||||||
|
/* Report failures */
|
||||||
|
if ( rc != 0 ) {
|
||||||
|
DBGC ( ibdev, "IBDEV %p QPN %lx join failed: %s\n",
|
||||||
|
ibdev, qp->qpn, strerror ( rc ) );
|
||||||
|
goto out;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Extract values from MAD */
|
||||||
|
joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP );
|
||||||
|
qkey = ntohl ( mc_member_record->qkey );
|
||||||
|
DBGC ( ibdev, "IBDEV %p QPN %lx %s %08x:%08x:%08x:%08x qkey %lx\n",
|
||||||
|
ibdev, qp->qpn, ( joined ? "joined" : "left" ),
|
||||||
|
ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ),
|
||||||
|
ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ),
|
||||||
|
qkey );
|
||||||
|
|
||||||
|
/* Set queue key */
|
||||||
|
qp->qkey = qkey;
|
||||||
|
if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
|
||||||
|
DBGC ( ibdev, "IBDEV %p QPN %lx could not modify qkey: %s\n",
|
||||||
|
ibdev, qp->qpn, strerror ( rc ) );
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
out:
|
||||||
|
/* Destroy the completed transaction */
|
||||||
|
ib_destroy_madx ( ibdev, mi, madx );
|
||||||
|
membership->madx = NULL;
|
||||||
|
|
||||||
|
/* Hand off to upper completion handler */
|
||||||
|
membership->complete ( ibdev, qp, membership, rc, mad );
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Multicast membership management transaction completion operations */
|
||||||
|
static struct ib_mad_transaction_operations ib_mcast_op = {
|
||||||
|
.complete = ib_mcast_complete,
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Join multicast group
|
* Join multicast group
|
||||||
*
|
*
|
||||||
* @v ibdev Infiniband device
|
* @v ibdev Infiniband device
|
||||||
* @v qp Queue pair
|
* @v qp Queue pair
|
||||||
* @v gid Multicast GID
|
* @v membership Multicast group membership
|
||||||
|
* @v gid Multicast GID to join
|
||||||
|
* @v joined Join completion handler
|
||||||
* @ret rc Return status code
|
* @ret rc Return status code
|
||||||
*/
|
*/
|
||||||
int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||||
struct ib_gid *gid ) {
|
struct ib_mc_membership *membership, struct ib_gid *gid,
|
||||||
struct ib_gma *gma = ibdev->gma;
|
void ( * complete ) ( struct ib_device *ibdev,
|
||||||
|
struct ib_queue_pair *qp,
|
||||||
|
struct ib_mc_membership *membership,
|
||||||
|
int rc, union ib_mad *mad ) ) {
|
||||||
|
union ib_mad mad;
|
||||||
int rc;
|
int rc;
|
||||||
|
|
||||||
DBGC ( gma, "GMA %p QPN %lx joining %08x:%08x:%08x:%08x\n",
|
DBGC ( ibdev, "IBDEV %p QPN %lx joining %08x:%08x:%08x:%08x\n",
|
||||||
gma, qp->qpn, ntohl ( gid->u.dwords[0] ),
|
ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ),
|
||||||
ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
|
ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
|
||||||
ntohl ( gid->u.dwords[3] ) );
|
ntohl ( gid->u.dwords[3] ) );
|
||||||
|
|
||||||
|
/* Initialise structure */
|
||||||
|
membership->qp = qp;
|
||||||
|
memcpy ( &membership->gid, gid, sizeof ( membership->gid ) );
|
||||||
|
membership->complete = complete;
|
||||||
|
|
||||||
/* Attach queue pair to multicast GID */
|
/* Attach queue pair to multicast GID */
|
||||||
if ( ( rc = ib_mcast_attach ( ibdev, qp, gid ) ) != 0 ) {
|
if ( ( rc = ib_mcast_attach ( ibdev, qp, gid ) ) != 0 ) {
|
||||||
DBGC ( gma, "GMA %p could not attach: %s\n",
|
DBGC ( ibdev, "IBDEV %p QPN %lx could not attach: %s\n",
|
||||||
gma, strerror ( rc ) );
|
ibdev, qp->qpn, strerror ( rc ) );
|
||||||
goto err_mcast_attach;
|
goto err_mcast_attach;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Initiate multicast membership join */
|
/* Initiate multicast membership join */
|
||||||
if ( ( rc = ib_mc_member_request ( gma, gid, 1 ) ) != 0 )
|
ib_mcast_mad ( ibdev, gid, 1, &mad );
|
||||||
goto err_mc_member_record;
|
membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
|
||||||
|
&ib_mcast_op );
|
||||||
|
if ( ! membership->madx ) {
|
||||||
|
DBGC ( ibdev, "IBDEV %p QPN %lx could not create join "
|
||||||
|
"transaction\n", ibdev, qp->qpn );
|
||||||
|
rc = -ENOMEM;
|
||||||
|
goto err_create_madx;
|
||||||
|
}
|
||||||
|
ib_madx_set_ownerdata ( membership->madx, membership );
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
err_mc_member_record:
|
ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
|
||||||
|
err_create_madx:
|
||||||
ib_mcast_detach ( ibdev, qp, gid );
|
ib_mcast_detach ( ibdev, qp, gid );
|
||||||
err_mcast_attach:
|
err_mcast_attach:
|
||||||
return rc;
|
return rc;
|
||||||
|
@ -116,121 +185,32 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||||
*
|
*
|
||||||
* @v ibdev Infiniband device
|
* @v ibdev Infiniband device
|
||||||
* @v qp Queue pair
|
* @v qp Queue pair
|
||||||
* @v gid Multicast GID
|
* @v membership Multicast group membership
|
||||||
*/
|
*/
|
||||||
void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||||
struct ib_gid *gid ) {
|
struct ib_mc_membership *membership ) {
|
||||||
struct ib_gma *gma = ibdev->gma;
|
struct ib_gid *gid = &membership->gid;
|
||||||
|
union ib_mad mad;
|
||||||
DBGC ( gma, "GMA %p QPN %lx leaving %08x:%08x:%08x:%08x\n",
|
|
||||||
gma, qp->qpn, ntohl ( gid->u.dwords[0] ),
|
|
||||||
ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
|
|
||||||
ntohl ( gid->u.dwords[3] ) );
|
|
||||||
|
|
||||||
/* Detach queue pair from multicast GID */
|
|
||||||
ib_mcast_detach ( ibdev, qp, gid );
|
|
||||||
|
|
||||||
/* Initiate multicast membership leave */
|
|
||||||
ib_mc_member_request ( gma, gid, 0 );
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle multicast membership record join response
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
* @v mad MAD
|
|
||||||
* @ret mad MAD response
|
|
||||||
*/
|
|
||||||
static union ib_mad * ib_handle_mc_member_join ( struct ib_gma *gma,
|
|
||||||
union ib_mad *mad ) {
|
|
||||||
struct ib_device *ibdev = gma->ibdev;
|
|
||||||
struct ib_mc_member_record *mc_member_record =
|
|
||||||
&mad->sa.sa_data.mc_member_record;
|
|
||||||
struct ib_queue_pair *qp;
|
|
||||||
struct ib_gid *gid;
|
|
||||||
unsigned long qkey;
|
|
||||||
int rc;
|
int rc;
|
||||||
|
|
||||||
/* Ignore if not a success */
|
DBGC ( ibdev, "IBDEV %p QPN %lx leaving %08x:%08x:%08x:%08x\n",
|
||||||
if ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) {
|
ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ),
|
||||||
DBGC ( gma, "GMA %p join failed with status %04x\n",
|
|
||||||
gma, ntohs ( mad->hdr.status ) );
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Extract MAD parameters */
|
|
||||||
gid = &mc_member_record->mgid;
|
|
||||||
qkey = ntohl ( mc_member_record->qkey );
|
|
||||||
|
|
||||||
/* Locate matching queue pair */
|
|
||||||
qp = ib_find_qp_mgid ( ibdev, gid );
|
|
||||||
if ( ! qp ) {
|
|
||||||
DBGC ( gma, "GMA %p has no QP to join %08x:%08x:%08x:%08x\n",
|
|
||||||
gma, ntohl ( gid->u.dwords[0] ),
|
|
||||||
ntohl ( gid->u.dwords[1] ),
|
|
||||||
ntohl ( gid->u.dwords[2] ),
|
|
||||||
ntohl ( gid->u.dwords[3] ) );
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
DBGC ( gma, "GMA %p QPN %lx joined %08x:%08x:%08x:%08x qkey %lx\n",
|
|
||||||
gma, qp->qpn, ntohl ( gid->u.dwords[0] ),
|
|
||||||
ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
|
ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
|
||||||
ntohl ( gid->u.dwords[3] ), qkey );
|
ntohl ( gid->u.dwords[3] ) );
|
||||||
|
|
||||||
/* Set queue key */
|
/* Detach from multicast GID */
|
||||||
qp->qkey = qkey;
|
ib_mcast_detach ( ibdev, qp, &membership->gid );
|
||||||
if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
|
|
||||||
DBGC ( gma, "GMA %p QPN %lx could not modify qkey: %s\n",
|
/* Cancel multicast membership join, if applicable */
|
||||||
gma, qp->qpn, strerror ( rc ) );
|
if ( membership->madx ) {
|
||||||
return NULL;
|
ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
|
||||||
|
membership->madx = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
return NULL;
|
/* Send a single group leave MAD */
|
||||||
|
ib_mcast_mad ( ibdev, &membership->gid, 0, &mad );
|
||||||
|
if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) {
|
||||||
|
DBGC ( ibdev, "IBDEV %p QPN %lx could not send leave request: "
|
||||||
|
"%s\n", ibdev, qp->qpn, strerror ( rc ) );
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle multicast membership record leave response
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
* @v mad MAD
|
|
||||||
* @v response MAD response
|
|
||||||
*/
|
|
||||||
static union ib_mad * ib_handle_mc_member_leave ( struct ib_gma *gma,
|
|
||||||
union ib_mad *mad ) {
|
|
||||||
struct ib_mc_member_record *mc_member_record =
|
|
||||||
&mad->sa.sa_data.mc_member_record;
|
|
||||||
struct ib_gid *gid;
|
|
||||||
|
|
||||||
/* Ignore if not a success */
|
|
||||||
if ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) {
|
|
||||||
DBGC ( gma, "GMA %p leave failed with status %04x\n",
|
|
||||||
gma, ntohs ( mad->hdr.status ) );
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Extract MAD parameters */
|
|
||||||
gid = &mc_member_record->mgid;
|
|
||||||
DBGC ( gma, "GMA %p left %08x:%08x:%08x:%08x\n", gma,
|
|
||||||
ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ),
|
|
||||||
ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ) );
|
|
||||||
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Multicast membership record response handler */
|
|
||||||
struct ib_gma_handler ib_mc_member_record_handlers[] __ib_gma_handler = {
|
|
||||||
{
|
|
||||||
.mgmt_class = IB_MGMT_CLASS_SUBN_ADM,
|
|
||||||
.class_version = IB_SA_CLASS_VERSION,
|
|
||||||
.method = IB_MGMT_METHOD_GET_RESP,
|
|
||||||
.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC ),
|
|
||||||
.handle = ib_handle_mc_member_join,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
.mgmt_class = IB_MGMT_CLASS_SUBN_ADM,
|
|
||||||
.class_version = IB_SA_CLASS_VERSION,
|
|
||||||
.method = IB_SA_METHOD_DELETE_RESP,
|
|
||||||
.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC ),
|
|
||||||
.handle = ib_handle_mc_member_leave,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
|
@ -19,11 +19,12 @@
|
||||||
FILE_LICENCE ( GPL2_OR_LATER );
|
FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <byteswap.h>
|
#include <byteswap.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <gpxe/infiniband.h>
|
#include <gpxe/infiniband.h>
|
||||||
#include <gpxe/ib_gma.h>
|
#include <gpxe/ib_mi.h>
|
||||||
#include <gpxe/ib_pathrec.h>
|
#include <gpxe/ib_pathrec.h>
|
||||||
|
|
||||||
/** @file
|
/** @file
|
||||||
|
@ -32,56 +33,162 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/** Number of path record cache entries
|
/**
|
||||||
|
* Handle path transaction completion
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v mi Management interface
|
||||||
|
* @v madx Management transaction
|
||||||
|
* @v rc Status code
|
||||||
|
* @v mad Received MAD (or NULL on error)
|
||||||
|
* @v av Source address vector (or NULL on error)
|
||||||
|
*/
|
||||||
|
static void ib_path_complete ( struct ib_device *ibdev,
|
||||||
|
struct ib_mad_interface *mi,
|
||||||
|
struct ib_mad_transaction *madx,
|
||||||
|
int rc, union ib_mad *mad,
|
||||||
|
struct ib_address_vector *av __unused ) {
|
||||||
|
struct ib_path *path = ib_madx_get_ownerdata ( madx );
|
||||||
|
struct ib_gid *dgid = &path->av.gid;
|
||||||
|
struct ib_path_record *pathrec = &mad->sa.sa_data.path_record;
|
||||||
|
|
||||||
|
/* Report failures */
|
||||||
|
if ( rc != 0 ) {
|
||||||
|
DBGC ( ibdev, "IBDEV %p path lookup for %08x:%08x:%08x:%08x "
|
||||||
|
"failed: %s\n", ibdev, htonl ( dgid->u.dwords[0] ),
|
||||||
|
htonl ( dgid->u.dwords[1] ),
|
||||||
|
htonl ( dgid->u.dwords[2] ),
|
||||||
|
htonl ( dgid->u.dwords[3] ), strerror ( rc ) );
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Extract values from MAD */
|
||||||
|
path->av.lid = ntohs ( pathrec->dlid );
|
||||||
|
path->av.sl = ( pathrec->reserved__sl & 0x0f );
|
||||||
|
path->av.rate = ( pathrec->rate_selector__rate & 0x3f );
|
||||||
|
DBGC ( ibdev, "IBDEV %p path to %08x:%08x:%08x:%08x is %04x sl %d "
|
||||||
|
"rate %d\n", ibdev, htonl ( dgid->u.dwords[0] ),
|
||||||
|
htonl ( dgid->u.dwords[1] ), htonl ( dgid->u.dwords[2] ),
|
||||||
|
htonl ( dgid->u.dwords[3] ), path->av.lid, path->av.sl,
|
||||||
|
path->av.rate );
|
||||||
|
|
||||||
|
out:
|
||||||
|
/* Destroy the completed transaction */
|
||||||
|
ib_destroy_madx ( ibdev, mi, madx );
|
||||||
|
path->madx = NULL;
|
||||||
|
|
||||||
|
/* Hand off to upper completion handler */
|
||||||
|
path->op->complete ( ibdev, path, rc, &path->av );
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Path transaction completion operations */
|
||||||
|
static struct ib_mad_transaction_operations ib_path_op = {
|
||||||
|
.complete = ib_path_complete,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create path
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v av Address vector to complete
|
||||||
|
* @v op Path operations
|
||||||
|
* @ret path Path
|
||||||
|
*/
|
||||||
|
struct ib_path *
|
||||||
|
ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av,
|
||||||
|
struct ib_path_operations *op ) {
|
||||||
|
struct ib_path *path;
|
||||||
|
union ib_mad mad;
|
||||||
|
struct ib_mad_sa *sa = &mad.sa;
|
||||||
|
|
||||||
|
/* Allocate and initialise structure */
|
||||||
|
path = zalloc ( sizeof ( *path ) );
|
||||||
|
if ( ! path )
|
||||||
|
goto err_alloc_path;
|
||||||
|
path->ibdev = ibdev;
|
||||||
|
memcpy ( &path->av, av, sizeof ( path->av ) );
|
||||||
|
path->op = op;
|
||||||
|
|
||||||
|
/* Construct path request */
|
||||||
|
memset ( sa, 0, sizeof ( *sa ) );
|
||||||
|
sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
|
||||||
|
sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
|
||||||
|
sa->mad_hdr.method = IB_MGMT_METHOD_GET;
|
||||||
|
sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC );
|
||||||
|
sa->sa_hdr.comp_mask[1] =
|
||||||
|
htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
|
||||||
|
memcpy ( &sa->sa_data.path_record.dgid, &path->av.gid,
|
||||||
|
sizeof ( sa->sa_data.path_record.dgid ) );
|
||||||
|
memcpy ( &sa->sa_data.path_record.sgid, &ibdev->gid,
|
||||||
|
sizeof ( sa->sa_data.path_record.sgid ) );
|
||||||
|
|
||||||
|
/* Create management transaction */
|
||||||
|
path->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
|
||||||
|
&ib_path_op );
|
||||||
|
if ( ! path->madx )
|
||||||
|
goto err_create_madx;
|
||||||
|
ib_madx_set_ownerdata ( path->madx, path );
|
||||||
|
|
||||||
|
return path;
|
||||||
|
|
||||||
|
ib_destroy_madx ( ibdev, ibdev->gsi, path->madx );
|
||||||
|
err_create_madx:
|
||||||
|
free ( path );
|
||||||
|
err_alloc_path:
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Destroy path
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v path Path
|
||||||
|
*/
|
||||||
|
void ib_destroy_path ( struct ib_device *ibdev, struct ib_path *path ) {
|
||||||
|
|
||||||
|
if ( path->madx )
|
||||||
|
ib_destroy_madx ( ibdev, ibdev->gsi, path->madx );
|
||||||
|
free ( path );
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Number of path cache entries
|
||||||
*
|
*
|
||||||
* Must be a power of two.
|
* Must be a power of two.
|
||||||
*/
|
*/
|
||||||
#define IB_NUM_CACHED_PATHS 4
|
#define IB_NUM_CACHED_PATHS 4
|
||||||
|
|
||||||
/** A path record cache entry */
|
/** A cached path */
|
||||||
struct ib_cached_path_record {
|
struct ib_cached_path {
|
||||||
/** Infiniband device's port GID
|
/** Path */
|
||||||
*
|
struct ib_path *path;
|
||||||
* Used to disambiguate cache entries when we have multiple
|
|
||||||
* Infiniband devices, without having to maintain a pointer to
|
|
||||||
* the Infiniband device.
|
|
||||||
*/
|
|
||||||
struct ib_gid sgid;
|
|
||||||
/** Destination GID */
|
|
||||||
struct ib_gid dgid;
|
|
||||||
/** Destination LID */
|
|
||||||
unsigned int dlid;
|
|
||||||
/** Rate */
|
|
||||||
unsigned int rate;
|
|
||||||
/** Service level */
|
|
||||||
unsigned int sl;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Path record cache */
|
/** Path cache */
|
||||||
static struct ib_cached_path_record ib_path_cache[IB_NUM_CACHED_PATHS];
|
static struct ib_cached_path ib_path_cache[IB_NUM_CACHED_PATHS];
|
||||||
|
|
||||||
/** Oldest path record cache entry index */
|
/** Oldest path cache entry index */
|
||||||
static unsigned int ib_path_cache_idx;
|
static unsigned int ib_path_cache_idx;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Find path record cache entry
|
* Find path cache entry
|
||||||
*
|
*
|
||||||
* @v ibdev Infiniband device
|
* @v ibdev Infiniband device
|
||||||
* @v dgid Destination GID
|
* @v dgid Destination GID
|
||||||
* @ret cached Path record cache entry, or NULL
|
* @ret path Path cache entry, or NULL
|
||||||
*/
|
*/
|
||||||
static struct ib_cached_path_record *
|
static struct ib_cached_path *
|
||||||
ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) {
|
ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) {
|
||||||
struct ib_cached_path_record *cached;
|
struct ib_cached_path *cached;
|
||||||
unsigned int i;
|
unsigned int i;
|
||||||
|
|
||||||
for ( i = 0 ; i < IB_NUM_CACHED_PATHS ; i++ ) {
|
for ( i = 0 ; i < IB_NUM_CACHED_PATHS ; i++ ) {
|
||||||
cached = &ib_path_cache[i];
|
cached = &ib_path_cache[i];
|
||||||
if ( memcmp ( &cached->sgid, &ibdev->gid,
|
if ( ! cached->path )
|
||||||
sizeof ( cached->sgid ) ) != 0 )
|
|
||||||
continue;
|
continue;
|
||||||
if ( memcmp ( &cached->dgid, dgid,
|
if ( cached->path->ibdev != ibdev )
|
||||||
sizeof ( cached->dgid ) ) != 0 )
|
continue;
|
||||||
|
if ( memcmp ( &cached->path->av.gid, dgid,
|
||||||
|
sizeof ( cached->path->av.gid ) ) != 0 )
|
||||||
continue;
|
continue;
|
||||||
return cached;
|
return cached;
|
||||||
}
|
}
|
||||||
|
@ -90,134 +197,98 @@ ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve path record
|
* Handle cached path transaction completion
|
||||||
|
*
|
||||||
|
* @v ibdev Infiniband device
|
||||||
|
* @v path Path
|
||||||
|
* @v rc Status code
|
||||||
|
* @v av Address vector, or NULL on error
|
||||||
|
*/
|
||||||
|
static void ib_cached_path_complete ( struct ib_device *ibdev,
|
||||||
|
struct ib_path *path, int rc,
|
||||||
|
struct ib_address_vector *av __unused ) {
|
||||||
|
struct ib_cached_path *cached = ib_path_get_ownerdata ( path );
|
||||||
|
|
||||||
|
/* If the transaction failed, erase the cache entry */
|
||||||
|
if ( rc != 0 ) {
|
||||||
|
/* Destroy the old cache entry */
|
||||||
|
ib_destroy_path ( ibdev, path );
|
||||||
|
memset ( cached, 0, sizeof ( *cached ) );
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Do not destroy the completed transaction; we still need to
|
||||||
|
* refer to the resolved path.
|
||||||
|
*/
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Cached path transaction completion operations */
|
||||||
|
static struct ib_path_operations ib_cached_path_op = {
|
||||||
|
.complete = ib_cached_path_complete,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve path
|
||||||
*
|
*
|
||||||
* @v ibdev Infiniband device
|
* @v ibdev Infiniband device
|
||||||
* @v av Address vector to complete
|
* @v av Address vector to complete
|
||||||
* @ret rc Return status code
|
* @ret rc Return status code
|
||||||
|
*
|
||||||
|
* This provides a non-transactional way to resolve a path, via a
|
||||||
|
* cache similar to ARP.
|
||||||
*/
|
*/
|
||||||
int ib_resolve_path ( struct ib_device *ibdev,
|
int ib_resolve_path ( struct ib_device *ibdev, struct ib_address_vector *av ) {
|
||||||
struct ib_address_vector *av ) {
|
|
||||||
struct ib_gma *gma = ibdev->gma;
|
|
||||||
struct ib_gid *gid = &av->gid;
|
struct ib_gid *gid = &av->gid;
|
||||||
struct ib_cached_path_record *cached;
|
struct ib_cached_path *cached;
|
||||||
union ib_mad mad;
|
|
||||||
struct ib_mad_sa *sa = &mad.sa;
|
|
||||||
unsigned int cache_idx;
|
unsigned int cache_idx;
|
||||||
int rc;
|
|
||||||
|
|
||||||
/* Sanity check */
|
/* Sanity check */
|
||||||
if ( ! av->gid_present ) {
|
if ( ! av->gid_present ) {
|
||||||
DBGC ( gma, "GMA %p attempt to look up path record "
|
DBGC ( ibdev, "IBDEV %p attempt to look up path "
|
||||||
"without GID\n", gma );
|
"without GID\n", ibdev );
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Look in cache for a matching entry */
|
/* Look in cache for a matching entry */
|
||||||
cached = ib_find_path_cache_entry ( ibdev, gid );
|
cached = ib_find_path_cache_entry ( ibdev, gid );
|
||||||
if ( cached && cached->dlid ) {
|
if ( cached && cached->path->av.lid ) {
|
||||||
/* Populated entry found */
|
/* Populated entry found */
|
||||||
av->lid = cached->dlid;
|
av->lid = cached->path->av.lid;
|
||||||
av->rate = cached->rate;
|
av->rate = cached->path->av.rate;
|
||||||
av->sl = cached->sl;
|
av->sl = cached->path->av.sl;
|
||||||
DBGC2 ( gma, "GMA %p cache hit for %08x:%08x:%08x:%08x\n",
|
DBGC2 ( ibdev, "IBDEV %p cache hit for %08x:%08x:%08x:%08x\n",
|
||||||
gma, htonl ( gid->u.dwords[0] ),
|
ibdev, htonl ( gid->u.dwords[0] ),
|
||||||
htonl ( gid->u.dwords[1] ), htonl ( gid->u.dwords[2] ),
|
htonl ( gid->u.dwords[1] ), htonl ( gid->u.dwords[2] ),
|
||||||
htonl ( gid->u.dwords[3] ) );
|
htonl ( gid->u.dwords[3] ) );
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
DBGC ( gma, "GMA %p cache miss for %08x:%08x:%08x:%08x%s\n", gma,
|
DBGC ( ibdev, "IBDEV %p cache miss for %08x:%08x:%08x:%08x%s\n",
|
||||||
htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ),
|
ibdev, htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ),
|
||||||
htonl ( gid->u.dwords[2] ), htonl ( gid->u.dwords[3] ),
|
htonl ( gid->u.dwords[2] ), htonl ( gid->u.dwords[3] ),
|
||||||
( cached ? " (in progress)" : "" ) );
|
( cached ? " (in progress)" : "" ) );
|
||||||
|
|
||||||
/* If no unresolved entry was found, then create a new one */
|
/* If lookup is already in progress, do nothing */
|
||||||
if ( ! cached ) {
|
if ( cached )
|
||||||
|
return -ENOENT;
|
||||||
|
|
||||||
|
/* Locate a new cache entry to use */
|
||||||
cache_idx = ( (ib_path_cache_idx++) % IB_NUM_CACHED_PATHS );
|
cache_idx = ( (ib_path_cache_idx++) % IB_NUM_CACHED_PATHS );
|
||||||
cached = &ib_path_cache[cache_idx];
|
cached = &ib_path_cache[cache_idx];
|
||||||
|
|
||||||
|
/* Destroy the old cache entry */
|
||||||
|
if ( cached->path )
|
||||||
|
ib_destroy_path ( ibdev, cached->path );
|
||||||
memset ( cached, 0, sizeof ( *cached ) );
|
memset ( cached, 0, sizeof ( *cached ) );
|
||||||
memcpy ( &cached->sgid, &ibdev->gid, sizeof ( cached->sgid ) );
|
|
||||||
memcpy ( &cached->dgid, gid, sizeof ( cached->dgid ) );
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Construct path record request */
|
/* Create new path */
|
||||||
memset ( sa, 0, sizeof ( *sa ) );
|
cached->path = ib_create_path ( ibdev, av, &ib_cached_path_op );
|
||||||
sa->mad_hdr.base_version = IB_MGMT_BASE_VERSION;
|
if ( ! cached->path ) {
|
||||||
sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
|
DBGC ( ibdev, "IBDEV %p could not create path\n",
|
||||||
sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
|
ibdev );
|
||||||
sa->mad_hdr.method = IB_MGMT_METHOD_GET;
|
return -ENOMEM;
|
||||||
sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC );
|
|
||||||
sa->sa_hdr.comp_mask[1] =
|
|
||||||
htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
|
|
||||||
memcpy ( &sa->sa_data.path_record.dgid, &cached->dgid,
|
|
||||||
sizeof ( sa->sa_data.path_record.dgid ) );
|
|
||||||
memcpy ( &sa->sa_data.path_record.sgid, &cached->sgid,
|
|
||||||
sizeof ( sa->sa_data.path_record.sgid ) );
|
|
||||||
|
|
||||||
/* Issue path record request */
|
|
||||||
if ( ( rc = ib_gma_request ( gma, &mad, NULL, 1 ) ) != 0 ) {
|
|
||||||
DBGC ( gma, "GMA %p could not get path record: %s\n",
|
|
||||||
gma, strerror ( rc ) );
|
|
||||||
return rc;
|
|
||||||
}
|
}
|
||||||
|
ib_path_set_ownerdata ( cached->path, cached );
|
||||||
|
|
||||||
/* Not found yet */
|
/* Not found yet */
|
||||||
return -ENOENT;
|
return -ENOENT;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle path record response
|
|
||||||
*
|
|
||||||
* @v gma General management agent
|
|
||||||
* @v mad MAD
|
|
||||||
* @ret response MAD response
|
|
||||||
*/
|
|
||||||
static union ib_mad * ib_handle_path_record ( struct ib_gma *gma,
|
|
||||||
union ib_mad *mad ) {
|
|
||||||
struct ib_device *ibdev = gma->ibdev;
|
|
||||||
struct ib_path_record *path_record = &mad->sa.sa_data.path_record;
|
|
||||||
struct ib_gid *dgid = &path_record->dgid;
|
|
||||||
struct ib_cached_path_record *cached;
|
|
||||||
unsigned int dlid;
|
|
||||||
unsigned int sl;
|
|
||||||
unsigned int rate;
|
|
||||||
|
|
||||||
/* Ignore if not a success */
|
|
||||||
if ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) {
|
|
||||||
DBGC ( gma, "GMA %p path record lookup failed with status "
|
|
||||||
"%04x\n", gma, ntohs ( mad->hdr.status ) );
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Extract values from MAD */
|
|
||||||
dlid = ntohs ( path_record->dlid );
|
|
||||||
sl = ( path_record->reserved__sl & 0x0f );
|
|
||||||
rate = ( path_record->rate_selector__rate & 0x3f );
|
|
||||||
DBGC ( gma, "GMA %p path to %08x:%08x:%08x:%08x is %04x sl %d "
|
|
||||||
"rate %d\n", gma, htonl ( dgid->u.dwords[0] ),
|
|
||||||
htonl ( dgid->u.dwords[1] ), htonl ( dgid->u.dwords[2] ),
|
|
||||||
htonl ( dgid->u.dwords[3] ), dlid, sl, rate );
|
|
||||||
|
|
||||||
/* Look for a matching cache entry to fill in */
|
|
||||||
if ( ( cached = ib_find_path_cache_entry ( ibdev, dgid ) ) != NULL ) {
|
|
||||||
DBGC ( gma, "GMA %p cache add for %08x:%08x:%08x:%08x\n",
|
|
||||||
gma, htonl ( dgid->u.dwords[0] ),
|
|
||||||
htonl ( dgid->u.dwords[1] ),
|
|
||||||
htonl ( dgid->u.dwords[2] ),
|
|
||||||
htonl ( dgid->u.dwords[3] ) );
|
|
||||||
cached->dlid = dlid;
|
|
||||||
cached->rate = rate;
|
|
||||||
cached->sl = sl;
|
|
||||||
}
|
|
||||||
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Path record response handler */
|
|
||||||
struct ib_gma_handler ib_path_record_handler __ib_gma_handler = {
|
|
||||||
.mgmt_class = IB_MGMT_CLASS_SUBN_ADM,
|
|
||||||
.class_version = IB_SA_CLASS_VERSION,
|
|
||||||
.method = IB_MGMT_METHOD_GET_RESP,
|
|
||||||
.attr_id = htons ( IB_SA_ATTR_PATH_REC ),
|
|
||||||
.handle = ib_handle_path_record,
|
|
||||||
};
|
|
||||||
|
|
Reference in New Issue