From 3c6a6bdc5d78ff8e1ee2ff190183bbea33f0579f Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Mon, 17 Sep 2007 00:24:44 +0100 Subject: [PATCH] Multicast join now works. --- src/drivers/net/mlx_ipoib/arbel.h | 25 ++++- src/drivers/net/mlx_ipoib/mt25218.c | 136 ++++++++++++++++++++++++++++ src/include/gpxe/infiniband.h | 47 ++++++++++ 3 files changed, 204 insertions(+), 4 deletions(-) diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h index cd6a48eb..c4b536a5 100644 --- a/src/drivers/net/mlx_ipoib/arbel.h +++ b/src/drivers/net/mlx_ipoib/arbel.h @@ -33,6 +33,9 @@ #define ARBEL_HCR_INIT2RTR_QPEE 0x001a #define ARBEL_HCR_RTR2RTS_QPEE 0x001b #define ARBEL_HCR_2RST_QPEE 0x0021 +#define ARBEL_HCR_READ_MGM 0x0025 +#define ARBEL_HCR_WRITE_MGM 0x0026 +#define ARBEL_HCR_MGID_HASH 0x0027 /* Service types */ #define ARBEL_ST_UD 0x03 @@ -42,6 +45,17 @@ #define ARBEL_INVALID_LKEY 0x00000100UL +/* + * Datatypes that seem to be missing from the autogenerated documentation + * + */ +struct arbelprm_mgm_hash_st { + pseudo_bit_t reserved0[0x00020]; +/* -------------- */ + pseudo_bit_t hash[0x00010]; + pseudo_bit_t reserved1[0x00010]; +}; + /* * Wrapper structures for hardware datatypes * @@ -53,6 +67,8 @@ struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error ); struct MLX_DECLARE_STRUCT ( arbelprm_cq_arm_db_record ); struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record ); struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register ); +struct MLX_DECLARE_STRUCT ( arbelprm_mgm_entry ); +struct MLX_DECLARE_STRUCT ( arbelprm_mgm_hash ); struct MLX_DECLARE_STRUCT ( arbelprm_qp_db_record ); struct MLX_DECLARE_STRUCT ( arbelprm_qp_ee_state_transitions ); struct MLX_DECLARE_STRUCT ( arbelprm_query_dev_lim ); @@ -266,7 +282,8 @@ struct arbel { #define ARBEL_HCR_OUT_LEN( _command ) ( ( (_command) >> 21 ) & 0x7fc ) /** Build HCR command from component parts */ -#define ARBEL_HCR_CMD( _opcode, _in_mbox, _in_len, _out_mbox, _out_len ) \ +#define ARBEL_HCR_INOUT_CMD( _opcode, _in_mbox, _in_len, \ + _out_mbox, _out_len ) \ ( (_opcode) | \ ( (_in_mbox) ? ARBEL_HCR_IN_MBOX : 0 ) | \ ( ( (_in_len) / 4 ) << 14 ) | \ @@ -274,13 +291,13 @@ struct arbel { ( ( (_out_len) / 4 ) << 23 ) ) #define ARBEL_HCR_IN_CMD( _opcode, _in_mbox, _in_len ) \ - ARBEL_HCR_CMD ( _opcode, _in_mbox, _in_len, 0, 0 ) + ARBEL_HCR_INOUT_CMD ( _opcode, _in_mbox, _in_len, 0, 0 ) #define ARBEL_HCR_OUT_CMD( _opcode, _out_mbox, _out_len ) \ - ARBEL_HCR_CMD ( _opcode, 0, 0, _out_mbox, _out_len ) + ARBEL_HCR_INOUT_CMD ( _opcode, 0, 0, _out_mbox, _out_len ) #define ARBEL_HCR_VOID_CMD( _opcode ) \ - ARBEL_HCR_CMD ( _opcode, 0, 0, 0, 0 ) + ARBEL_HCR_INOUT_CMD ( _opcode, 0, 0, 0, 0 ) /* * Doorbell record allocation diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index 6cef5927..13b7d78b 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -549,6 +549,34 @@ arbel_cmd_2rst_qpee ( struct arbel *arbel, unsigned long qpn ) { 0x03, NULL, qpn, NULL ); } +static inline int +arbel_cmd_read_mgm ( struct arbel *arbel, unsigned int index, + struct arbelprm_mgm_entry *mgm ) { + return arbel_cmd ( arbel, + ARBEL_HCR_OUT_CMD ( ARBEL_HCR_READ_MGM, + 1, sizeof ( *mgm ) ), + 0, NULL, index, mgm ); +} + +static inline int +arbel_cmd_write_mgm ( struct arbel *arbel, unsigned int index, + const struct arbelprm_mgm_entry *mgm ) { + return arbel_cmd ( arbel, + ARBEL_HCR_IN_CMD ( ARBEL_HCR_WRITE_MGM, + 1, sizeof ( *mgm ) ), + 0, mgm, index, NULL ); +} + +static inline int +arbel_cmd_mgid_hash ( struct arbel *arbel, const struct ib_gid *gid, + struct arbelprm_mgm_hash *hash ) { + return arbel_cmd ( arbel, + ARBEL_HCR_INOUT_CMD ( ARBEL_HCR_MGID_HASH, + 1, sizeof ( *gid ), + 0, sizeof ( *hash ) ), + 0, gid, 0, hash ); +} + /*************************************************************************** * * Completion queue operations @@ -1253,6 +1281,104 @@ static void arbel_poll_cq ( struct ib_device *ibdev, } } +/*************************************************************************** + * + * Multicast group operations + * + *************************************************************************** + */ + +/** + * Attach to multicast group + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v gid Multicast GID + * @ret rc Return status code + */ +static int arbel_mcast_attach ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct ib_gid *gid ) { + struct arbel *arbel = ibdev->dev_priv; + struct arbelprm_mgm_hash hash; + struct arbelprm_mgm_entry mgm; + unsigned int index; + int rc; + + /* Generate hash table index */ + if ( ( rc = arbel_cmd_mgid_hash ( arbel, gid, &hash ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not hash GID: %s\n", + arbel, strerror ( rc ) ); + return rc; + } + index = MLX_GET ( &hash, hash ); + + /* Check for existing hash table entry */ + if ( ( rc = arbel_cmd_read_mgm ( arbel, index, &mgm ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not read MGM %#x: %s\n", + arbel, index, strerror ( rc ) ); + return rc; + } + if ( MLX_GET ( &mgm, mgmqp_0.qi ) != 0 ) { + /* FIXME: this implementation allows only a single QP + * per multicast group, and doesn't handle hash + * collisions. Sufficient for IPoIB but may need to + * be extended in future. + */ + DBGC ( arbel, "Arbel %p MGID index %#x already in use\n", + arbel, index ); + return -EBUSY; + } + + /* Update hash table entry */ + MLX_FILL_2 ( &mgm, 8, + mgmqp_0.qpn_i, qp->qpn, + mgmqp_0.qi, 1 ); + memcpy ( &mgm.u.dwords[4], gid, sizeof ( *gid ) ); + if ( ( rc = arbel_cmd_write_mgm ( arbel, index, &mgm ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not write MGM %#x: %s\n", + arbel, index, strerror ( rc ) ); + return rc; + } + + return 0; +} + +/** + * Detach from multicast group + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v gid Multicast GID + */ +static void arbel_mcast_detach ( struct ib_device *ibdev, + struct ib_queue_pair *qp __unused, + struct ib_gid *gid ) { + struct arbel *arbel = ibdev->dev_priv; + struct arbelprm_mgm_hash hash; + struct arbelprm_mgm_entry mgm; + unsigned int index; + int rc; + + /* Generate hash table index */ + if ( ( rc = arbel_cmd_mgid_hash ( arbel, gid, &hash ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not hash GID: %s\n", + arbel, strerror ( rc ) ); + return; + } + index = MLX_GET ( &hash, hash ); + + /* Clear hash table entry */ + memset ( &mgm, 0, sizeof ( mgm ) ); + if ( ( rc = arbel_cmd_write_mgm ( arbel, index, &mgm ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not write MGM %#x: %s\n", + arbel, index, strerror ( rc ) ); + return; + } +} + + + /** Arbel Infiniband operations */ static struct ib_device_operations arbel_ib_operations = { .create_cq = arbel_create_cq, @@ -1262,6 +1388,8 @@ static struct ib_device_operations arbel_ib_operations = { .post_send = arbel_post_send, .post_recv = arbel_post_recv, .poll_cq = arbel_poll_cq, + .mcast_attach = arbel_mcast_attach, + .mcast_detach = arbel_mcast_detach, }; /** @@ -1379,6 +1507,14 @@ static int arbel_probe ( struct pci_device *pci, return -EIO; } mlx->own_qp->owner_priv = netdev; + struct ib_gid *bcast_gid = ( struct ib_gid * ) &ib_data.bcast_gid; + if ( ( rc = ib_mcast_attach ( ibdev, mlx->own_qp, + bcast_gid ) ) != 0 ) { + DBG ( "Could not attach to broadcast GID: %s\n", + strerror ( rc ) ); + return rc; + } + mac = ( ( struct ib_mac * ) netdev->ll_addr ); mac->qpn = htonl ( mlx->own_qp->qpn ); diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index 632a214e..6a38a1b8 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -252,6 +252,27 @@ struct ib_device_operations { struct ib_completion_queue *cq, ib_completer_t complete_send, ib_completer_t complete_recv ); + /** + * Attach to multicast group + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v gid Multicast GID + * @ret rc Return status code + */ + int ( * mcast_attach ) ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct ib_gid *gid ); + /** + * Detach from multicast group + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v gid Multicast GID + */ + void ( * mcast_detach ) ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct ib_gid *gid ); }; /** An Infiniband device */ @@ -275,6 +296,32 @@ extern void ib_destroy_qp ( struct ib_device *ibdev, extern struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq, unsigned long qpn, int is_send ); +/** + * Attach to multicast group + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v gid Multicast GID + * @ret rc Return status code + */ +static inline __attribute__ (( always_inline )) int +ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp, + struct ib_gid *gid ) { + return ibdev->op->mcast_attach ( ibdev, qp, gid ); +} + +/** + * Detach from multicast group + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v gid Multicast GID + */ +static inline __attribute__ (( always_inline )) void +ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp, + struct ib_gid *gid ) { + ibdev->op->mcast_detach ( ibdev, qp, gid ); +} extern struct ll_protocol infiniband_protocol;