From 8423b0b109bd611fd3fe39725b14142c26a3d156 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Tue, 18 Sep 2007 05:39:34 +0100 Subject: [PATCH] Gets most of the way through initialisation. --- src/drivers/net/mlx_ipoib/arbel.h | 11 +- src/drivers/net/mlx_ipoib/ib_mt25218.c | 2 +- src/drivers/net/mlx_ipoib/mt25218.c | 164 ++++++++++++++++++++++--- 3 files changed, 156 insertions(+), 21 deletions(-) diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h index 2755182f..dcca5107 100644 --- a/src/drivers/net/mlx_ipoib/arbel.h +++ b/src/drivers/net/mlx_ipoib/arbel.h @@ -40,7 +40,12 @@ #define ARBEL_HCR_QUERY_FW 0x0004 #define ARBEL_HCR_INIT_HCA 0x0007 #define ARBEL_HCR_CLOSE_HCA 0x0008 +#define ARBEL_HCR_INIT_IB 0x0009 +#define ARBEL_HCR_CLOSE_IB 0x000a #define ARBEL_HCR_SW2HW_MPT 0x000d +#define ARBEL_HCR_MAP_EQ 0x0012 +#define ARBEL_HCR_SW2HW_EQ 0x0013 +#define ARBEL_HCR_HW2SW_EQ 0x0014 #define ARBEL_HCR_SW2HW_CQ 0x0016 #define ARBEL_HCR_HW2SW_CQ 0x0017 #define ARBEL_HCR_RST2INIT_QPEE 0x0019 @@ -68,6 +73,8 @@ /* MTUs */ #define ARBEL_MTU_2048 0x04 +#define ARBEL_NO_EQ 64 + #define ARBEL_INVALID_LKEY 0x00000100UL /* @@ -98,8 +105,10 @@ struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_entry ); struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error ); struct MLX_DECLARE_STRUCT ( arbelprm_cq_arm_db_record ); struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record ); +struct MLX_DECLARE_STRUCT ( arbelprm_eqc ); struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register ); struct MLX_DECLARE_STRUCT ( arbelprm_init_hca ); +struct MLX_DECLARE_STRUCT ( arbelprm_init_ib ); struct MLX_DECLARE_STRUCT ( arbelprm_mad_ifc ); struct MLX_DECLARE_STRUCT ( arbelprm_mgm_entry ); struct MLX_DECLARE_STRUCT ( arbelprm_mgm_hash ); @@ -321,8 +330,6 @@ struct arbel { * Used to get unrestricted memory access. */ unsigned long reserved_lkey; - /** Event queue number */ - unsigned long eqn; /** Completion queue in-use bitmask */ arbel_bitmask_t cq_inuse[ ARBEL_BITMASK_SIZE ( ARBEL_MAX_CQS ) ]; diff --git a/src/drivers/net/mlx_ipoib/ib_mt25218.c b/src/drivers/net/mlx_ipoib/ib_mt25218.c index 9473af89..174a2309 100644 --- a/src/drivers/net/mlx_ipoib/ib_mt25218.c +++ b/src/drivers/net/mlx_ipoib/ib_mt25218.c @@ -811,7 +811,7 @@ static int setup_hca(__u8 port, void **eq_p) DBG ( "mpt_base_addr_l = %lx\n", icm_start ); init_hca.mpt_base_addr_l = icm_start; init_hca.log_mpt_sz = log2_entries; - DBG ( "log2_entries for mpt = %d\n", log2_entries ); + DBG ( "log2_entries for mpt = %ld\n", log2_entries ); icm_start += (tmp << 12); icm_size += (tmp << 12); diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index 0bd5385f..cf3c9f00 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -30,9 +30,8 @@ Skeleton NIC driver for Etherboot #include "arbel.h" - - - +/* Port to use */ +#define PXE_IB_PORT 1 /*************************************************************************** * @@ -242,6 +241,22 @@ arbel_cmd_close_hca ( struct arbel *arbel ) { 0, NULL, 0, NULL ); } +static inline int +arbel_cmd_init_ib ( struct arbel *arbel, unsigned int port, + const struct arbelprm_init_ib *init_ib ) { + return arbel_cmd ( arbel, + ARBEL_HCR_IN_CMD ( ARBEL_HCR_INIT_IB, + 1, sizeof ( *init_ib ) ), + 0, init_ib, port, NULL ); +} + +static inline int +arbel_cmd_close_ib ( struct arbel *arbel, unsigned int port ) { + return arbel_cmd ( arbel, + ARBEL_HCR_VOID_CMD ( ARBEL_HCR_CLOSE_IB ), + 0, NULL, port, NULL ); +} + static inline int arbel_cmd_sw2hw_mpt ( struct arbel *arbel, unsigned int index, const struct arbelprm_mpt *mpt ) { @@ -251,6 +266,22 @@ arbel_cmd_sw2hw_mpt ( struct arbel *arbel, unsigned int index, 0, mpt, index, NULL ); } +static inline int +arbel_cmd_sw2hw_eq ( struct arbel *arbel, unsigned int index, + const struct arbelprm_eqc *eqc ) { + return arbel_cmd ( arbel, + ARBEL_HCR_IN_CMD ( ARBEL_HCR_SW2HW_EQ, + 1, sizeof ( *eqc ) ), + 0, eqc, index, NULL ); +} + +static inline int +arbel_cmd_hw2sw_eq ( struct arbel *arbel, unsigned int index ) { + return arbel_cmd ( arbel, + ARBEL_HCR_VOID_CMD ( ARBEL_HCR_HW2SW_EQ ), + 1, NULL, index, NULL ); +} + static inline int arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn, const struct arbelprm_completion_queue_context *cqctx ) { @@ -421,16 +452,6 @@ arbel_cmd_map_fa ( struct arbel *arbel, 0, map, 1, NULL ); } -/*************************************************************************** - * - * Event queue operations - * - *************************************************************************** - */ - -static int arbel_create_eq ( struct arbel *arbel ) { -} - /*************************************************************************** * * Completion queue operations @@ -508,7 +529,7 @@ static int arbel_create_cq ( struct ib_device *ibdev, MLX_FILL_2 ( &cqctx, 3, usr_page, arbel->limits.reserved_uars, log_cq_size, fls ( cq->num_cqes - 1 ) ); - MLX_FILL_1 ( &cqctx, 5, c_eqn, arbel->eqn ); + MLX_FILL_1 ( &cqctx, 5, c_eqn, ARBEL_NO_EQ ); MLX_FILL_1 ( &cqctx, 6, pd, ARBEL_GLOBAL_PD ); MLX_FILL_1 ( &cqctx, 7, l_key, arbel->reserved_lkey ); MLX_FILL_1 ( &cqctx, 12, cqn, cq->cqn ); @@ -1423,6 +1444,38 @@ static int arbel_get_pkey ( struct arbel *arbel, unsigned int *pkey ) { return 0; } +/** + * Wait for link up + * + * @v arbel Arbel device + * @ret rc Return status code + * + * This function shouldn't really exist. Unfortunately, IB links take + * a long time to come up, and we can't get various key parameters + * e.g. our own IPoIB MAC address without information from the subnet + * manager). We should eventually make link-up an asynchronous event. + */ +static int arbel_wait_for_link ( struct arbel *arbel ) { + struct ib_mad_port_info port_info; + unsigned int retries; + int rc; + + printf ( "Waiting for Infiniband link-up..." ); + for ( retries = 20 ; retries ; retries-- ) { + if ( ( rc = arbel_get_port_info ( arbel, &port_info ) ) != 0 ) + continue; + if ( ( ( port_info.port_state__link_speed_supported ) & 0xf ) + == 4 ) { + printf ( "ok\n" ); + return 0; + } + printf ( "." ); + sleep ( 1 ); + } + printf ( "failed\n" ); + return -ENODEV; +}; + /** * Get MAD parameters * @@ -1819,6 +1872,54 @@ static void arbel_free_icm ( struct arbel *arbel ) { arbel->icm = UNULL; } +/*************************************************************************** + * + * Infiniband link-layer operations + * + *************************************************************************** + */ + +/** + * Initialise Infiniband link + * + * @v arbel Arbel device + * @ret rc Return status code + */ +static int arbel_init_ib ( struct arbel *arbel ) { + struct arbelprm_init_ib init_ib; + int rc; + + memset ( &init_ib, 0, sizeof ( init_ib ) ); + MLX_FILL_3 ( &init_ib, 0, + mtu_cap, ARBEL_MTU_2048, + port_width_cap, 3, + vl_cap, 1 ); + MLX_FILL_1 ( &init_ib, 2, max_pkey, 64 ); + if ( ( rc = arbel_cmd_init_ib ( arbel, PXE_IB_PORT, + &init_ib ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not intialise IB: %s\n", + arbel, strerror ( rc ) ); + return rc; + } + + return 0; +} + +/** + * Close Infiniband link + * + * @v arbel Arbel device + */ +static void arbel_close_ib ( struct arbel *arbel ) { + int rc; + + if ( ( rc = arbel_cmd_close_ib ( arbel, PXE_IB_PORT ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not close IB: %s\n", + arbel, strerror ( rc ) ); + /* Nothing we can do about this */ + } +} + /*************************************************************************** * * PCI interface @@ -1933,6 +2034,16 @@ static int arbel_probe ( struct pci_device *pci, if ( ( rc = arbel_alloc_icm ( arbel, &init_hca ) ) != 0 ) goto err_alloc_icm; + + unsigned long uar_offset = ( arbel->limits.reserved_uars * 4096 ); + arbel->db_rec = phys_to_virt ( user_to_phys ( arbel->icm, + uar_offset ) ); + memset ( arbel->db_rec, 0, 4096 ); + union arbelprm_doorbell_record *db_rec; + db_rec = &arbel->db_rec[ARBEL_GROUP_SEPARATOR_DOORBELL]; + MLX_FILL_1 ( &db_rec->qp, 1, res, ARBEL_UAR_RES_GROUP_SEP ); + + /* Initialise HCA */ MLX_FILL_1 ( &init_hca, 74, uar_parameters.log_max_uars, 1 ); if ( ( rc = arbel_cmd_init_hca ( arbel, &init_hca ) ) != 0 ) { @@ -1945,7 +2056,14 @@ static int arbel_probe ( struct pci_device *pci, if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 ) goto err_setup_mpt; - + /* Bring up IB layer */ + if ( ( rc = arbel_init_ib ( arbel ) ) != 0 ) + goto err_init_ib; + + /* Wait for link */ + if ( ( rc = arbel_wait_for_link ( arbel ) ) != 0 ) + goto err_wait_for_link; + #endif @@ -1957,17 +2075,24 @@ static int arbel_probe ( struct pci_device *pci, arbel->mailbox_in = dev_buffers_p->inprm_buf; arbel->mailbox_out = dev_buffers_p->outprm_buf; #endif -#if ! SELF_INIT +#if SELF_INIT +#else arbel->reserved_lkey = dev_ib_data.mkey; -#endif arbel->db_rec = dev_ib_data.uar_context_base; - arbel->eqn = dev_ib_data.eq.eqn; +#endif + // arbel->eqn = dev_ib_data.eq.eqn; /* Get MAD parameters */ if ( ( rc = arbel_get_mad_params ( ibdev ) ) != 0 ) goto err_get_mad_params; + DBGC ( arbel, "Arbel %p port GID is %08lx:%08lx:%08lx:%08lx\n", arbel, + htonl ( ibdev->port_gid.u.dwords[0] ), + htonl ( ibdev->port_gid.u.dwords[1] ), + htonl ( ibdev->port_gid.u.dwords[2] ), + htonl ( ibdev->port_gid.u.dwords[3] ) ); + /* Add IPoIB device */ if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) { DBGC ( arbel, "Arbel %p could not add IPoIB device: %s\n", @@ -1982,6 +2107,9 @@ static int arbel_probe ( struct pci_device *pci, ib_driver_close ( 0 ); err_ib_driver_init: + err_wait_for_link: + arbel_close_ib ( arbel ); + err_init_ib: err_setup_mpt: arbel_cmd_close_hca ( arbel ); err_init_hca: