From 57bf8d9905baead5485055b0d3717b9a252c8feb Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Tue, 18 Sep 2007 03:58:59 +0100 Subject: [PATCH] Seems to get through the ICM mapping process --- src/drivers/net/mlx_ipoib/arbel.h | 3 ++ src/drivers/net/mlx_ipoib/cmdif_comm.c | 27 +++++++--------- src/drivers/net/mlx_ipoib/ib_mt25218.c | 13 ++++++++ src/drivers/net/mlx_ipoib/mt25218.c | 45 ++++++++++++++++++++++---- 4 files changed, 67 insertions(+), 21 deletions(-) diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h index 79e19a91..243aaa47 100644 --- a/src/drivers/net/mlx_ipoib/arbel.h +++ b/src/drivers/net/mlx_ipoib/arbel.h @@ -50,6 +50,8 @@ #define ARBEL_HCR_WRITE_MGM 0x0026 #define ARBEL_HCR_MGID_HASH 0x0027 #define ARBEL_HCR_RUN_FW 0x0ff6 +#define ARBEL_HCR_DISABLE_LAM 0x0ff7 +#define ARBEL_HCR_ENABLE_LAM 0x0ff8 #define ARBEL_HCR_UNMAP_ICM 0x0ff9 #define ARBEL_HCR_MAP_ICM 0x0ffa #define ARBEL_HCR_UNMAP_ICM_AUX 0x0ffb @@ -88,6 +90,7 @@ struct arbelprm_scalar_parameter_st { * */ +struct MLX_DECLARE_STRUCT ( arbelprm_access_lam ); struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_context ); struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_entry ); struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error ); diff --git a/src/drivers/net/mlx_ipoib/cmdif_comm.c b/src/drivers/net/mlx_ipoib/cmdif_comm.c index 97f288bc..363e4209 100644 --- a/src/drivers/net/mlx_ipoib/cmdif_comm.c +++ b/src/drivers/net/mlx_ipoib/cmdif_comm.c @@ -134,12 +134,10 @@ static XHH_cmd_status_t cmd_invoke(command_fields_t * cmd_prms) DBG_HD ( &hcr[0], sizeof ( hcr ) ); if ( cmd_prms->in_trans == TRANS_MAILBOX ) { size_t size = ( 4 * cmd_prms->in_param_size ); - if ( size > 256 ) - size = 256; -#if ! CREATE_OWN - DBG ( "Input mailbox:\n" ); - DBG_HD ( &cmd_prms->in_param[0], size ); -#endif + if ( size > 512 ) + size = 512; + DBG2 ( "Input mailbox:\n" ); + DBG2_HD ( &cmd_prms->in_param[0], size ); } for (i = 0; i < 7; ++i) { @@ -172,6 +170,14 @@ static XHH_cmd_status_t cmd_invoke(command_fields_t * cmd_prms) return status; } + if ( cmd_prms->out_trans == TRANS_MAILBOX ) { + size_t size = ( 4 * cmd_prms->out_param_size ); + if ( size > 512 ) + size = 512; + DBG2 ( "Output mailbox:\n" ); + DBG2_HD ( &cmd_prms->out_param[0], size ); + } + if (cmd_prms->out_trans == TRANS_MAILBOX) be_to_cpu_buf(cmd_prms->out_param, cmd_prms->out_param_size); else if (cmd_prms->out_trans == TRANS_IMMEDIATE) { @@ -181,15 +187,6 @@ static XHH_cmd_status_t cmd_invoke(command_fields_t * cmd_prms) return -1; } - if ( cmd_prms->out_trans == TRANS_MAILBOX ) { - size_t size = ( 4 * cmd_prms->out_param_size ); - if ( size > 256 ) - size = 256; -#if ! CREATE_OWN - DBG ( "Output mailbox:\n" ); - DBG_HD ( &cmd_prms->out_param[0], size ); -#endif - } DBG ( "Command executed successfully\n" ); return 0; diff --git a/src/drivers/net/mlx_ipoib/ib_mt25218.c b/src/drivers/net/mlx_ipoib/ib_mt25218.c index a5d251d4..caf924a3 100644 --- a/src/drivers/net/mlx_ipoib/ib_mt25218.c +++ b/src/drivers/net/mlx_ipoib/ib_mt25218.c @@ -758,17 +758,20 @@ static int setup_hca(__u8 port, void **eq_p) tmp = get_req_icm_pages(dev_lim.log2_rsvd_qps, MAX_APP_QPS, dev_lim.qpc_entry_sz, &log2_entries); + DBG ( "qpc_base_addr_l = %lx\n", icm_start ); init_hca.qpc_base_addr_l = icm_start; init_hca.log_num_of_qp = log2_entries; icm_start += (tmp << 12); icm_size += (tmp << 12); + DBG ( "eqpc_base_addr_l = %lx\n", icm_start ); init_hca.eqpc_base_addr_l = icm_start; icm_start += (tmp << 12); icm_size += (tmp << 12); tmp = get_req_icm_pages(dev_lim.log2_rsvd_srqs, 0, dev_lim.srq_entry_sz, &log2_entries); + DBG ( "srqc_base_addr_l = %lx\n", icm_start ); init_hca.srqc_base_addr_l = icm_start; init_hca.log_num_of_srq = log2_entries; icm_start += (tmp << 12); @@ -776,15 +779,18 @@ static int setup_hca(__u8 port, void **eq_p) tmp = get_req_icm_pages(dev_lim.log2_rsvd_ees, 0, dev_lim.eec_entry_sz, &log2_entries); + DBG ( "eec_base_addr_l = %lx\n", icm_start ); init_hca.eec_base_addr_l = icm_start; init_hca.log_num_of_ee = log2_entries; icm_start += (tmp << 12); icm_size += (tmp << 12); + DBG ( "eeec_base_addr_l = %lx\n", icm_start ); init_hca.eeec_base_addr_l = icm_start; icm_start += (tmp << 12); icm_size += (tmp << 12); + DBG ( "cqc_base_addr_l = %lx\n", icm_start ); tmp = get_req_icm_pages(dev_lim.log2_rsvd_cqs, MAX_APP_CQS, dev_lim.cqc_entry_sz, &log2_entries); @@ -795,12 +801,14 @@ static int setup_hca(__u8 port, void **eq_p) tmp = get_req_icm_pages(dev_lim.log2_rsvd_mtts, 0, dev_lim.mtt_entry_sz, &log2_entries); + DBG ( "mtt_base_addr_l = %lx\n", icm_start ); init_hca.mtt_base_addr_l = icm_start; icm_start += (tmp << 12); icm_size += (tmp << 12); tmp = get_req_icm_pages(dev_lim.log2_rsvd_mrws, 1, dev_lim.mpt_entry_sz, &log2_entries); + DBG ( "mpt_base_addr_l = %lx\n", icm_start ); init_hca.mpt_base_addr_l = icm_start; init_hca.log_mpt_sz = log2_entries; icm_start += (tmp << 12); @@ -808,16 +816,19 @@ static int setup_hca(__u8 port, void **eq_p) tmp = get_req_icm_pages(dev_lim.log2_rsvd_rdbs, 1, 32, /* size of rdb entry */ &log2_entries); + DBG ( "rdb_base_addr_l = %lx\n", icm_start ); init_hca.rdb_base_addr_l = icm_start; icm_start += (tmp << 12); icm_size += (tmp << 12); + DBG ( "eqc_base_addr_l = %lx\n", icm_start ); init_hca.eqc_base_addr_l = icm_start; init_hca.log_num_of_eq = LOG2_EQS; tmp = dev_lim.eqc_entry_sz * (1 << LOG2_EQS); icm_start += tmp; icm_size += tmp; + DBG ( "mc_base_addr_l = %lx\n", icm_start ); init_hca.mc_base_addr_l = icm_start; init_hca.log_mc_table_entry_sz = my_log2(MT_STRUCT_SIZE(arbelprm_mgm_entry_st)); @@ -828,6 +839,8 @@ static int setup_hca(__u8 port, void **eq_p) icm_start += (MT_STRUCT_SIZE(arbelprm_mgm_entry_st) * init_hca.mc_table_hash_sz); + DBG ( "icm_size = %lx\n", icm_size ); + rc = cmd_set_icm_size(icm_size, &aux_pages); if (rc) { ret = -1; diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index 69f24054..1f343d62 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -331,6 +331,21 @@ arbel_cmd_run_fw ( struct arbel *arbel ) { 0, NULL, 0, NULL ); } +static inline int +arbel_cmd_disable_lam ( struct arbel *arbel ) { + return arbel_cmd ( arbel, + ARBEL_HCR_VOID_CMD ( ARBEL_HCR_DISABLE_LAM ), + 0, NULL, 0, NULL ); +} + +static inline int +arbel_cmd_enable_lam ( struct arbel *arbel, struct arbelprm_access_lam *lam ) { + return arbel_cmd ( arbel, + ARBEL_HCR_OUT_CMD ( ARBEL_HCR_ENABLE_LAM, + 1, sizeof ( *lam ) ), + 1, NULL, 0, lam ); +} + static inline int arbel_cmd_unmap_icm ( struct arbel *arbel, unsigned int page_count ) { return arbel_cmd ( arbel, @@ -1431,6 +1446,7 @@ static int arbel_get_mad_params ( struct ib_device *ibdev ) { */ static int arbel_start_firmware ( struct arbel *arbel ) { struct arbelprm_query_fw fw; + struct arbelprm_access_lam lam; struct arbelprm_virtual_physical_mapping map_fa; unsigned int fw_pages; unsigned int log2_fw_pages; @@ -1453,6 +1469,11 @@ static int arbel_start_firmware ( struct arbel *arbel ) { DBGC ( arbel, "Arbel %p requires %d kB for firmware\n", arbel, ( fw_pages * 4 ) ); + /* Enable locally-attached memory. Ignore failure; there may + * be no attached memory. + */ + arbel_cmd_enable_lam ( arbel, &lam ); + /* Allocate firmware pages and map firmware area */ fw_size = ( fw_pages * 4096 ); arbel->firmware_area = umalloc ( fw_size ); @@ -1605,13 +1626,16 @@ static int arbel_alloc_icm ( struct arbel *arbel, ( icm_offset >> 7 ), qpc_eec_cqc_eqc_rdb_parameters.log_num_of_qp, log_num_qps ); + DBGC ( arbel, "Arbel %p ICM QPC base = %zx\n", arbel, icm_offset ); icm_offset += icm_usage ( log_num_qps, arbel->limits.qpc_entry_size ); /* Extended queue pair contexts */ MLX_FILL_1 ( init_hca, 25, qpc_eec_cqc_eqc_rdb_parameters.eqpc_base_addr_l, icm_offset ); - icm_offset += icm_usage ( log_num_qps, arbel->limits.eqpc_entry_size ); + DBGC ( arbel, "Arbel %p ICM EQPC base = %zx\n", arbel, icm_offset ); + // icm_offset += icm_usage ( log_num_qps, arbel->limits.eqpc_entry_size ); + icm_offset += icm_usage ( log_num_qps, arbel->limits.qpc_entry_size ); /* Shared receive queue contexts */ log_num_srqs = fls ( arbel->limits.reserved_srqs - 1 ); @@ -1620,6 +1644,7 @@ static int arbel_alloc_icm ( struct arbel *arbel, ( icm_offset >> 5 ), qpc_eec_cqc_eqc_rdb_parameters.log_num_of_srq, log_num_srqs ); + DBGC ( arbel, "Arbel %p ICM SRQC base = %zx\n", arbel, icm_offset ); icm_offset += icm_usage ( log_num_srqs, arbel->limits.srqc_entry_size ); /* End-to-end contexts */ @@ -1629,12 +1654,14 @@ static int arbel_alloc_icm ( struct arbel *arbel, ( icm_offset >> 7 ), qpc_eec_cqc_eqc_rdb_parameters.log_num_of_ee, log_num_ees ); + DBGC ( arbel, "Arbel %p ICM EEC base = %zx\n", arbel, icm_offset ); icm_offset += icm_usage ( log_num_ees, arbel->limits.eec_entry_size ); /* Extended end-to-end contexts */ MLX_FILL_1 ( init_hca, 29, qpc_eec_cqc_eqc_rdb_parameters.eeec_base_addr_l, icm_offset ); + DBGC ( arbel, "Arbel %p ICM EEEC base = %zx\n", arbel, icm_offset ); icm_offset += icm_usage ( log_num_ees, arbel->limits.eeec_entry_size ); /* Completion queue contexts */ @@ -1644,12 +1671,14 @@ static int arbel_alloc_icm ( struct arbel *arbel, ( icm_offset >> 6 ), qpc_eec_cqc_eqc_rdb_parameters.log_num_of_cq, log_num_cqs ); + DBGC ( arbel, "Arbel %p ICM CQC base = %zx\n", arbel, icm_offset ); icm_offset += icm_usage ( log_num_cqs, arbel->limits.cqc_entry_size ); /* Memory translation table */ log_num_mtts = fls ( arbel->limits.reserved_mtts - 1 ); MLX_FILL_1 ( init_hca, 65, tpt_parameters.mtt_base_addr_l, icm_offset ); + DBGC ( arbel, "Arbel %p ICM MTT base = %zx\n", arbel, icm_offset ); icm_offset += icm_usage ( log_num_mtts, arbel->limits.mtt_entry_size ); /* Memory protection table */ @@ -1658,6 +1687,7 @@ static int arbel_alloc_icm ( struct arbel *arbel, tpt_parameters.mpt_base_adr_l, icm_offset ); MLX_FILL_1 ( init_hca, 62, tpt_parameters.log_mpt_sz, log_num_mpts ); + DBGC ( arbel, "Arbel %p ICM MTT base = %zx\n", arbel, icm_offset ); icm_offset += icm_usage ( log_num_mpts, arbel->limits.mpt_entry_size ); /* RDMA something or other */ @@ -1665,6 +1695,7 @@ static int arbel_alloc_icm ( struct arbel *arbel, MLX_FILL_1 ( init_hca, 37, qpc_eec_cqc_eqc_rdb_parameters.rdb_base_addr_l, icm_offset ); + DBGC ( arbel, "Arbel %p ICM RDB base = %zx\n", arbel, icm_offset ); icm_offset += icm_usage ( log_num_rdbs, 32 ); /* Event queue contexts */ @@ -1674,6 +1705,7 @@ static int arbel_alloc_icm ( struct arbel *arbel, ( icm_offset >> 6 ), qpc_eec_cqc_eqc_rdb_parameters.log_num_eq, log_num_eqs ); + DBGC ( arbel, "Arbel %p ICM EQ base = %zx\n", arbel, icm_offset ); icm_offset += ( ( 1 << log_num_eqs ) * arbel->limits.eqc_entry_size ); /* Multicast table */ @@ -1686,6 +1718,7 @@ static int arbel_alloc_icm ( struct arbel *arbel, multicast_parameters.mc_table_hash_sz, 8 ); MLX_FILL_1 ( init_hca, 54, multicast_parameters.log_mc_table_sz, 3 ); + DBGC ( arbel, "Arbel %p ICM MC base = %zx\n", arbel, icm_offset ); icm_offset += ( 8 * sizeof ( struct arbelprm_mgm_entry ) ); arbel->icm_len = icm_offset; @@ -1700,10 +1733,9 @@ static int arbel_alloc_icm ( struct arbel *arbel, arbel, strerror ( rc ) ); goto err_set_icm_size; } - arbel->icm_aux_len = MLX_GET ( &icm_aux_size, value ); + arbel->icm_aux_len = ( MLX_GET ( &icm_aux_size, value ) * 4096 ); /* Allocate ICM data and auxiliary area */ - arbel->icm_aux_len = ( ( arbel->icm_aux_len + 4095 ) & ~4095 ); DBGC ( arbel, "Arbel %p requires %zd kB ICM and %zd kB AUX ICM\n", arbel, ( arbel->icm_len / 1024 ), ( arbel->icm_aux_len / 1024 ) ); @@ -1717,7 +1749,8 @@ static int arbel_alloc_icm ( struct arbel *arbel, memset ( &map_icm_aux, 0, sizeof ( map_icm_aux ) ); MLX_FILL_2 ( &map_icm_aux, 3, log2size, fls ( ( arbel->icm_aux_len / 4096 ) - 1 ), - pa_l, user_to_phys ( arbel->icm, arbel->icm_len ) ); + pa_l, + ( user_to_phys ( arbel->icm, arbel->icm_len ) >> 12 ) ); if ( ( rc = arbel_cmd_map_icm_aux ( arbel, &map_icm_aux ) ) != 0 ) { DBGC ( arbel, "Arbel %p could not map AUX ICM: %s\n", arbel, strerror ( rc ) ); @@ -1728,7 +1761,7 @@ static int arbel_alloc_icm ( struct arbel *arbel, memset ( &map_icm, 0, sizeof ( map_icm ) ); MLX_FILL_2 ( &map_icm, 3, log2size, fls ( ( arbel->icm_len / 4096 ) - 1 ), - pa_l, user_to_phys ( arbel->icm, 0 ) ); + pa_l, ( user_to_phys ( arbel->icm, 0 ) >> 12 ) ); if ( ( rc = arbel_cmd_map_icm ( arbel, &map_icm ) ) != 0 ) { DBGC ( arbel, "Arbel %p could not map ICM: %s\n", arbel, strerror ( rc ) ); @@ -1815,7 +1848,7 @@ static int arbel_probe ( struct pci_device *pci, goto err_mailbox_out; } -#define SELF_INIT 0 +#define SELF_INIT 1 #if SELF_INIT /* Start firmware */