[hermon] Tidy up ICM allocations
Signed-off-by: Michael Brown <mcb30@ipxe.org>
This commit is contained in:
Michael Brown
parent
92ced72080
commit
0d6b1d98fa
|
|
@ -2226,7 +2226,7 @@ static int hermon_start_firmware ( struct hermon *hermon ) {
|
|||
MLX_GET ( &fw, fw_rev_subminor ) );
|
||||
fw_pages = MLX_GET ( &fw, fw_pages );
|
||||
DBGC ( hermon, "Hermon %p requires %d pages (%d kB) for firmware\n",
|
||||
hermon, fw_pages, ( fw_pages * ( HERMON_PAGE_SIZE / 1024 ) ) );
|
||||
hermon, fw_pages, ( fw_pages * 4 ) );
|
||||
|
||||
/* Allocate firmware pages and map firmware area */
|
||||
fw_size = ( fw_pages * HERMON_PAGE_SIZE );
|
||||
|
|
@ -2341,43 +2341,19 @@ static int hermon_get_cap ( struct hermon *hermon ) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get ICM usage
|
||||
* Align ICM table
|
||||
*
|
||||
* @v log_num_entries Log2 of the number of entries
|
||||
* @v entry_size Entry size
|
||||
* @ret usage Usage size in ICM
|
||||
* @v icm_offset Current ICM offset
|
||||
* @v len ICM table length
|
||||
* @ret icm_offset ICM offset
|
||||
*/
|
||||
static size_t icm_usage ( unsigned int log_num_entries, size_t entry_size ) {
|
||||
size_t usage;
|
||||
static uint64_t icm_align ( uint64_t icm_offset, size_t len ) {
|
||||
|
||||
usage = ( ( 1 << log_num_entries ) * entry_size );
|
||||
usage = ( ( usage + HERMON_PAGE_SIZE - 1 ) &
|
||||
~( HERMON_PAGE_SIZE - 1 ) );
|
||||
return usage;
|
||||
/* Round up to a multiple of the table size */
|
||||
assert ( len == ( 1UL << ( fls ( len ) - 1 ) ) );
|
||||
return ( ( icm_offset + len - 1 ) & ~( ( ( uint64_t ) len ) - 1 ) );
|
||||
}
|
||||
|
||||
/**
|
||||
* Align ICM
|
||||
*
|
||||
* @v member_size Member size
|
||||
* @v cur_icm_offset Current ICM offset
|
||||
* @ret align_offset Align to offset
|
||||
*/
|
||||
static size_t icm_align ( u32 member_size,
|
||||
u64 cur_icm_offset ) {
|
||||
size_t align_offset = 0;
|
||||
|
||||
member_size = member_size & 0xfffff000;
|
||||
if ( member_size ) {
|
||||
while ( ( cur_icm_offset + align_offset ) % member_size ) {
|
||||
align_offset += HERMON_PAGE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
return align_offset;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Allocate ICM
|
||||
*
|
||||
|
|
@ -2390,12 +2366,11 @@ static int hermon_alloc_icm ( struct hermon *hermon,
|
|||
struct hermonprm_scalar_parameter icm_size;
|
||||
struct hermonprm_scalar_parameter icm_aux_size;
|
||||
uint64_t icm_offset = 0;
|
||||
u32 icm_member_size = 0;
|
||||
unsigned int log_num_qps, log_num_srqs, log_num_cqs, log_num_eqs;
|
||||
unsigned int log_num_mtts, log_num_mpts;
|
||||
unsigned int log_num_mtts, log_num_mpts, log_num_mcs;
|
||||
size_t cmpt_max_len;
|
||||
size_t qp_cmpt_len, srq_cmpt_len, cq_cmpt_len, eq_cmpt_len;
|
||||
size_t icm_len, icm_aux_len;
|
||||
size_t len;
|
||||
physaddr_t icm_phys;
|
||||
int i;
|
||||
int rc;
|
||||
|
|
@ -2412,32 +2387,38 @@ static int hermon_alloc_icm ( struct hermon *hermon,
|
|||
log_num_cqs = fls ( hermon->cap.reserved_cqs + HERMON_MAX_CQS - 1 );
|
||||
log_num_eqs = fls ( hermon->cap.reserved_eqs + HERMON_MAX_EQS - 1 );
|
||||
log_num_mtts = fls ( hermon->cap.reserved_mtts + HERMON_MAX_MTTS - 1 );
|
||||
log_num_mpts = fls ( hermon->cap.reserved_mrws + 1 - 1 );
|
||||
log_num_mcs = HERMON_LOG_MULTICAST_HASH_SIZE;
|
||||
|
||||
/* ICM starts with the cMPT tables, which are sparse */
|
||||
cmpt_max_len = ( HERMON_CMPT_MAX_ENTRIES *
|
||||
( ( uint64_t ) hermon->cap.cmpt_entry_size ) );
|
||||
qp_cmpt_len = icm_usage ( log_num_qps, hermon->cap.cmpt_entry_size );
|
||||
len = ( ( ( ( 1 << log_num_qps ) * hermon->cap.cmpt_entry_size ) +
|
||||
HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
|
||||
hermon->icm_map[HERMON_ICM_QP_CMPT].offset = icm_offset;
|
||||
hermon->icm_map[HERMON_ICM_QP_CMPT].len = qp_cmpt_len;
|
||||
hermon->icm_map[HERMON_ICM_QP_CMPT].len = len;
|
||||
icm_offset += cmpt_max_len;
|
||||
srq_cmpt_len = icm_usage ( log_num_srqs, hermon->cap.cmpt_entry_size );
|
||||
len = ( ( ( ( 1 << log_num_srqs ) * hermon->cap.cmpt_entry_size ) +
|
||||
HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
|
||||
hermon->icm_map[HERMON_ICM_SRQ_CMPT].offset = icm_offset;
|
||||
hermon->icm_map[HERMON_ICM_SRQ_CMPT].len = srq_cmpt_len;
|
||||
hermon->icm_map[HERMON_ICM_SRQ_CMPT].len = len;
|
||||
icm_offset += cmpt_max_len;
|
||||
cq_cmpt_len = icm_usage ( log_num_cqs, hermon->cap.cmpt_entry_size );
|
||||
len = ( ( ( ( 1 << log_num_cqs ) * hermon->cap.cmpt_entry_size ) +
|
||||
HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
|
||||
hermon->icm_map[HERMON_ICM_CQ_CMPT].offset = icm_offset;
|
||||
hermon->icm_map[HERMON_ICM_CQ_CMPT].len = cq_cmpt_len;
|
||||
hermon->icm_map[HERMON_ICM_CQ_CMPT].len = len;
|
||||
icm_offset += cmpt_max_len;
|
||||
eq_cmpt_len = icm_usage ( log_num_eqs, hermon->cap.cmpt_entry_size );
|
||||
len = ( ( ( ( 1 << log_num_eqs ) * hermon->cap.cmpt_entry_size ) +
|
||||
HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
|
||||
hermon->icm_map[HERMON_ICM_EQ_CMPT].offset = icm_offset;
|
||||
hermon->icm_map[HERMON_ICM_EQ_CMPT].len = eq_cmpt_len;
|
||||
hermon->icm_map[HERMON_ICM_EQ_CMPT].len = len;
|
||||
icm_offset += cmpt_max_len;
|
||||
|
||||
hermon->icm_map[HERMON_ICM_OTHER].offset = icm_offset;
|
||||
|
||||
/* Queue pair contexts */
|
||||
icm_member_size = icm_usage ( log_num_qps, hermon->cap.qpc_entry_size );
|
||||
icm_offset += icm_align ( icm_member_size, icm_offset );
|
||||
len = ( ( 1 << log_num_qps ) * hermon->cap.qpc_entry_size );
|
||||
icm_offset = icm_align ( icm_offset, len );
|
||||
MLX_FILL_1 ( init_hca, 12,
|
||||
qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_h,
|
||||
( icm_offset >> 32 ) );
|
||||
|
|
@ -2446,38 +2427,42 @@ static int hermon_alloc_icm ( struct hermon *hermon,
|
|||
( icm_offset >> 5 ),
|
||||
qpc_eec_cqc_eqc_rdb_parameters.log_num_of_qp,
|
||||
log_num_qps );
|
||||
DBGC ( hermon, "Hermon %p ICM QPC base = %llx\n", hermon, icm_offset );
|
||||
icm_offset += icm_member_size;
|
||||
DBGC ( hermon, "Hermon %p ICM QPC is %d x %#zx at [%08llx,%08llx)\n",
|
||||
hermon, ( 1 << log_num_qps ), hermon->cap.qpc_entry_size,
|
||||
icm_offset, ( icm_offset + len ) );
|
||||
icm_offset += len;
|
||||
|
||||
/* Extended alternate path contexts */
|
||||
icm_member_size = icm_usage ( log_num_qps,
|
||||
hermon->cap.altc_entry_size );
|
||||
icm_offset += icm_align ( icm_member_size, icm_offset );
|
||||
len = ( ( 1 << log_num_qps ) * hermon->cap.altc_entry_size );
|
||||
icm_offset = icm_align ( icm_offset, len );
|
||||
MLX_FILL_1 ( init_hca, 24,
|
||||
qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_h,
|
||||
( icm_offset >> 32 ) );
|
||||
MLX_FILL_1 ( init_hca, 25,
|
||||
qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_l,
|
||||
icm_offset );
|
||||
DBGC ( hermon, "Hermon %p ICM ALTC base = %llx\n", hermon, icm_offset);
|
||||
icm_offset += icm_member_size;
|
||||
DBGC ( hermon, "Hermon %p ICM ALTC is %d x %#zx at [%08llx,%08llx)\n",
|
||||
hermon, ( 1 << log_num_qps ), hermon->cap.altc_entry_size,
|
||||
icm_offset, ( icm_offset + len ) );
|
||||
icm_offset += len;
|
||||
|
||||
/* Extended auxiliary contexts */
|
||||
icm_member_size = icm_usage ( log_num_qps,
|
||||
hermon->cap.auxc_entry_size );
|
||||
icm_offset += icm_align ( icm_member_size, icm_offset );
|
||||
len = ( ( 1 << log_num_qps ) * hermon->cap.auxc_entry_size );
|
||||
icm_offset = icm_align ( icm_offset, len );
|
||||
MLX_FILL_1 ( init_hca, 28,
|
||||
qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_h,
|
||||
( icm_offset >> 32 ) );
|
||||
MLX_FILL_1 ( init_hca, 29,
|
||||
qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_l,
|
||||
icm_offset );
|
||||
DBGC ( hermon, "Hermon %p ICM AUXC base = %llx\n", hermon, icm_offset);
|
||||
icm_offset += icm_member_size;
|
||||
DBGC ( hermon, "Hermon %p ICM AUXC is %d x %#zx at [%08llx,%08llx)\n",
|
||||
hermon, ( 1 << log_num_qps ), hermon->cap.auxc_entry_size,
|
||||
icm_offset, ( icm_offset + len ) );
|
||||
icm_offset += len;
|
||||
|
||||
/* Shared receive queue contexts */
|
||||
icm_member_size = icm_usage ( log_num_srqs,
|
||||
hermon->cap.srqc_entry_size );
|
||||
icm_offset += icm_align ( icm_member_size, icm_offset );
|
||||
len = ( ( 1 << log_num_srqs ) * hermon->cap.srqc_entry_size );
|
||||
icm_offset = icm_align ( icm_offset, len );
|
||||
MLX_FILL_1 ( init_hca, 18,
|
||||
qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_h,
|
||||
( icm_offset >> 32 ) );
|
||||
|
|
@ -2486,12 +2471,14 @@ static int hermon_alloc_icm ( struct hermon *hermon,
|
|||
( icm_offset >> 5 ),
|
||||
qpc_eec_cqc_eqc_rdb_parameters.log_num_of_srq,
|
||||
log_num_srqs );
|
||||
DBGC ( hermon, "Hermon %p ICM SRQC base = %llx\n", hermon, icm_offset);
|
||||
icm_offset += icm_member_size;
|
||||
DBGC ( hermon, "Hermon %p ICM SRQC is %d x %#zx at [%08llx,%08llx)\n",
|
||||
hermon, ( 1 << log_num_srqs ), hermon->cap.srqc_entry_size,
|
||||
icm_offset, ( icm_offset + len ) );
|
||||
icm_offset += len;
|
||||
|
||||
/* Completion queue contexts */
|
||||
icm_member_size = icm_usage ( log_num_cqs, hermon->cap.cqc_entry_size );
|
||||
icm_offset += icm_align ( icm_member_size, icm_offset );
|
||||
len = ( ( 1 << log_num_cqs ) * hermon->cap.cqc_entry_size );
|
||||
icm_offset = icm_align ( icm_offset, len );
|
||||
MLX_FILL_1 ( init_hca, 20,
|
||||
qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_h,
|
||||
( icm_offset >> 32 ) );
|
||||
|
|
@ -2500,12 +2487,14 @@ static int hermon_alloc_icm ( struct hermon *hermon,
|
|||
( icm_offset >> 5 ),
|
||||
qpc_eec_cqc_eqc_rdb_parameters.log_num_of_cq,
|
||||
log_num_cqs );
|
||||
DBGC ( hermon, "Hermon %p ICM CQC base = %llx\n", hermon, icm_offset );
|
||||
icm_offset += icm_member_size;
|
||||
DBGC ( hermon, "Hermon %p ICM CQC is %d x %#zx at [%08llx,%08llx)\n",
|
||||
hermon, ( 1 << log_num_cqs ), hermon->cap.cqc_entry_size,
|
||||
icm_offset, ( icm_offset + len ) );
|
||||
icm_offset += len;
|
||||
|
||||
/* Event queue contexts */
|
||||
icm_member_size = icm_usage ( log_num_eqs, hermon->cap.eqc_entry_size );
|
||||
icm_offset += icm_align ( icm_member_size, icm_offset );
|
||||
len = ( ( 1 << log_num_eqs ) * hermon->cap.eqc_entry_size );
|
||||
icm_offset = icm_align ( icm_offset, len );
|
||||
MLX_FILL_1 ( init_hca, 32,
|
||||
qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_h,
|
||||
( icm_offset >> 32 ) );
|
||||
|
|
@ -2514,39 +2503,40 @@ static int hermon_alloc_icm ( struct hermon *hermon,
|
|||
( icm_offset >> 5 ),
|
||||
qpc_eec_cqc_eqc_rdb_parameters.log_num_of_eq,
|
||||
log_num_eqs );
|
||||
DBGC ( hermon, "Hermon %p ICM EQC base = %llx\n", hermon, icm_offset );
|
||||
icm_offset += icm_member_size;
|
||||
DBGC ( hermon, "Hermon %p ICM EQC is %d x %#zx at [%08llx,%08llx)\n",
|
||||
hermon, ( 1 << log_num_eqs ), hermon->cap.eqc_entry_size,
|
||||
icm_offset, ( icm_offset + len ) );
|
||||
icm_offset += len;
|
||||
|
||||
/* Memory translation table */
|
||||
icm_member_size = icm_usage ( log_num_mtts,
|
||||
hermon->cap.mtt_entry_size );
|
||||
icm_offset += icm_align ( icm_member_size, icm_offset );
|
||||
len = ( ( 1 << log_num_mtts ) * hermon->cap.mtt_entry_size );
|
||||
icm_offset = icm_align ( icm_offset, len );
|
||||
MLX_FILL_1 ( init_hca, 64,
|
||||
tpt_parameters.mtt_base_addr_h, ( icm_offset >> 32 ) );
|
||||
MLX_FILL_1 ( init_hca, 65,
|
||||
tpt_parameters.mtt_base_addr_l, icm_offset );
|
||||
DBGC ( hermon, "Hermon %p ICM MTT base = %llx\n", hermon, icm_offset );
|
||||
icm_offset += icm_member_size;
|
||||
DBGC ( hermon, "Hermon %p ICM MTT is %d x %#zx at [%08llx,%08llx)\n",
|
||||
hermon, ( 1 << log_num_mtts ), hermon->cap.mtt_entry_size,
|
||||
icm_offset, ( icm_offset + len ) );
|
||||
icm_offset += len;
|
||||
|
||||
/* Memory protection table */
|
||||
log_num_mpts = fls ( hermon->cap.reserved_mrws + 1 - 1 );
|
||||
icm_member_size = icm_usage ( log_num_mpts,
|
||||
hermon->cap.dmpt_entry_size );
|
||||
icm_offset += icm_align ( icm_member_size, icm_offset );
|
||||
len = ( ( 1 << log_num_mpts ) * hermon->cap.dmpt_entry_size );
|
||||
icm_offset = icm_align ( icm_offset, len );
|
||||
MLX_FILL_1 ( init_hca, 60,
|
||||
tpt_parameters.dmpt_base_adr_h, ( icm_offset >> 32 ) );
|
||||
MLX_FILL_1 ( init_hca, 61,
|
||||
tpt_parameters.dmpt_base_adr_l, icm_offset );
|
||||
MLX_FILL_1 ( init_hca, 62,
|
||||
tpt_parameters.log_dmpt_sz, log_num_mpts );
|
||||
DBGC ( hermon, "Hermon %p ICM DMPT base = %llx\n", hermon, icm_offset);
|
||||
icm_offset += icm_usage ( log_num_mpts,
|
||||
hermon->cap.dmpt_entry_size );
|
||||
DBGC ( hermon, "Hermon %p ICM DMPT is %d x %#zx at [%08llx,%08llx)\n",
|
||||
hermon, ( 1 << log_num_mpts ), hermon->cap.dmpt_entry_size,
|
||||
icm_offset, ( icm_offset + len ) );
|
||||
icm_offset += len;
|
||||
|
||||
/* Multicast table */
|
||||
icm_member_size = ( ( 128 * sizeof ( struct hermonprm_mcg_entry ) +
|
||||
HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
|
||||
icm_offset += icm_align ( icm_member_size, icm_offset );
|
||||
len = ( ( 1 << log_num_mcs ) * sizeof ( struct hermonprm_mcg_entry ) );
|
||||
icm_offset = icm_align ( icm_offset, len );
|
||||
MLX_FILL_1 ( init_hca, 48,
|
||||
multicast_parameters.mc_base_addr_h,
|
||||
( icm_offset >> 32 ) );
|
||||
|
|
@ -2556,11 +2546,14 @@ static int hermon_alloc_icm ( struct hermon *hermon,
|
|||
multicast_parameters.log_mc_table_entry_sz,
|
||||
fls ( sizeof ( struct hermonprm_mcg_entry ) - 1 ) );
|
||||
MLX_FILL_1 ( init_hca, 53,
|
||||
multicast_parameters.log_mc_table_hash_sz, 7 );
|
||||
multicast_parameters.log_mc_table_hash_sz, log_num_mcs );
|
||||
MLX_FILL_1 ( init_hca, 54,
|
||||
multicast_parameters.log_mc_table_sz, 7 );
|
||||
DBGC ( hermon, "Hermon %p ICM MC base = %llx\n", hermon, icm_offset );
|
||||
icm_offset += icm_member_size;
|
||||
multicast_parameters.log_mc_table_sz, log_num_mcs );
|
||||
DBGC ( hermon, "Hermon %p ICM MC is %d x %#zx at [%08llx,%08llx)\n",
|
||||
hermon, ( 1 << log_num_mcs ),
|
||||
sizeof ( struct hermonprm_mcg_entry ),
|
||||
icm_offset, ( icm_offset + len ) );
|
||||
icm_offset += len;
|
||||
|
||||
|
||||
hermon->icm_map[HERMON_ICM_OTHER].len =
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
|||
|
||||
#define HERMON_INVALID_LKEY 0x00000100UL
|
||||
|
||||
#define HERMON_PAGE_SIZE 4096
|
||||
#define HERMON_PAGE_SIZE ( ( size_t ) 4096 )
|
||||
|
||||
#define HERMON_DB_POST_SND_OFFSET 0x14
|
||||
#define HERMON_DB_EQ_OFFSET(_eqn) \
|
||||
|
|
@ -107,6 +107,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
|||
#define HERMON_SCHED_QP0 0x3f
|
||||
#define HERMON_SCHED_DEFAULT 0x83
|
||||
|
||||
#define HERMON_LOG_MULTICAST_HASH_SIZE 7
|
||||
|
||||
#define HERMON_PM_STATE_ARMED 0x00
|
||||
#define HERMON_PM_STATE_REARM 0x01
|
||||
#define HERMON_PM_STATE_MIGRATED 0x03
|
||||
|
|
|
|||
Reference in New Issue