diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c index 41494a5a..7d1b30ac 100644 --- a/src/drivers/infiniband/hermon.c +++ b/src/drivers/infiniband/hermon.c @@ -344,6 +344,15 @@ hermon_cmd_hw2sw_eq ( struct hermon *hermon, unsigned int index, 1, NULL, index, eqctx ); } +static inline int +hermon_cmd_query_eq ( struct hermon *hermon, unsigned int index, + struct hermonprm_eqc *eqctx ) { + return hermon_cmd ( hermon, + HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_EQ, + 1, sizeof ( *eqctx ) ), + 0, NULL, index, eqctx ); +} + static inline int hermon_cmd_sw2hw_cq ( struct hermon *hermon, unsigned long cqn, const struct hermonprm_completion_queue_context *cqctx ){ @@ -667,7 +676,7 @@ static int hermon_create_cq ( struct ib_device *ibdev, MLX_FILL_1 ( &cqctx, 2, page_offset, ( hermon_cq->mtt.page_offset >> 5 ) ); MLX_FILL_2 ( &cqctx, 3, - usr_page, HERMON_UAR_PAGE, + usr_page, HERMON_UAR_NON_EQ_PAGE, log_cq_size, fls ( cq->num_cqes - 1 ) ); MLX_FILL_1 ( &cqctx, 7, mtt_base_addr_l, ( hermon_cq->mtt.mtt_base_addr >> 3 ) ); @@ -773,6 +782,11 @@ static int hermon_create_qp ( struct ib_device *ibdev, goto err_hermon_qp; } + /* Calculate doorbell address */ + hermon_qp->send.doorbell = + ( hermon->uar + HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE + + HERMON_DB_POST_SND_OFFSET ); + /* Allocate work queue buffer */ hermon_qp->send.num_wqes = ( qp->send.num_wqes /* headroom */ + 1 + ( 2048 / sizeof ( hermon_qp->send.wqe[0] ) ) ); @@ -817,7 +831,7 @@ static int hermon_create_qp ( struct ib_device *ibdev, qpc_eec_data.log_sq_stride, ( fls ( sizeof ( hermon_qp->send.wqe[0] ) - 1 ) - 4 ) ); MLX_FILL_1 ( &qpctx, 5, - qpc_eec_data.usr_page, HERMON_UAR_PAGE ); + qpc_eec_data.usr_page, HERMON_UAR_NON_EQ_PAGE ); MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn ); MLX_FILL_1 ( &qpctx, 38, qpc_eec_data.page_offset, ( hermon_qp->mtt.page_offset >> 6 ) ); @@ -1029,9 +1043,8 @@ static int hermon_post_send ( struct ib_device *ibdev, /* Ring doorbell register */ MLX_FILL_1 ( &db_reg.send, 0, qn, qp->qpn ); DBGCP ( hermon, "Ringing doorbell %08lx with %08lx\n", - virt_to_phys ( hermon->uar + HERMON_DB_POST_SND_OFFSET ), - db_reg.dword[0] ); - writel ( db_reg.dword[0], ( hermon->uar + HERMON_DB_POST_SND_OFFSET )); + virt_to_phys ( hermon_send_wq->doorbell ), db_reg.dword[0] ); + writel ( db_reg.dword[0], ( hermon_send_wq->doorbell ) ); /* Update work queue's index */ wq->next_idx++; @@ -1209,7 +1222,7 @@ static void hermon_poll_cq ( struct ib_device *ibdev, /* Update doorbell record */ MLX_FILL_1 ( &hermon_cq->doorbell, 0, update_ci, - ( cq->next_idx & 0xffffffUL ) ); + ( cq->next_idx & 0x00ffffffUL ) ); } } @@ -1442,6 +1455,15 @@ static int hermon_create_eq ( struct hermon *hermon ) { unsigned int i; int rc; + /* Select event queue number */ + hermon_eq->eqn = ( 4 * hermon->cap.reserved_uars ); + if ( hermon_eq->eqn < hermon->cap.reserved_eqs ) + hermon_eq->eqn = hermon->cap.reserved_eqs; + + /* Calculate doorbell address */ + hermon_eq->doorbell = + ( hermon->uar + HERMON_DB_EQ_OFFSET ( hermon_eq->eqn ) ); + /* Allocate event queue itself */ hermon_eq->eqe_size = ( HERMON_NUM_EQES * sizeof ( hermon_eq->eqe[0] ) ); @@ -1471,7 +1493,8 @@ static int hermon_create_eq ( struct hermon *hermon ) { MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( HERMON_NUM_EQES - 1 ) ); MLX_FILL_1 ( &eqctx, 7, mtt_base_addr_l, ( hermon_eq->mtt.mtt_base_addr >> 3 ) ); - if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, 0, &eqctx ) ) != 0 ) { + if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, hermon_eq->eqn, + &eqctx ) ) != 0 ) { DBGC ( hermon, "Hermon %p SW2HW_EQ failed: %s\n", hermon, strerror ( rc ) ); goto err_sw2hw_eq; @@ -1480,17 +1503,21 @@ static int hermon_create_eq ( struct hermon *hermon ) { /* Map events to this event queue */ memset ( &mask, 0, sizeof ( mask ) ); MLX_FILL_1 ( &mask, 1, port_state_change, 1 ); - if ( ( rc = hermon_cmd_map_eq ( hermon, ( HERMON_MAP_EQ_MAP | 0 ), + if ( ( rc = hermon_cmd_map_eq ( hermon, + ( HERMON_MAP_EQ | hermon_eq->eqn ), &mask ) ) != 0 ) { DBGC ( hermon, "Hermon %p MAP_EQ failed: %s\n", hermon, strerror ( rc ) ); goto err_map_eq; } + DBGC ( hermon, "Hermon %p EQN %#lx ring at [%p,%p])\n", + hermon, hermon_eq->eqn, hermon_eq->eqe, + ( ( ( void * ) hermon_eq->eqe ) + hermon_eq->eqe_size ) ); return 0; err_map_eq: - hermon_cmd_hw2sw_eq ( hermon, 0, &eqctx ); + hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, &eqctx ); err_sw2hw_eq: hermon_free_mtt ( hermon, &hermon_eq->mtt ); err_alloc_mtt: @@ -1514,7 +1541,8 @@ static void hermon_destroy_eq ( struct hermon *hermon ) { /* Unmap events from event queue */ memset ( &mask, 0, sizeof ( mask ) ); MLX_FILL_1 ( &mask, 1, port_state_change, 1 ); - if ( ( rc = hermon_cmd_map_eq ( hermon, ( HERMON_MAP_EQ_UNMAP | 0 ), + if ( ( rc = hermon_cmd_map_eq ( hermon, + ( HERMON_UNMAP_EQ | hermon_eq->eqn ), &mask ) ) != 0 ) { DBGC ( hermon, "Hermon %p FATAL MAP_EQ failed to unmap: %s\n", hermon, strerror ( rc ) ); @@ -1522,7 +1550,8 @@ static void hermon_destroy_eq ( struct hermon *hermon ) { } /* Take ownership back from hardware */ - if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, 0, &eqctx ) ) != 0 ) { + if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, + &eqctx ) ) != 0 ) { DBGC ( hermon, "Hermon %p FATAL HW2SW_EQ failed: %s\n", hermon, strerror ( rc ) ); /* Leak memory and return; at least we avoid corruption */ @@ -1578,6 +1607,7 @@ static void hermon_poll_eq ( struct hermon *hermon ) { unsigned int event_type; while ( 1 ) { + /* Look for event entry */ eqe_idx_mask = ( HERMON_NUM_EQES - 1 ); eqe = &hermon_eq->eqe[hermon_eq->next_idx & eqe_idx_mask]; if ( MLX_GET ( &eqe->generic, owner ) ^ @@ -1605,13 +1635,12 @@ static void hermon_poll_eq ( struct hermon *hermon ) { hermon_eq->next_idx++; /* Ring doorbell */ - memset ( &db_reg, 0, sizeof ( db_reg ) ); - MLX_FILL_1 ( &db_reg.event, 0, ci, hermon_eq->next_idx ); + MLX_FILL_1 ( &db_reg.event, 0, + ci, ( hermon_eq->next_idx & 0x00ffffffUL ) ); DBGCP ( hermon, "Ringing doorbell %08lx with %08lx\n", - virt_to_phys ( hermon->uar + HERMON_DB_EQ0_OFFSET ), + virt_to_phys ( hermon_eq->doorbell ), db_reg.dword[0] ); - writel ( db_reg.dword[0], - ( hermon->uar + HERMON_DB_EQ0_OFFSET ) ); + writel ( db_reg.dword[0], hermon_eq->doorbell ); } } @@ -2161,9 +2190,8 @@ static int hermon_probe ( struct pci_device *pci, /* Get PCI BARs */ hermon->config = ioremap ( pci_bar_start ( pci, HERMON_PCI_CONFIG_BAR), HERMON_PCI_CONFIG_BAR_SIZE ); - hermon->uar = ioremap ( ( pci_bar_start ( pci, HERMON_PCI_UAR_BAR ) + - HERMON_UAR_PAGE * HERMON_PAGE_SIZE ), - HERMON_PAGE_SIZE ); + hermon->uar = ioremap ( pci_bar_start ( pci, HERMON_PCI_UAR_BAR ), + HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE ); /* Allocate space for mailboxes */ hermon->mailbox_in = malloc_dma ( HERMON_MBOX_SIZE, diff --git a/src/drivers/infiniband/hermon.h b/src/drivers/infiniband/hermon.h index d9e3dd11..45d7a1ce 100644 --- a/src/drivers/infiniband/hermon.h +++ b/src/drivers/infiniband/hermon.h @@ -44,6 +44,7 @@ #define HERMON_HCR_MAP_EQ 0x0012 #define HERMON_HCR_SW2HW_EQ 0x0013 #define HERMON_HCR_HW2SW_EQ 0x0014 +#define HERMON_HCR_QUERY_EQ 0x0015 #define HERMON_HCR_SW2HW_CQ 0x0016 #define HERMON_HCR_HW2SW_CQ 0x0017 #define HERMON_HCR_RST2INIT_QP 0x0019 @@ -77,12 +78,13 @@ #define HERMON_PAGE_SIZE 4096 #define HERMON_DB_POST_SND_OFFSET 0x14 -#define HERMON_DB_EQ0_OFFSET 0x800 +#define HERMON_DB_EQ_OFFSET(_eqn) \ + ( 0x800 + HERMON_PAGE_SIZE * ( (_eqn) / 4 ) + 0x08 * ( (_eqn) % 4 ) ) #define HERMON_QP_OPT_PARAM_QKEY 0x00000020UL -#define HERMON_MAP_EQ_MAP ( 0UL << 31 ) -#define HERMON_MAP_EQ_UNMAP ( 1UL << 31 ) +#define HERMON_MAP_EQ ( 0UL << 31 ) +#define HERMON_UNMAP_EQ ( 1UL << 31 ) #define HERMON_EV_PORT_STATE_CHANGE 0x09 @@ -292,7 +294,7 @@ enum hermon_icm_map_regions { * Pages 0-127 are reserved for event queue doorbells only, so we use * page 128. */ -#define HERMON_UAR_PAGE 128 +#define HERMON_UAR_NON_EQ_PAGE 128 /** Maximum number of allocatable MTT entries * @@ -334,6 +336,8 @@ struct hermon_send_work_queue { union hermon_send_wqe *wqe; /** Size of work queue */ size_t wqe_size; + /** Doorbell register */ + void *doorbell; }; /** Alignment of Hermon receive work queue entries */ @@ -400,7 +404,7 @@ struct hermon_completion_queue { * * This is a policy decision, not a device limit. */ -#define HERMON_MAX_EQS 4 +#define HERMON_MAX_EQS 8 /** A Hermon event queue */ struct hermon_event_queue { @@ -410,8 +414,12 @@ struct hermon_event_queue { size_t eqe_size; /** MTT descriptor */ struct hermon_mtt mtt; + /** Event queue number */ + unsigned long eqn; /** Next event queue entry index */ unsigned long next_idx; + /** Doorbell register */ + void *doorbell; }; /** Number of event queue entries