From e238bb1e439cb614f2986a70546559efef16239f Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Sun, 16 Sep 2007 14:39:51 +0100 Subject: [PATCH] destroy_cq() now implemented (not tested). --- src/drivers/net/mlx_ipoib/arbel.h | 4 ++ src/drivers/net/mlx_ipoib/mt25218.c | 65 ++++++++++++++++++++++++++++- src/include/gpxe/infiniband.h | 19 +++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h index 37fe3693..d3842467 100644 --- a/src/drivers/net/mlx_ipoib/arbel.h +++ b/src/drivers/net/mlx_ipoib/arbel.h @@ -28,6 +28,7 @@ /* HCA command register opcodes */ #define ARBEL_HCR_QUERY_DEV_LIM 0x0003 #define ARBEL_HCR_SW2HW_CQ 0x0016 +#define ARBEL_HCR_HW2SW_CQ 0x0017 /* * Wrapper structures for hardware datatypes @@ -247,6 +248,9 @@ struct arbel { #define ARBEL_HCR_OUT_CMD( _opcode, _out_mbox, _out_len ) \ ARBEL_HCR_CMD ( _opcode, 0, 0, _out_mbox, _out_len ) +#define ARBEL_HCR_VOID_CMD( _opcode ) \ + ARBEL_HCR_CMD ( _opcode, 0, 0, 0, 0 ) + /* * Doorbell record allocation * diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index dfdba4b5..64ae992b 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -270,6 +270,13 @@ static struct net_device_operations mlx_operations = { +/*************************************************************************** + * + * Queue number allocation + * + *************************************************************************** + */ + /** * Allocate queue number * @@ -444,6 +451,13 @@ arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn, 0, cqctx, cqn, NULL ); } +static inline int +arbel_cmd_hw2sw_cq ( struct arbel *arbel, unsigned long cqn ) { + return arbel_cmd ( arbel, + ARBEL_HCR_VOID_CMD ( ARBEL_HCR_HW2SW_CQ ), + 1, NULL, cqn, NULL ); +} + /*************************************************************************** * * Completion queue operations @@ -548,8 +562,9 @@ static int arbel_create_cq ( struct ib_device *ibdev, return 0; err_sw2hw: - memset ( ci_db_rec, 0, sizeof ( *ci_db_rec ) ); - memset ( arm_db_rec, 0, sizeof ( *arm_db_rec ) ); + MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE ); + MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE ); + free_dma ( arbel_cq->cqe, cqe_size ); err_cqe: free ( arbel_cq ); err_arbel_cq: @@ -558,6 +573,50 @@ static int arbel_create_cq ( struct ib_device *ibdev, return rc; } +/** + * Destroy completion queue + * + * @v ibdev Infiniband device + * @v cq Completion queue + */ +static void arbel_destroy_cq ( struct ib_device *ibdev, + struct ib_completion_queue *cq ) { + struct arbel *arbel = ibdev->priv; + struct arbel_completion_queue *arbel_cq = + container_of ( cq, struct arbel_completion_queue, cq ); + struct arbelprm_cq_ci_db_record *ci_db_rec; + struct arbelprm_cq_arm_db_record *arm_db_rec; + int cqn_offset; + size_t cqe_size; + unsigned int ci_doorbell_idx; + unsigned int arm_doorbell_idx; + int rc; + + assert ( list_empty ( &cq->work_queues ) ); + + /* Take ownership back from hardware */ + if ( ( rc = arbel_cmd_hw2sw_cq ( arbel, cq->cqn ) ) != 0 ) { + DBGC ( arbel, "Arbel %p FATAL HW2SW_CQ failed: %s\n", + arbel, strerror ( rc ) ); + /* Leak memory and return; at least we avoid corruption */ + return; + } + + /* Clear doorbell records */ + cqn_offset = ( cq->cqn - arbel->limits.reserved_cqs ); + ci_doorbell_idx = arbel_cq_ci_doorbell_idx ( cqn_offset ); + arm_doorbell_idx = arbel_cq_arm_doorbell_idx ( cqn_offset ); + ci_db_rec = &arbel->db_rec[ci_doorbell_idx].cq_ci; + arm_db_rec = &arbel->db_rec[arm_doorbell_idx].cq_arm; + MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE ); + MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE ); + + /* Free memory */ + cqe_size = ( cq->num_cqes * sizeof ( arbel_cq->cqe[0] ) ); + free_dma ( arbel_cq->cqe, cqe_size ); + free ( arbel_cq ); + arbel_free_qn_offset ( arbel->cq_inuse, cqn_offset ); +} /*************************************************************************** * @@ -863,6 +922,8 @@ static void arbel_poll_cq ( struct ib_device *ibdev, /** Arbel Infiniband operations */ static struct ib_device_operations arbel_ib_operations = { + .create_cq = arbel_create_cq, + .destroy_cq = arbel_destroy_cq, .post_send = arbel_post_send, .post_recv = arbel_post_recv, .poll_cq = arbel_poll_cq, diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index dd8022fb..973c5823 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -168,6 +168,25 @@ struct ib_address_vector { * These represent a subset of the Infiniband Verbs. */ struct ib_device_operations { + /** + * Create completion queue + * + * @v ibdev Infiniband device + * @v log2_num_cqes Log2 of the number of completion queue entries + * @ret new_cq New completion queue + * @ret rc Return status code + */ + int ( * create_cq ) ( struct ib_device *ibdev, + unsigned int log2_num_cqes, + struct ib_completion_queue **new_cq ); + /** + * Destroy completion queue + * + * @v ibdev Infiniband device + * @v cq Completion queue + */ + void ( * destroy_cq ) ( struct ib_device *ibdev, + struct ib_completion_queue *cq ); /** Post send work queue entry * * @v ibdev Infiniband device