From bf9bd938565f4fa99a53c4820f1aea4d9cdb6bbb Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Sun, 16 Sep 2007 01:44:57 +0100 Subject: [PATCH] First (working) draft of command interface. --- src/drivers/net/mlx_ipoib/arbel.h | 27 ++++++ src/drivers/net/mlx_ipoib/mt25218.c | 136 +++++++++++++++++++++++++++- 2 files changed, 159 insertions(+), 4 deletions(-) diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h index 9da6bef9..79d01b2b 100644 --- a/src/drivers/net/mlx_ipoib/arbel.h +++ b/src/drivers/net/mlx_ipoib/arbel.h @@ -16,14 +16,34 @@ #define ARBEL_OPCODE_RECV_ERROR 0xfe #define ARBEL_OPCODE_SEND_ERROR 0xff +/* + * HCA commands + * + */ + +#define ARBEL_HCR_BASE 0x80680 +#define ARBEL_HCR_REG(x) ( ARBEL_HCR_BASE + 4 * (x) ) +#define ARBEL_HCR_MAX_WAIT_MS 2000 + +#define ARBEL_HCR_OPCODE_MASK 0x0000ffffUL +#define ARBEL_HCR_IN_IMMEDIATE 0x00010000UL +#define ARBEL_HCR_IN_MAILBOX 0x00020000UL +#define ARBEL_HCR_OUT_IMMEDIATE 0x00040000UL +#define ARBEL_HCR_OUT_MAILBOX 0x00080000UL + +#define ARBEL_HCR_OP_SW2HW_CQ ( 0x0016 | ARBEL_HCR_IN_MAILBOX ) +#define ARBEL_HCR_OP_NOP ( 0x0031 ) + /* * Wrapper structures for hardware datatypes * */ +struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_context ); struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_entry ); struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error ); struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record ); +struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register ); struct MLX_DECLARE_STRUCT ( arbelprm_qp_db_record ); struct MLX_DECLARE_STRUCT ( arbelprm_recv_wqe_segment_next ); struct MLX_DECLARE_STRUCT ( arbelprm_send_doorbell ); @@ -126,6 +146,13 @@ struct arbel_completion_queue { /** An Arbel device */ struct arbel { + /** Configuration registers */ + void *config; + /** Command input mailbox */ + void *mailbox_in; + /** Command output mailbox */ + void *mailbox_out; + /** User Access Region */ void *uar; /** Doorbell records */ diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c index 2e147a0e..3625991c 100644 --- a/src/drivers/net/mlx_ipoib/mt25218.c +++ b/src/drivers/net/mlx_ipoib/mt25218.c @@ -26,6 +26,11 @@ Skeleton NIC driver for Etherboot #include "arbel.h" +static const struct ib_gid arbel_no_gid = { + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 } +}; + + #define MLX_RX_MAX_FILL NUM_IPOIB_RCV_WQES struct mlx_nic { @@ -263,11 +268,121 @@ static struct net_device_operations mlx_operations = { .irq = mlx_irq, }; +/** + * Wait for Arbel command completion + * + * @v arbel Arbel device + * @ret rc Return status code + */ +static int arbel_command_wait ( struct arbel *arbel, + struct arbelprm_hca_command_register *hcr ) { + unsigned int wait; + + for ( wait = ARBEL_HCR_MAX_WAIT_MS ; wait ; wait-- ) { + hcr->u.dwords[6] = + readl ( arbel->config + ARBEL_HCR_REG ( 6 ) ); + if ( MLX_GET ( hcr, go ) == 0 ) + return 0; + mdelay ( 1 ); + } + return -EBUSY; +} + +/** + * Issue HCA command + * + * @v arbel Arbel device + * @v op_fl Opcode (plus implied flags) + * @v op_mod Opcode modifier (0 if no modifier applicable) + * @v in_param Input parameter + * @v in_param_len Input parameter length + * @v in_mod Input modifier (0 if no modifier applicable) + * @v out_param Output parameter + * @ret rc Return status code + */ +static int arbel_command ( struct arbel *arbel, unsigned int op_fl, + unsigned int op_mod, const void *in_param, + size_t in_param_len, unsigned int in_mod, + void *out_param, size_t out_param_len ) { + struct arbelprm_hca_command_register hcr; + unsigned int status; + unsigned int i; + int rc; + + /* Check that HCR is free */ + if ( ( rc = arbel_command_wait ( arbel, &hcr ) ) != 0 ) { + DBGC ( arbel, "Arbel %p command interface locked\n", arbel ); + return rc; + } + + /* Prepare HCR */ + memset ( &hcr, 0, sizeof ( hcr ) ); + if ( op_fl & ARBEL_HCR_IN_IMMEDIATE ) { + memcpy ( &hcr.u.dwords[0], in_param, 8 ); + } else if ( op_fl & ARBEL_HCR_IN_MAILBOX ) { + memcpy ( arbel->mailbox_in, in_param, in_param_len ); + MLX_FILL_1 ( &hcr, 1, in_param_l, + virt_to_bus ( arbel->mailbox_in ) ); + } + MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod ); + if ( op_fl & ARBEL_HCR_OUT_MAILBOX ) { + MLX_FILL_1 ( &hcr, 4, out_param_l, + virt_to_bus ( arbel->mailbox_out ) ); + } + MLX_FILL_3 ( &hcr, 6, + opcode, ( op_fl & ARBEL_HCR_OPCODE_MASK ), + opcode_modifier, op_mod, + go, 1 ); + + /* Issue command */ + for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ; + i++ ) { + writel ( hcr.u.dwords[i], + arbel->config + ARBEL_HCR_REG ( i ) ); + barrier(); + } + + /* Wait for command completion */ + if ( ( rc = arbel_command_wait ( arbel, &hcr ) ) != 0 ) { + DBGC ( arbel, "Arbel %p timed out waiting for command:\n", + arbel ); + DBGC_HD ( arbel, &hcr, sizeof ( hcr ) ); + return rc; + } + + /* Check command status */ + status = MLX_GET ( &hcr, status ); + if ( status != 0 ) { + DBGC ( arbel, "Arbel %p command failed with status %02x:\n", + arbel, status ); + DBGC_HD ( arbel, &hcr, sizeof ( hcr ) ); + return -EIO; + } + + /* Read output parameters, if any */ + hcr.u.dwords[3] = readl ( arbel->config + ARBEL_HCR_REG ( 3 ) ); + hcr.u.dwords[4] = readl ( arbel->config + ARBEL_HCR_REG ( 4 ) ); + if ( op_fl & ARBEL_HCR_OUT_IMMEDIATE ) { + memcpy ( out_param, &hcr.u.dwords[3], 8 ); + } else if ( op_fl & ARBEL_HCR_OUT_MAILBOX ) { + memcpy ( out_param, arbel->mailbox_out, out_param_len ); + } + + return 0; +} + +/** + * Create completion queue + * + * @v ibdev Infiniband device + * @v + */ +static int arbel_create_cq ( struct ib_device *ibdev ) { + struct arbelprm_completion_queue_context *cqctx; -static struct ib_gid arbel_no_gid = { - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 } -}; +} + /** * Ring doorbell register in UAR @@ -310,7 +425,7 @@ static int arbel_post_send ( struct ib_device *ibdev, struct arbelprm_ud_send_wqe *wqe; union arbelprm_doorbell_record *db_rec; union arbelprm_doorbell_register db_reg; - struct ib_gid *gid; + const struct ib_gid *gid; unsigned int wqe_idx_mask; size_t nds; @@ -616,6 +731,9 @@ static int arbel_probe ( struct pci_device *pci, memcpy ( &mac->gid, ib_data.port_gid.raw, sizeof ( mac->gid ) ); /* Hack up IB structures */ + static_arbel.config = memfree_pci_dev.cr_space; + static_arbel.mailbox_in = dev_buffers_p->inprm_buf; + static_arbel.mailbox_out = dev_buffers_p->outprm_buf; static_arbel.uar = memfree_pci_dev.uar; static_arbel.db_rec = dev_ib_data.uar_context_base; static_arbel.reserved_lkey = dev_ib_data.mkey; @@ -634,6 +752,16 @@ static int arbel_probe ( struct pci_device *pci, list_add ( &static_ipoib_qp.recv.list, &static_ipoib_recv_cq.work_queues ); + uint8_t buf[512]; + memset ( buf, 0xaa, sizeof ( buf ) ); + if ( ( rc = arbel_command ( &static_arbel, + ( 0x03 | ARBEL_HCR_OUT_MAILBOX ), 0, + NULL, 0, 0, buf, 256 ) ) != 0 ) { + DBG ( "QUERY_DEV_LIM failed: %s\n", strerror ( rc ) ); + } + DBG ( "Device limits:\n "); + DBG_HD ( &buf[0], sizeof ( buf ) ); + /* Register network device */ if ( ( rc = register_netdev ( netdev ) ) != 0 ) goto err_register_netdev;