Skip to content

Commit

Permalink
[hermon] Add support for multiple ports and detecting non-IB ports
Browse files Browse the repository at this point in the history 
Originally-fixed-by: Itay Gazit <itaygazit@gmail.com>
  • Loading branch information
Michael Brown committed Aug 6, 2009
1 parent b0c5638 commit ad66465
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 31 deletions.
6 changes: 5 additions & 1 deletion src/drivers/infiniband/MT25408_PRM.h
View file
Expand Up @@ -2071,7 +2071,11 @@ struct hermonprm_query_dev_cap_st { /* Little Endian */
pseudo_bit_t pkv[0x00001]; /* PKey Violation Counter Supported */
pseudo_bit_t qkv[0x00001]; /* QKey Violation Coutner Supported */
pseudo_bit_t vmm[0x00001]; /* Hermon New */
pseudo_bit_t reserved27[0x00005];
pseudo_bit_t fcoe[0x00001];
pseudo_bit_t dpdp[0x00001]; /* Dual Port Different Protocols */
pseudo_bit_t raw_ethertype[0x00001];
pseudo_bit_t raw_ipv6[0x00001];
pseudo_bit_t blh[0x00001];
pseudo_bit_t mw[0x00001]; /* Memory windows supported */
pseudo_bit_t apm[0x00001]; /* Automatic Path Migration Supported */
pseudo_bit_t atm[0x00001]; /* Atomic operations supported (atomicity is guaranteed between QPs on this HCA) */
Expand Down
117 changes: 89 additions & 28 deletions src/drivers/infiniband/hermon.c
View file
Expand Up @@ -541,6 +541,16 @@ hermon_cmd_map_fa ( struct hermon *hermon,
0, map, 1, NULL );
}

static inline int
hermon_cmd_sense_port ( struct hermon *hermon, unsigned int port,
struct hermonprm_sense_port *port_type ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_SENSE_PORT,
1, sizeof ( *port_type ) ),
0, NULL, port, port_type );
}


/***************************************************************************
*
* Memory translation table operations
Expand Down Expand Up @@ -1664,7 +1674,7 @@ static void hermon_event_port_state_change ( struct hermon *hermon,
( link_up ? "up" : "down" ) );

/* Sanity check */
if ( port >= HERMON_NUM_PORTS ) {
if ( port >= hermon->cap.num_ports ) {
DBGC ( hermon, "Hermon %p port %d does not exist!\n",
hermon, ( port + 1 ) );
return;
Expand Down Expand Up @@ -1735,6 +1745,36 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) {
***************************************************************************
*/

/**
* Sense port type
*
* @v ibdev Infiniband device
* @ret port_type Port type, or negative error
*/
static int hermon_sense_port_type ( struct ib_device *ibdev ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermonprm_sense_port sense_port;
int port_type;
int rc;

/* If DPDP is not supported, always assume Infiniband */
if ( ! hermon->cap.dpdp )
return HERMON_PORT_TYPE_IB;

/* Sense the port type */
if ( ( rc = hermon_cmd_sense_port ( hermon, ibdev->port,
&sense_port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d sense failed: %s\n",
hermon, ibdev->port, strerror ( rc ) );
return rc;
}
port_type = MLX_GET ( &sense_port, port_type );

DBGC ( hermon, "Hermon %p port %d type %d\n",
hermon, ibdev->port, port_type );
return port_type;
}

/**
* Initialise Infiniband link
*
Expand All @@ -1744,8 +1784,19 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) {
static int hermon_open ( struct ib_device *ibdev ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermonprm_init_port init_port;
int port_type;
int rc;

/* Check we are connected to an Infiniband network */
if ( ( rc = port_type = hermon_sense_port_type ( ibdev ) ) < 0 )
return rc;
if ( port_type != HERMON_PORT_TYPE_IB ) {
DBGC ( hermon, "Hermon %p port %d not connected to an "
"Infiniband network", hermon, ibdev->port );
return -ENOTCONN;
}

/* Init Port */
memset ( &init_port, 0, sizeof ( init_port ) );
MLX_FILL_2 ( &init_port, 0,
port_width_cap, 3,
Expand Down Expand Up @@ -2099,6 +2150,15 @@ static int hermon_get_cap ( struct hermon *hermon ) {
( 1 << MLX_GET ( &dev_cap, log2_rsvd_mrws ) );
hermon->cap.dmpt_entry_size = MLX_GET ( &dev_cap, d_mpt_entry_sz );
hermon->cap.reserved_uars = MLX_GET ( &dev_cap, num_rsvd_uars );
hermon->cap.num_ports = MLX_GET ( &dev_cap, num_ports );
hermon->cap.dpdp = MLX_GET ( &dev_cap, dpdp );

/* Sanity check */
if ( hermon->cap.num_ports > HERMON_MAX_PORTS ) {
DBGC ( hermon, "Hermon %p has %d ports (only %d supported)\n",
hermon, hermon->cap.num_ports, HERMON_MAX_PORTS );
hermon->cap.num_ports = HERMON_MAX_PORTS;
}

return 0;
}
Expand Down Expand Up @@ -2478,7 +2538,7 @@ static int hermon_probe ( struct pci_device *pci,
struct hermon *hermon;
struct ib_device *ibdev;
struct hermonprm_init_hca init_hca;
int i;
unsigned int i;
int rc;

/* Allocate Hermon device */
Expand All @@ -2489,20 +2549,6 @@ static int hermon_probe ( struct pci_device *pci,
}
pci_set_drvdata ( pci, hermon );

/* Allocate Infiniband devices */
for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) {
ibdev = alloc_ibdev ( 0 );
if ( ! ibdev ) {
rc = -ENOMEM;
goto err_alloc_ibdev;
}
hermon->ibdev[i] = ibdev;
ibdev->op = &hermon_ib_operations;
ibdev->dev = &pci->dev;
ibdev->port = ( HERMON_PORT_BASE + i );
ib_set_drvdata ( ibdev, hermon );
}

/* Fix up PCI device */
adjust_pci_device ( pci );

Expand Down Expand Up @@ -2534,6 +2580,20 @@ static int hermon_probe ( struct pci_device *pci,
if ( ( rc = hermon_get_cap ( hermon ) ) != 0 )
goto err_get_cap;

/* Allocate Infiniband devices */
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
ibdev = alloc_ibdev ( 0 );
if ( ! ibdev ) {
rc = -ENOMEM;
goto err_alloc_ibdev;
}
hermon->ibdev[i] = ibdev;
ibdev->op = &hermon_ib_operations;
ibdev->dev = &pci->dev;
ibdev->port = ( HERMON_PORT_BASE + i );
ib_set_drvdata ( ibdev, hermon );
}

/* Allocate ICM */
memset ( &init_hca, 0, sizeof ( init_hca ) );
if ( ( rc = hermon_alloc_icm ( hermon, &init_hca ) ) != 0 )
Expand All @@ -2552,7 +2612,7 @@ static int hermon_probe ( struct pci_device *pci,
/* Set up memory protection */
if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 )
goto err_setup_mpt;
for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ )
for ( i = 0 ; i < hermon->cap.num_ports ; i++ )
hermon->ibdev[i]->rdma_key = hermon->lkey;

/* Set up event queue */
Expand All @@ -2563,12 +2623,13 @@ static int hermon_probe ( struct pci_device *pci,
if ( ( rc = hermon_configure_special_qps ( hermon ) ) != 0 )
goto err_conf_special_qps;

/* Update MAD parameters */
for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ )
/* Update IPoIB MAC address */
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
ib_smc_update ( hermon->ibdev[i], hermon_mad );
}

/* Register Infiniband devices */
for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) {
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
if ( ( rc = register_ibdev ( hermon->ibdev[i] ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not register IB "
"device: %s\n", hermon, strerror ( rc ) );
Expand All @@ -2578,9 +2639,9 @@ static int hermon_probe ( struct pci_device *pci,

return 0;

i = HERMON_NUM_PORTS;
i = hermon->cap.num_ports;
err_register_ibdev:
for ( i-- ; i >= 0 ; i-- )
for ( i-- ; ( signed int ) i >= 0 ; i-- )
unregister_ibdev ( hermon->ibdev[i] );
err_conf_special_qps:
hermon_destroy_eq ( hermon );
Expand All @@ -2590,17 +2651,17 @@ static int hermon_probe ( struct pci_device *pci,
err_init_hca:
hermon_free_icm ( hermon );
err_alloc_icm:
i = hermon->cap.num_ports;
err_alloc_ibdev:
for ( i-- ; ( signed int ) i >= 0 ; i-- )
ibdev_put ( hermon->ibdev[i] );
err_get_cap:
hermon_stop_firmware ( hermon );
err_start_firmware:
free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE );
err_mailbox_out:
free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
err_mailbox_in:
i = HERMON_NUM_PORTS;
err_alloc_ibdev:
for ( i-- ; i >= 0 ; i-- )
ibdev_put ( hermon->ibdev[i] );
free ( hermon );
err_alloc_hermon:
return rc;
Expand All @@ -2615,7 +2676,7 @@ static void hermon_remove ( struct pci_device *pci ) {
struct hermon *hermon = pci_get_drvdata ( pci );
int i;

for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- )
unregister_ibdev ( hermon->ibdev[i] );
hermon_destroy_eq ( hermon );
hermon_cmd_close_hca ( hermon );
Expand All @@ -2624,7 +2685,7 @@ static void hermon_remove ( struct pci_device *pci ) {
hermon_stop_firmware ( hermon );
free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE );
free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- )
ibdev_put ( hermon->ibdev[i] );
free ( hermon );
}
Expand Down
18 changes: 16 additions & 2 deletions src/drivers/infiniband/hermon.h
View file
Expand Up @@ -21,7 +21,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
*/

/* Ports in existence */
#define HERMON_NUM_PORTS 2
#define HERMON_MAX_PORTS 2
#define HERMON_PORT_BASE 1

/* PCI BARs */
Expand Down Expand Up @@ -61,6 +61,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
#define HERMON_HCR_READ_MCG 0x0025
#define HERMON_HCR_WRITE_MCG 0x0026
#define HERMON_HCR_MGID_HASH 0x0027
#define HERMON_HCR_SENSE_PORT 0x004d
#define HERMON_HCR_RUN_FW 0x0ff6
#define HERMON_HCR_DISABLE_LAM 0x0ff7
#define HERMON_HCR_ENABLE_LAM 0x0ff8
Expand Down Expand Up @@ -164,6 +165,14 @@ struct hermonprm_port_state_change_event_st {
struct hermonprm_port_state_change_st data;
} __attribute__ (( packed ));

/** Hermon sense port */
struct hermonprm_sense_port_st {
pseudo_bit_t port_type[0x00020];
/* -------------- */
pseudo_bit_t reserved[0x00020];
};
#define HERMON_PORT_TYPE_IB 1

/*
* Wrapper structures for hardware datatypes
*
Expand Down Expand Up @@ -192,6 +201,7 @@ struct MLX_DECLARE_STRUCT ( hermonprm_query_dev_cap );
struct MLX_DECLARE_STRUCT ( hermonprm_query_fw );
struct MLX_DECLARE_STRUCT ( hermonprm_queue_pair_ee_context_entry );
struct MLX_DECLARE_STRUCT ( hermonprm_scalar_parameter );
struct MLX_DECLARE_STRUCT ( hermonprm_sense_port );
struct MLX_DECLARE_STRUCT ( hermonprm_send_db_register );
struct MLX_DECLARE_STRUCT ( hermonprm_ud_address_vector );
struct MLX_DECLARE_STRUCT ( hermonprm_virtual_physical_mapping );
Expand Down Expand Up @@ -296,6 +306,10 @@ struct hermon_dev_cap {
size_t dmpt_entry_size;
/** Number of reserved UARs */
unsigned int reserved_uars;
/** Number of ports */
unsigned int num_ports;
/** Dual-port different protocol */
int dpdp;
};

/** Number of cMPT entries of each type */
Expand Down Expand Up @@ -523,7 +537,7 @@ struct hermon {
unsigned long qpn_base;

/** Infiniband devices */
struct ib_device *ibdev[HERMON_NUM_PORTS];
struct ib_device *ibdev[HERMON_MAX_PORTS];
};

/** Global protection domain */
Expand Down

0 comments on commit ad66465

Please sign in to comment.