Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[infiniband] Add infrastructure for RC queue pairs
Queue pairs are now assumed to be created in the INIT state, with a
call to ib_modify_qp() required to bring the queue pair to the RTS
state.

ib_modify_qp() no longer takes a modification list; callers should
modify the relevant queue pair parameters (e.g. qkey) directly and
then call ib_modify_qp() to synchronise the changes to the hardware.

The packet sequence number is now a property of the queue pair, rather
than of the device.

Each queue pair may have an associated address vector.  For RC queue
pairs, this is the address vector that will be programmed in to the
hardware as the remote address.  For UD queue pairs, it will be used
as the default address vector if none is supplied to ib_post_send().
  • Loading branch information
Michael Brown committed Jul 17, 2009
1 parent ea6eb7f commit c939bc5
Show file tree
Hide file tree
Showing 9 changed files with 148 additions and 93 deletions.
12 changes: 2 additions & 10 deletions src/drivers/infiniband/arbel.c
Expand Up @@ -855,7 +855,6 @@ static int arbel_create_qp ( struct ib_device *ibdev,
( virt_to_bus ( arbel_qp->recv.wqe ) >> 6 ) );
MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.rcv_db_record_index,
arbel_qp->recv.doorbell_idx );
MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
if ( ( rc = arbel_cmd_rst2init_qpee ( arbel, qp->qpn, &qpctx )) != 0 ){
DBGC ( arbel, "Arbel %p RST2INIT_QPEE failed: %s\n",
arbel, strerror ( rc ) );
Expand Down Expand Up @@ -908,24 +907,17 @@ static int arbel_create_qp ( struct ib_device *ibdev,
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v mod_list Modification list
* @ret rc Return status code
*/
static int arbel_modify_qp ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
unsigned long mod_list ) {
struct ib_queue_pair *qp ) {
struct arbel *arbel = ib_get_drvdata ( ibdev );
struct arbelprm_qp_ee_state_transitions qpctx;
unsigned long optparammask = 0;
int rc;

/* Construct optparammask */
if ( mod_list & IB_MODIFY_QKEY )
optparammask |= ARBEL_QPEE_OPT_PARAM_QKEY;

/* Issue RTS2RTS_QP */
memset ( &qpctx, 0, sizeof ( qpctx ) );
MLX_FILL_1 ( &qpctx, 0, opt_param_mask, optparammask );
MLX_FILL_1 ( &qpctx, 0, opt_param_mask, ARBEL_QPEE_OPT_PARAM_QKEY );
MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
if ( ( rc = arbel_cmd_rts2rts_qp ( arbel, qp->qpn, &qpctx ) ) != 0 ){
DBGC ( arbel, "Arbel %p RTS2RTS_QP failed: %s\n",
Expand Down
12 changes: 2 additions & 10 deletions src/drivers/infiniband/hermon.c
Expand Up @@ -885,7 +885,6 @@ static int hermon_create_qp ( struct ib_device *ibdev,
MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn );
MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.db_record_addr_l,
( virt_to_phys ( &hermon_qp->recv.doorbell ) >> 2 ) );
MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
MLX_FILL_1 ( &qpctx, 53, qpc_eec_data.mtt_base_addr_l,
( hermon_qp->mtt.mtt_base_addr >> 3 ) );
if ( ( rc = hermon_cmd_rst2init_qp ( hermon, qp->qpn,
Expand Down Expand Up @@ -946,24 +945,17 @@ static int hermon_create_qp ( struct ib_device *ibdev,
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v mod_list Modification list
* @ret rc Return status code
*/
static int hermon_modify_qp ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
unsigned long mod_list ) {
struct ib_queue_pair *qp ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermonprm_qp_ee_state_transitions qpctx;
unsigned long optparammask = 0;
int rc;

/* Construct optparammask */
if ( mod_list & IB_MODIFY_QKEY )
optparammask |= HERMON_QP_OPT_PARAM_QKEY;

/* Issue RTS2RTS_QP */
memset ( &qpctx, 0, sizeof ( qpctx ) );
MLX_FILL_1 ( &qpctx, 0, opt_param_mask, optparammask );
MLX_FILL_1 ( &qpctx, 0, opt_param_mask, HERMON_QP_OPT_PARAM_QKEY );
MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
if ( ( rc = hermon_cmd_rts2rts_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ){
DBGC ( hermon, "Hermon %p RTS2RTS_QP failed: %s\n",
Expand Down
17 changes: 14 additions & 3 deletions src/drivers/infiniband/linda.c
Expand Up @@ -268,6 +268,18 @@ static int linda_set_port_info ( struct ib_device *ibdev, union ib_mad *mad ) {
return 0;
}

/**
* Set partition key table
*
* @v ibdev Infiniband device
* @v mad Set partition key table MAD
*/
static int linda_set_pkey_table ( struct ib_device *ibdev __unused,
union ib_mad *mad __unused ) {
/* Nothing to do */
return 0;
}

/***************************************************************************
*
* Context allocation
Expand Down Expand Up @@ -852,12 +864,10 @@ static int linda_create_qp ( struct ib_device *ibdev,
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v mod_list Modification list
* @ret rc Return status code
*/
static int linda_modify_qp ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
unsigned long mod_list __unused ) {
struct ib_queue_pair *qp ) {
struct linda *linda = ib_get_drvdata ( ibdev );

/* Nothing to do; the hardware doesn't have a notion of queue
Expand Down Expand Up @@ -1456,6 +1466,7 @@ static struct ib_device_operations linda_ib_operations = {
.mcast_attach = linda_mcast_attach,
.mcast_detach = linda_mcast_detach,
.set_port_info = linda_set_port_info,
.set_pkey_table = linda_set_pkey_table,
};

/***************************************************************************
Expand Down
2 changes: 1 addition & 1 deletion src/drivers/net/ipoib.c
Expand Up @@ -521,7 +521,7 @@ static int ipoib_open ( struct net_device *netdev ) {
/* Allocate queue pair */
ipoib->qp = ib_create_qp ( ibdev, IB_QPT_UD,
IPOIB_NUM_SEND_WQES, ipoib->cq,
IPOIB_NUM_RECV_WQES, ipoib->cq, 0 );
IPOIB_NUM_RECV_WQES, ipoib->cq );
if ( ! ipoib->qp ) {
DBGC ( ipoib, "IPoIB %p could not allocate queue pair\n",
ipoib );
Expand Down
114 changes: 63 additions & 51 deletions src/include/gpxe/infiniband.h
Expand Up @@ -47,6 +47,43 @@ struct ib_address_vector;
struct ib_completion_queue;
struct ib_gma;

/** Infiniband transmission rates */
enum ib_rate {
IB_RATE_2_5 = 2,
IB_RATE_10 = 3,
IB_RATE_30 = 4,
IB_RATE_5 = 5,
IB_RATE_20 = 6,
IB_RATE_40 = 7,
IB_RATE_60 = 8,
IB_RATE_80 = 9,
IB_RATE_120 = 10,
};

/** An Infiniband Address Vector */
struct ib_address_vector {
/** Queue Pair Number */
unsigned long qpn;
/** Queue key
*
* Not specified for received packets.
*/
unsigned long qkey;
/** Local ID */
unsigned int lid;
/** Rate
*
* Not specified for received packets.
*/
enum ib_rate rate;
/** Service level */
unsigned int sl;
/** GID is present */
unsigned int gid_present;
/** GID, if present */
struct ib_gid gid;
};

/** An Infiniband Work Queue */
struct ib_work_queue {
/** Containing queue pair */
Expand All @@ -57,6 +94,8 @@ struct ib_work_queue {
struct ib_completion_queue *cq;
/** List of work queues on this completion queue */
struct list_head list;
/** Packet sequence number */
uint32_t psn;
/** Number of work queue entries */
unsigned int num_wqes;
/** Number of occupied work queue entries */
Expand Down Expand Up @@ -88,6 +127,7 @@ enum ib_queue_pair_type {
IB_QPT_SMA,
IB_QPT_GMA,
IB_QPT_UD,
IB_QPT_RC,
};

/** An Infiniband Queue Pair */
Expand Down Expand Up @@ -115,54 +155,14 @@ struct ib_queue_pair {
struct ib_work_queue recv;
/** List of multicast GIDs */
struct list_head mgids;
/** Address vector */
struct ib_address_vector av;
/** Driver private data */
void *drv_priv;
/** Queue owner private data */
void *owner_priv;
};

/** Infiniband queue pair modification flags */
enum ib_queue_pair_mods {
IB_MODIFY_QKEY = 0x0001,
};

/** An Infiniband Address Vector */
struct ib_address_vector {
/** Queue Pair Number */
unsigned long qpn;
/** Queue key
*
* Not specified for received packets.
*/
unsigned long qkey;
/** Local ID */
unsigned int lid;
/** Rate
*
* Not specified for received packets.
*/
unsigned int rate;
/** Service level */
unsigned int sl;
/** GID is present */
unsigned int gid_present;
/** GID, if present */
struct ib_gid gid;
};

/** Infiniband transmission rates */
enum ib_rate {
IB_RATE_2_5 = 2,
IB_RATE_10 = 3,
IB_RATE_30 = 4,
IB_RATE_5 = 5,
IB_RATE_20 = 6,
IB_RATE_40 = 7,
IB_RATE_60 = 8,
IB_RATE_80 = 9,
IB_RATE_120 = 10,
};

/** Infiniband completion queue operations */
struct ib_completion_queue_operations {
/**
Expand Down Expand Up @@ -250,12 +250,10 @@ struct ib_device_operations {
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v mod_list Modification list
* @ret rc Return status code
*/
int ( * modify_qp ) ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
unsigned long mod_list );
struct ib_queue_pair *qp );
/** Destroy queue pair
*
* @v ibdev Infiniband device
Expand Down Expand Up @@ -352,6 +350,16 @@ struct ib_device_operations {
* an embedded SMA.
*/
int ( * set_port_info ) ( struct ib_device *ibdev, union ib_mad *mad );
/** Set partition key table
*
* @v ibdev Infiniband device
* @v mad Set partition key table MAD
*
* This method is required only by adapters that do not have
* an embedded SMA.
*/
int ( * set_pkey_table ) ( struct ib_device *ibdev,
union ib_mad *mad );
};

/** An Infiniband device */
Expand Down Expand Up @@ -398,8 +406,12 @@ struct ib_device {
/** Partition key */
uint16_t pkey;

/** Outbound packet sequence number */
uint32_t psn;
/** RDMA key
*
* This is a single key allowing unrestricted access to
* memory.
*/
uint32_t rdma_key;

/** Subnet management agent */
struct ib_gma *sma;
Expand All @@ -422,10 +434,9 @@ extern void ib_poll_cq ( struct ib_device *ibdev,
extern struct ib_queue_pair *
ib_create_qp ( struct ib_device *ibdev, enum ib_queue_pair_type type,
unsigned int num_send_wqes, struct ib_completion_queue *send_cq,
unsigned int num_recv_wqes, struct ib_completion_queue *recv_cq,
unsigned long qkey );
extern int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp,
unsigned long mod_list, unsigned long qkey );
unsigned int num_recv_wqes,
struct ib_completion_queue *recv_cq );
extern int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp );
extern void ib_destroy_qp ( struct ib_device *ibdev,
struct ib_queue_pair *qp );
extern struct ib_queue_pair * ib_find_qp_qpn ( struct ib_device *ibdev,
Expand Down Expand Up @@ -457,6 +468,7 @@ extern void ib_mcast_detach ( struct ib_device *ibdev,
extern int ib_get_hca_info ( struct ib_device *ibdev,
struct ib_gid_half *hca_guid );
extern int ib_set_port_info ( struct ib_device *ibdev, union ib_mad *mad );
extern int ib_set_pkey_table ( struct ib_device *ibdev, union ib_mad *mad );
extern struct ib_device * alloc_ibdev ( size_t priv_size );
extern int register_ibdev ( struct ib_device *ibdev );
extern void unregister_ibdev ( struct ib_device *ibdev );
Expand Down

0 comments on commit c939bc5

Please sign in to comment.