Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[infiniband] Allow external QPN to differ from real QPN
Most IB hardware seems not to allow allocation of the genuine QPNs 0
and 1, so allow for the externally-visible QPN (as constructed and
parsed by ib_packet, where used) to differ from the real
hardware-allocated QPN.
  • Loading branch information
Michael Brown committed Jul 17, 2009
1 parent 92cf240 commit f1d92fa
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 8 deletions.
9 changes: 8 additions & 1 deletion src/include/gpxe/infiniband.h
Expand Up @@ -96,8 +96,15 @@ struct ib_queue_pair {
struct ib_device *ibdev;
/** List of queue pairs on this Infiniband device */
struct list_head list;
/** Queue Pair Number */
/** Queue pair number */
unsigned long qpn;
/** Externally-visible queue pair number
*
* This may differ from the real queue pair number (e.g. when
* the HCA cannot use the management QPNs 0 and 1 as hardware
* QPNs and needs to remap them).
*/
unsigned long ext_qpn;
/** Queue pair type */
enum ib_queue_pair_type type;
/** Queue key */
Expand Down
21 changes: 19 additions & 2 deletions src/net/infiniband.c
Expand Up @@ -196,7 +196,6 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
"%s\n", ibdev, strerror ( rc ) );
goto err_dev_create_qp;
}

DBGC ( ibdev, "IBDEV %p created queue pair %p (%p) with QPN %#lx\n",
ibdev, qp, ib_qp_get_drvdata ( qp ), qp->qpn );
DBGC ( ibdev, "IBDEV %p QPN %#lx has %d send entries at [%p,%p)\n",
Expand All @@ -205,6 +204,24 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
DBGC ( ibdev, "IBDEV %p QPN %#lx has %d receive entries at [%p,%p)\n",
ibdev, qp->qpn, num_recv_wqes, qp->recv.iobufs,
( ( ( void * ) qp ) + total_size ) );

/* Calculate externally-visible QPN */
switch ( type ) {
case IB_QPT_SMA:
qp->ext_qpn = IB_QPN_SMA;
break;
case IB_QPT_GMA:
qp->ext_qpn = IB_QPN_GMA;
break;
default:
qp->ext_qpn = qp->qpn;
break;
}
if ( qp->ext_qpn != qp->qpn ) {
DBGC ( ibdev, "IBDEV %p QPN %#lx has external QPN %#lx\n",
ibdev, qp->qpn, qp->ext_qpn );
}

return qp;

ibdev->op->destroy_qp ( ibdev, qp );
Expand Down Expand Up @@ -295,7 +312,7 @@ struct ib_queue_pair * ib_find_qp_qpn ( struct ib_device *ibdev,
struct ib_queue_pair *qp;

list_for_each_entry ( qp, &ibdev->qps, list ) {
if ( qp->qpn == qpn )
if ( ( qpn == qp->qpn ) || ( qpn == qp->ext_qpn ) )
return qp;
}
return NULL;
Expand Down
10 changes: 5 additions & 5 deletions src/net/infiniband/ib_packet.c
Expand Up @@ -58,7 +58,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
unsigned int lnh;

DBGC2 ( ibdev, "IBDEV %p TX %04x:%08lx => %04x:%08lx (key %08lx)\n",
ibdev, ibdev->lid, qp->qpn, av->lid, av->qpn, av->qkey );
ibdev, ibdev->lid, qp->ext_qpn, av->lid, av->qpn, av->qkey );

/* Calculate packet length */
pad_len = ( (-payload_len) & 0x3 );
Expand All @@ -76,7 +76,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );

/* Construct LRH */
vl = ( ( av->qpn == IB_QPN_SMA ) ? IB_VL_SMP : IB_VL_DEFAULT );
vl = ( ( qp->ext_qpn == IB_QPN_SMA ) ? IB_VL_SMP : IB_VL_DEFAULT );
lrh->vl__lver = ( vl << 4 );
lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH );
lrh->sl__lnh = ( ( av->sl << 4 ) | lnh );
Expand Down Expand Up @@ -104,7 +104,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,

/* Construct DETH */
deth->qkey = htonl ( av->qkey );
deth->src_qp = htonl ( qp->qpn );
deth->src_qp = htonl ( qp->ext_qpn );

DBGCP_HDA ( ibdev, 0, iobuf->data,
( iob_len ( iobuf ) - orig_iob_len ) );
Expand Down Expand Up @@ -233,8 +233,8 @@ int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
}

DBGC2 ( ibdev, "IBDEV %p RX %04x:%08lx <= %04x:%08lx (key %08x)\n",
ibdev, lid,
( IB_LID_MULTICAST( lid ) ? ( qp ? (*qp)->qpn : -1UL ) : qpn ),
ibdev, lid, ( IB_LID_MULTICAST( lid ) ?
( qp ? (*qp)->ext_qpn : -1UL ) : qpn ),
av->lid, av->qpn, ntohl ( deth->qkey ) );
DBGCP_HDA ( ibdev, 0,
( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ),
Expand Down

0 comments on commit f1d92fa

Please sign in to comment.