Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[ipoib] Expose the real broadcast MAC
The IPoIB broadcast MAC address varies according to the partition key.
Now that the broadcast MAC address is a property of the network device
rather than of the link layer, we can expose this real MAC address
directly.

The broadcast LID is now identified via a path record lookup; this is
marginally inefficient (since it was present in the MCMemberRecord
GetResponse), but avoids the need to special-case broadcasts when
constructing the address vector in ipoib_transmit().
  • Loading branch information
Michael Brown committed Jul 17, 2009
1 parent 4d00296 commit 06ad481
Showing 1 changed file with 37 additions and 46 deletions.
83 changes: 37 additions & 46 deletions src/drivers/net/ipoib.c
Expand Up @@ -65,10 +65,8 @@ struct ipoib_device {
struct ib_queue_set data;
/** Data queue set */
struct ib_queue_set meta;
/** Broadcast GID */
struct ib_gid broadcast_gid;
/** Broadcast LID */
unsigned int broadcast_lid;
/** Broadcast MAC */
struct ipoib_mac broadcast;
/** Data queue key */
unsigned long data_qkey;
/** Attached to multicast group
Expand Down Expand Up @@ -254,9 +252,10 @@ static int ipoib_push ( struct net_device *netdev __unused,
* @ret net_proto Network-layer protocol, in network-byte order
* @ret rc Return status code
*/
static int ipoib_pull ( struct net_device *netdev __unused,
static int ipoib_pull ( struct net_device *netdev,
struct io_buffer *iobuf, const void **ll_dest,
const void **ll_source, uint16_t *net_proto ) {
struct ipoib_device *ipoib = netdev->priv;
struct ipoib_hdr *ipoib_hdr = iobuf->data;
struct ipoib_peer *dest;
struct ipoib_peer *source;
Expand All @@ -279,8 +278,8 @@ static int ipoib_pull ( struct net_device *netdev __unused,
ipoib_hdr->u.reserved = 0;

/* Fill in required fields */
*ll_dest = ( dest ? &dest->mac : &ipoib_broadcast );
*ll_source = ( source ? &source->mac : &ipoib_broadcast );
*ll_dest = ( dest ? &dest->mac : &ipoib->broadcast );
*ll_source = ( source ? &source->mac : &ipoib->broadcast );
*net_proto = ipoib_hdr->proto;

return 0;
Expand Down Expand Up @@ -331,6 +330,24 @@ struct ll_protocol ipoib_protocol __ll_protocol = {
.mc_hash = ipoib_mc_hash,
};

/**
* Allocate IPoIB device
*
* @v priv_size Size of driver private data
* @ret netdev Network device, or NULL
*/
struct net_device * alloc_ipoibdev ( size_t priv_size ) {
struct net_device *netdev;

netdev = alloc_netdev ( priv_size );
if ( netdev ) {
netdev->ll_protocol = &ipoib_protocol;
netdev->ll_broadcast = ( uint8_t * ) &ipoib_broadcast;
netdev->max_pkt_len = IB_MAX_PAYLOAD_SIZE;
}
return netdev;
}

/****************************************************************************
*
* IPoIB network device
Expand Down Expand Up @@ -439,17 +456,10 @@ static int ipoib_transmit ( struct net_device *netdev,
av.qpn = ntohl ( dest->mac.qpn );
av.qkey = ipoib->data_qkey;
av.gid_present = 1;
if ( av.qpn == IB_QPN_BROADCAST ) {
/* Broadcast */
av.lid = ipoib->broadcast_lid;
memcpy ( &av.gid, &ipoib->broadcast_gid, sizeof ( av.gid ) );
} else {
/* Unicast */
memcpy ( &av.gid, &dest->mac.gid, sizeof ( av.gid ) );
if ( ( rc = ib_resolve_path ( ibdev, &av ) ) != 0 ) {
/* Path not resolved yet */
return rc;
}
memcpy ( &av.gid, &dest->mac.gid, sizeof ( av.gid ) );
if ( ( rc = ib_resolve_path ( ibdev, &av ) ) != 0 ) {
/* Path not resolved yet */
return rc;
}

return ib_post_send ( ibdev, ipoib->data.qp, &av, iobuf );
Expand Down Expand Up @@ -554,10 +564,8 @@ static void ipoib_recv_mc_member_record ( struct ipoib_device *ipoib,
/* Record parameters */
joined = ( mc_member_record->scope__join_state & 0x0f );
ipoib->data_qkey = ntohl ( mc_member_record->qkey );
ipoib->broadcast_lid = ntohs ( mc_member_record->mlid );
DBGC ( ipoib, "IPoIB %p %s broadcast group: qkey %lx mlid %x\n",
ipoib, ( joined ? "joined" : "left" ), ipoib->data_qkey,
ipoib->broadcast_lid );
DBGC ( ipoib, "IPoIB %p %s broadcast group: qkey %lx\n",
ipoib, ( joined ? "joined" : "left" ), ipoib->data_qkey );

/* Update data queue pair qkey */
if ( ( rc = ib_modify_qp ( ipoib->ibdev, ipoib->data.qp,
Expand Down Expand Up @@ -666,15 +674,15 @@ static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
/* Attach data queue to broadcast multicast GID */
assert ( ipoib->broadcast_attached == 0 );
if ( ( rc = ib_mcast_attach ( ipoib->ibdev, ipoib->data.qp,
&ipoib->broadcast_gid ) ) != 0 ){
&ipoib->broadcast.gid ) ) != 0 ){
DBGC ( ipoib, "IPoIB %p could not attach to broadcast GID: "
"%s\n", ipoib, strerror ( rc ) );
return rc;
}
ipoib->broadcast_attached = 1;

/* Initiate broadcast group join */
if ( ( rc = ipoib_mc_member_record ( ipoib, &ipoib->broadcast_gid,
if ( ( rc = ipoib_mc_member_record ( ipoib, &ipoib->broadcast.gid,
1 ) ) != 0 ) {
DBGC ( ipoib, "IPoIB %p could not send broadcast join: %s\n",
ipoib, strerror ( rc ) );
Expand All @@ -699,7 +707,7 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
if ( ipoib->broadcast_attached ) {
assert ( ipoib->data.qp != NULL );
ib_mcast_detach ( ipoib->ibdev, ipoib->data.qp,
&ipoib->broadcast_gid );
&ipoib->broadcast.gid );
ipoib->broadcast_attached = 0;
}
}
Expand Down Expand Up @@ -825,9 +833,9 @@ static void ipoib_set_ib_params ( struct ipoib_device *ipoib ) {
memcpy ( &mac->gid, &ibdev->gid, sizeof ( mac->gid ) );

/* Calculate broadcast GID based on partition key */
memcpy ( &ipoib->broadcast_gid, &ipoib_broadcast.gid,
sizeof ( ipoib->broadcast_gid ) );
ipoib->broadcast_gid.u.words[2] = htons ( ibdev->pkey );
memcpy ( &ipoib->broadcast, &ipoib_broadcast,
sizeof ( ipoib->broadcast ) );
ipoib->broadcast.gid.u.words[2] = htons ( ibdev->pkey );

/* Set net device link state to reflect Infiniband link state */
if ( ib_link_ok ( ibdev ) ) {
Expand Down Expand Up @@ -882,6 +890,7 @@ int ipoib_probe ( struct ib_device *ibdev ) {
ipoib = netdev->priv;
ib_set_ownerdata ( ibdev, netdev );
netdev->dev = ibdev->dev;
netdev->ll_broadcast = ( ( uint8_t * ) &ipoib->broadcast );
memset ( ipoib, 0, sizeof ( *ipoib ) );
ipoib->netdev = netdev;
ipoib->ibdev = ibdev;
Expand Down Expand Up @@ -916,21 +925,3 @@ void ipoib_remove ( struct ib_device *ibdev ) {
netdev_nullify ( netdev );
netdev_put ( netdev );
}

/**
* Allocate IPoIB device
*
* @v priv_size Size of driver private data
* @ret netdev Network device, or NULL
*/
struct net_device * alloc_ipoibdev ( size_t priv_size ) {
struct net_device *netdev;

netdev = alloc_netdev ( priv_size );
if ( netdev ) {
netdev->ll_protocol = &ipoib_protocol;
netdev->ll_broadcast = ( uint8_t * ) &ipoib_broadcast;
netdev->max_pkt_len = IB_MAX_PAYLOAD_SIZE;
}
return netdev;
}

0 comments on commit 06ad481

Please sign in to comment.