Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[infiniband] Add raw packet parser and constructor
This can be used with cards that require the driver to construct and
parse packet headers manually.  Headers are optionally handled
out-of-line from the packet payload, since some such cards will split
received headers into a separate ring buffer.
  • Loading branch information
Michael Brown committed Nov 11, 2008
1 parent c0ec00f commit 9e5fd8e
Show file tree
Hide file tree
Showing 5 changed files with 423 additions and 36 deletions.
234 changes: 234 additions & 0 deletions src/drivers/infiniband/ib_packet.c
@@ -0,0 +1,234 @@
/*
* Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/

#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <byteswap.h>
#include <gpxe/iobuf.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_packet.h>

/**
* @file
*
* Infiniband Packet Formats
*
*/

/**
* Add IB headers
*
* @v ibdev Infiniband device
* @v iobuf I/O buffer to contain headers
* @v qp Queue pair
* @v payload_len Payload length
* @v av Address vector
*/
int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
struct ib_queue_pair *qp, size_t payload_len,
const struct ib_address_vector *av ) {
struct ib_local_route_header *lrh;
struct ib_global_route_header *grh;
struct ib_base_transport_header *bth;
struct ib_datagram_extended_transport_header *deth;
size_t orig_iob_len = iob_len ( iobuf );
size_t pad_len;
size_t lrh_len;
size_t grh_len;
unsigned int vl;
unsigned int lnh;

DBGC2 ( ibdev, "IBDEV %p TX %04x:%08lx => %04x:%08lx (key %08lx)\n",
ibdev, ibdev->lid, qp->qpn, av->lid, av->qpn, av->qkey );

/* Calculate packet length */
pad_len = ( (-payload_len) & 0x3 );
payload_len += pad_len;
payload_len += 4; /* ICRC */

/* Reserve space for headers */
orig_iob_len = iob_len ( iobuf );
deth = iob_push ( iobuf, sizeof ( *deth ) );
bth = iob_push ( iobuf, sizeof ( *bth ) );
grh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
grh = ( av->gid_present ?
iob_push ( iobuf, sizeof ( *grh ) ) : NULL );
lrh = iob_push ( iobuf, sizeof ( *lrh ) );
lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );

/* Construct LRH */
vl = ( ( av->qpn == IB_QPN_SMP ) ? IB_VL_SMP : IB_VL_DEFAULT );
lrh->vl__lver = ( vl << 4 );
lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH );
lrh->sl__lnh = ( ( av->sl << 4 ) | lnh );
lrh->dlid = htons ( av->lid );
lrh->length = htons ( lrh_len >> 2 );
lrh->slid = htons ( ibdev->lid );

/* Construct GRH, if required */
if ( grh ) {
grh->ipver__tclass__flowlabel =
htonl ( IB_GRH_IPVER_IPv6 << 28 );
grh->paylen = htons ( grh_len );
grh->nxthdr = IB_GRH_NXTHDR_IBA;
grh->hoplmt = 0;
memcpy ( &grh->sgid, &ibdev->gid, sizeof ( grh->sgid ) );
memcpy ( &grh->dgid, &av->gid, sizeof ( grh->dgid ) );
}

/* Construct BTH */
bth->opcode = BTH_OPCODE_UD_SEND;
bth->se__m__padcnt__tver = ( pad_len << 4 );
bth->pkey = htons ( ibdev->pkey );
bth->dest_qp = htonl ( av->qpn );
bth->ack__psn = htonl ( ( ibdev->psn++ ) & 0xffffffUL );

/* Construct DETH */
deth->qkey = htonl ( av->qkey );
deth->src_qp = htonl ( qp->qpn );

DBGCP_HDA ( ibdev, 0, iobuf->data,
( iob_len ( iobuf ) - orig_iob_len ) );

return 0;
}

/**
* Remove IB headers
*
* @v ibdev Infiniband device
* @v iobuf I/O buffer containing headers
* @v qp Queue pair to fill in, or NULL
* @v payload_len Payload length to fill in, or NULL
* @v av Address vector to fill in
*/
int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
struct ib_queue_pair **qp, size_t *payload_len,
struct ib_address_vector *av ) {
struct ib_local_route_header *lrh;
struct ib_global_route_header *grh;
struct ib_base_transport_header *bth;
struct ib_datagram_extended_transport_header *deth;
size_t orig_iob_len = iob_len ( iobuf );
unsigned int lnh;
size_t pad_len;
unsigned long qpn;
unsigned int lid;

/* Clear return values */
if ( qp )
*qp = NULL;
if ( payload_len )
*payload_len = 0;
memset ( av, 0, sizeof ( *av ) );

/* Extract LRH */
if ( iob_len ( iobuf ) < sizeof ( *lrh ) ) {
DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for LRH\n",
ibdev, iob_len ( iobuf ) );
return -EINVAL;
}
lrh = iobuf->data;
iob_pull ( iobuf, sizeof ( *lrh ) );
av->lid = ntohs ( lrh->slid );
av->sl = ( lrh->sl__lnh >> 4 );
lnh = ( lrh->sl__lnh & 0x3 );
lid = ntohs ( lrh->dlid );

/* Reject unsupported packets */
if ( ! ( ( lnh == IB_LNH_BTH ) || ( lnh == IB_LNH_GRH ) ) ) {
DBGC ( ibdev, "IBDEV %p RX unsupported LNH %x\n",
ibdev, lnh );
return -ENOTSUP;
}

/* Extract GRH, if present */
if ( lnh == IB_LNH_GRH ) {
if ( iob_len ( iobuf ) < sizeof ( *grh ) ) {
DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) "
"for GRH\n", ibdev, iob_len ( iobuf ) );
return -EINVAL;
}
grh = iobuf->data;
iob_pull ( iobuf, sizeof ( *grh ) );
av->gid_present = 1;
memcpy ( &av->gid, &grh->sgid, sizeof ( av->gid ) );
} else {
grh = NULL;
}

/* Extract BTH */
if ( iob_len ( iobuf ) < sizeof ( *bth ) ) {
DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for BTH\n",
ibdev, iob_len ( iobuf ) );
return -EINVAL;
}
bth = iobuf->data;
iob_pull ( iobuf, sizeof ( *bth ) );
if ( bth->opcode != BTH_OPCODE_UD_SEND ) {
DBGC ( ibdev, "IBDEV %p unsupported BTH opcode %x\n",
ibdev, bth->opcode );
return -ENOTSUP;
}
qpn = ntohl ( bth->dest_qp );

/* Extract DETH */
if ( iob_len ( iobuf ) < sizeof ( *deth ) ) {
DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for DETH\n",
ibdev, iob_len ( iobuf ) );
return -EINVAL;
}
deth = iobuf->data;
iob_pull ( iobuf, sizeof ( *deth ) );
av->qpn = ntohl ( deth->src_qp );
av->qkey = ntohl ( deth->qkey );

/* Calculate payload length, if applicable */
if ( payload_len ) {
pad_len = ( ( bth->se__m__padcnt__tver >> 4 ) & 0x3 );
*payload_len = ( ( ntohs ( lrh->length ) << 2 )
- ( orig_iob_len - iob_len ( iobuf ) )
- pad_len - 4 /* ICRC */ );
}

/* Determine destination QP, if applicable */
if ( qp ) {
if ( IB_LID_MULTICAST ( lid ) && grh ) {
*qp = ib_find_qp_mgid ( ibdev, &grh->dgid );
} else {
*qp = ib_find_qp_qpn ( ibdev, qpn );
}
if ( ! *qp ) {
DBGC ( ibdev, "IBDEV %p RX for nonexistent QP\n",
ibdev );
return -ENODEV;
}
}

DBGC2 ( ibdev, "IBDEV %p RX %04x:%08lx <= %04x:%08lx (key %08lx)\n",
ibdev, lid,
( IB_LID_MULTICAST( lid ) ? ( qp ? (*qp)->qpn : -1UL ) : qpn ),
av->lid, av->qpn, ntohl ( deth->qkey ) );
DBGCP_HDA ( ibdev, 0,
( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ),
( orig_iob_len - iob_len ( iobuf ) ) );

return 0;
}
1 change: 1 addition & 0 deletions src/include/gpxe/errfile.h
Expand Up @@ -136,6 +136,7 @@
#define ERRFILE_dhcppkt ( ERRFILE_NET | 0x00150000 )
#define ERRFILE_slam ( ERRFILE_NET | 0x00160000 )
#define ERRFILE_ib_sma ( ERRFILE_NET | 0x00170000 )
#define ERRFILE_ib_packet ( ERRFILE_NET | 0x00180000 )

#define ERRFILE_image ( ERRFILE_IMAGE | 0x00000000 )
#define ERRFILE_elf ( ERRFILE_IMAGE | 0x00010000 )
Expand Down
35 changes: 34 additions & 1 deletion src/include/gpxe/ib_packet.h
Expand Up @@ -7,6 +7,11 @@
*
*/

struct ib_device;
struct ib_queue_pair;
struct ib_address_vector;
struct io_buffer;

/** Half of an Infiniband Global Identifier */
struct ib_gid_half {
uint8_t bytes[8];
Expand Down Expand Up @@ -53,6 +58,9 @@ enum ib_lnh {
/** Default Infiniband LID */
#define IB_LID_NONE 0xffff

/** Test for multicast LID */
#define IB_LID_MULTICAST( lid ) ( ( (lid) >= 0xc000 ) && ( (lid) <= 0xfffe ) )

/** An Infiniband Global Route Header */
struct ib_global_route_header {
/** IP version, traffic class, and flow label
Expand All @@ -76,7 +84,6 @@ struct ib_global_route_header {

#define IB_GRH_IPVER_IPv6 0x06
#define IB_GRH_NXTHDR_IBA 0x1b
#define IB_GRH_HOPLMT_MAX 0xff

/** An Infiniband Base Transport Header */
struct ib_base_transport_header {
Expand Down Expand Up @@ -111,4 +118,30 @@ struct ib_datagram_extended_transport_header {
uint32_t src_qp;
} __attribute__ (( packed ));

/** All known IB header formats */
union ib_headers {
struct ib_local_route_header lrh;
struct {
struct ib_local_route_header lrh;
struct ib_global_route_header grh;
struct ib_base_transport_header bth;
struct ib_datagram_extended_transport_header deth;
} __attribute__ (( packed )) lrh__grh__bth__deth;
struct {
struct ib_local_route_header lrh;
struct ib_base_transport_header bth;
struct ib_datagram_extended_transport_header deth;
} __attribute__ (( packed )) lrh__bth__deth;
} __attribute__ (( packed ));

/** Maximum size required for IB headers */
#define IB_MAX_HEADER_SIZE sizeof ( union ib_headers )

extern int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
struct ib_queue_pair *qp, size_t payload_len,
const struct ib_address_vector *av );
extern int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
struct ib_queue_pair **qp, size_t *payload_len,
struct ib_address_vector *av );

#endif /* _GPXE_IB_PACKET_H */

0 comments on commit 9e5fd8e

Please sign in to comment.