[infiniband] Add raw packet parser and constructor

This can be used with cards that require the driver to construct and parse packet headers manually. Headers are optionally handled out-of-line from the packet payload, since some such cards will split received headers into a separate ring buffer.
ipxe · Nov 11, 2008 · 9e5fd8e · 9e5fd8e
1 parent c0ec00f
commit 9e5fd8e
Show file tree

Hide file tree

Showing 5 changed files with 423 additions and 36 deletions.
diff --git a/src/drivers/infiniband/ib_packet.c b/src/drivers/infiniband/ib_packet.c
@@ -0,0 +1,234 @@
+/*
+ * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <gpxe/iobuf.h>
+#include <gpxe/infiniband.h>
+#include <gpxe/ib_packet.h>
+
+/**
+ * @file
+ *
+ * Infiniband Packet Formats
+ *
+ */
+
+/**
+ * Add IB headers
+ *
+ * @v ibdev		Infiniband device
+ * @v iobuf		I/O buffer to contain headers
+ * @v qp		Queue pair
+ * @v payload_len	Payload length
+ * @v av		Address vector
+ */
+int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
+	      struct ib_queue_pair *qp, size_t payload_len,
+	      const struct ib_address_vector *av ) {
+	struct ib_local_route_header *lrh;
+	struct ib_global_route_header *grh;
+	struct ib_base_transport_header *bth;
+	struct ib_datagram_extended_transport_header *deth;
+	size_t orig_iob_len = iob_len ( iobuf );
+	size_t pad_len;
+	size_t lrh_len;
+	size_t grh_len;
+	unsigned int vl;
+	unsigned int lnh;
+
+	DBGC2 ( ibdev, "IBDEV %p TX %04x:%08lx => %04x:%08lx (key %08lx)\n",
+		ibdev, ibdev->lid, qp->qpn, av->lid, av->qpn, av->qkey );
+
+	/* Calculate packet length */
+	pad_len = ( (-payload_len) & 0x3 );
+	payload_len += pad_len;
+	payload_len += 4; /* ICRC */
+
+	/* Reserve space for headers */
+	orig_iob_len = iob_len ( iobuf );
+	deth = iob_push ( iobuf, sizeof ( *deth ) );
+	bth = iob_push ( iobuf, sizeof ( *bth ) );
+	grh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
+	grh = ( av->gid_present ?
+		iob_push ( iobuf, sizeof ( *grh ) ) : NULL );
+	lrh = iob_push ( iobuf, sizeof ( *lrh ) );
+	lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
+
+	/* Construct LRH */
+	vl = ( ( av->qpn == IB_QPN_SMP ) ? IB_VL_SMP : IB_VL_DEFAULT );
+	lrh->vl__lver = ( vl << 4 );
+	lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH );
+	lrh->sl__lnh = ( ( av->sl << 4 ) | lnh );
+	lrh->dlid = htons ( av->lid );
+	lrh->length = htons ( lrh_len >> 2 );
+	lrh->slid = htons ( ibdev->lid );
+
+	/* Construct GRH, if required */
+	if ( grh ) {
+		grh->ipver__tclass__flowlabel =
+			htonl ( IB_GRH_IPVER_IPv6 << 28 );
+		grh->paylen = htons ( grh_len );
+		grh->nxthdr = IB_GRH_NXTHDR_IBA;
+		grh->hoplmt = 0;
+		memcpy ( &grh->sgid, &ibdev->gid, sizeof ( grh->sgid ) );
+		memcpy ( &grh->dgid, &av->gid, sizeof ( grh->dgid ) );
+	}
+
+	/* Construct BTH */
+	bth->opcode = BTH_OPCODE_UD_SEND;
+	bth->se__m__padcnt__tver = ( pad_len << 4 );
+	bth->pkey = htons ( ibdev->pkey );
+	bth->dest_qp = htonl ( av->qpn );
+	bth->ack__psn = htonl ( ( ibdev->psn++ ) & 0xffffffUL );
+
+	/* Construct DETH */
+	deth->qkey = htonl ( av->qkey );
+	deth->src_qp = htonl ( qp->qpn );
+
+	DBGCP_HDA ( ibdev, 0, iobuf->data,
+		    ( iob_len ( iobuf ) - orig_iob_len ) );
+
+	return 0;
+}
+
+/**
+ * Remove IB headers
+ *
+ * @v ibdev		Infiniband device
+ * @v iobuf		I/O buffer containing headers
+ * @v qp		Queue pair to fill in, or NULL
+ * @v payload_len	Payload length to fill in, or NULL
+ * @v av		Address vector to fill in
+ */
+int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
+	      struct ib_queue_pair **qp, size_t *payload_len,
+	      struct ib_address_vector *av ) {
+	struct ib_local_route_header *lrh;
+	struct ib_global_route_header *grh;
+	struct ib_base_transport_header *bth;
+	struct ib_datagram_extended_transport_header *deth;
+	size_t orig_iob_len = iob_len ( iobuf );
+	unsigned int lnh;
+	size_t pad_len;
+	unsigned long qpn;
+	unsigned int lid;
+
+	/* Clear return values */
+	if ( qp )
+		*qp = NULL;
+	if ( payload_len )
+		*payload_len = 0;
+	memset ( av, 0, sizeof ( *av ) );
+
+	/* Extract LRH */
+	if ( iob_len ( iobuf ) < sizeof ( *lrh ) ) {
+		DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for LRH\n",
+		       ibdev, iob_len ( iobuf ) );
+		return -EINVAL;
+	}
+	lrh = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *lrh ) );
+	av->lid = ntohs ( lrh->slid );
+	av->sl = ( lrh->sl__lnh >> 4 );
+	lnh = ( lrh->sl__lnh & 0x3 );
+	lid = ntohs ( lrh->dlid );
+
+	/* Reject unsupported packets */
+	if ( ! ( ( lnh == IB_LNH_BTH ) || ( lnh == IB_LNH_GRH ) ) ) {
+		DBGC ( ibdev, "IBDEV %p RX unsupported LNH %x\n",
+		       ibdev, lnh );
+		return -ENOTSUP;
+	}
+
+	/* Extract GRH, if present */
+	if ( lnh == IB_LNH_GRH ) {
+		if ( iob_len ( iobuf ) < sizeof ( *grh ) ) {
+			DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) "
+			       "for GRH\n", ibdev, iob_len ( iobuf ) );
+			return -EINVAL;
+		}
+		grh = iobuf->data;
+		iob_pull ( iobuf, sizeof ( *grh ) );
+		av->gid_present = 1;
+		memcpy ( &av->gid, &grh->sgid, sizeof ( av->gid ) );
+	} else {
+		grh = NULL;
+	}
+
+	/* Extract BTH */
+	if ( iob_len ( iobuf ) < sizeof ( *bth ) ) {
+		DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for BTH\n",
+		       ibdev, iob_len ( iobuf ) );
+		return -EINVAL;
+	}
+	bth = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *bth ) );
+	if ( bth->opcode != BTH_OPCODE_UD_SEND ) {
+		DBGC ( ibdev, "IBDEV %p unsupported BTH opcode %x\n",
+		       ibdev, bth->opcode );
+		return -ENOTSUP;
+	}
+	qpn = ntohl ( bth->dest_qp );
+
+	/* Extract DETH */
+	if ( iob_len ( iobuf ) < sizeof ( *deth ) ) {
+		DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for DETH\n",
+		       ibdev, iob_len ( iobuf ) );
+		return -EINVAL;
+	}
+	deth = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *deth ) );
+	av->qpn = ntohl ( deth->src_qp );
+	av->qkey = ntohl ( deth->qkey );
+
+	/* Calculate payload length, if applicable */
+	if ( payload_len ) {
+		pad_len = ( ( bth->se__m__padcnt__tver >> 4 ) & 0x3 );
+		*payload_len = ( ( ntohs ( lrh->length ) << 2 )
+				 - ( orig_iob_len - iob_len ( iobuf ) )
+				 - pad_len - 4 /* ICRC */ );
+	}
+
+	/* Determine destination QP, if applicable */
+	if ( qp ) {
+		if ( IB_LID_MULTICAST ( lid ) && grh ) {
+			*qp = ib_find_qp_mgid ( ibdev, &grh->dgid );
+		} else {
+			*qp = ib_find_qp_qpn ( ibdev, qpn );
+		}
+		if ( ! *qp ) {
+			DBGC ( ibdev, "IBDEV %p RX for nonexistent QP\n",
+			       ibdev );
+			return -ENODEV;
+		}
+	}
+
+	DBGC2 ( ibdev, "IBDEV %p RX %04x:%08lx <= %04x:%08lx (key %08lx)\n",
+		ibdev, lid,
+		( IB_LID_MULTICAST( lid ) ? ( qp ? (*qp)->qpn : -1UL ) : qpn ),
+		av->lid, av->qpn, ntohl ( deth->qkey ) );
+	DBGCP_HDA ( ibdev, 0,
+		    ( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ),
+		    ( orig_iob_len - iob_len ( iobuf ) ) );
+
+	return 0;
+}
diff --git a/src/include/gpxe/errfile.h b/src/include/gpxe/errfile.h
@@ -136,6 +136,7 @@
 #define ERRFILE_dhcppkt			( ERRFILE_NET | 0x00150000 )
 #define ERRFILE_slam			( ERRFILE_NET | 0x00160000 )
 #define ERRFILE_ib_sma			( ERRFILE_NET | 0x00170000 )
+#define ERRFILE_ib_packet		( ERRFILE_NET | 0x00180000 )
 
 #define ERRFILE_image		      ( ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_elf		      ( ERRFILE_IMAGE | 0x00010000 )

diff --git a/src/include/gpxe/ib_packet.h b/src/include/gpxe/ib_packet.h
@@ -7,6 +7,11 @@
  *
  */
 
+struct ib_device;
+struct ib_queue_pair;
+struct ib_address_vector;
+struct io_buffer;
+
 /** Half of an Infiniband Global Identifier */
 struct ib_gid_half {
 	uint8_t bytes[8];
@@ -53,6 +58,9 @@ enum ib_lnh {
 /** Default Infiniband LID */
 #define IB_LID_NONE 0xffff
 
+/** Test for multicast LID */
+#define IB_LID_MULTICAST( lid ) ( ( (lid) >= 0xc000 ) && ( (lid) <= 0xfffe ) )
+
 /** An Infiniband Global Route Header */
 struct ib_global_route_header {
 	/** IP version, traffic class, and flow label
@@ -76,7 +84,6 @@ struct ib_global_route_header {
 
 #define IB_GRH_IPVER_IPv6 0x06
 #define IB_GRH_NXTHDR_IBA 0x1b
-#define IB_GRH_HOPLMT_MAX 0xff
 
 /** An Infiniband Base Transport Header */
 struct ib_base_transport_header {
@@ -111,4 +118,30 @@ struct ib_datagram_extended_transport_header {
 	uint32_t src_qp;
 } __attribute__ (( packed ));
 
+/** All known IB header formats */
+union ib_headers {
+	struct ib_local_route_header lrh;
+	struct {
+		struct ib_local_route_header lrh;
+		struct ib_global_route_header grh;
+		struct ib_base_transport_header bth;
+		struct ib_datagram_extended_transport_header deth;
+	} __attribute__ (( packed )) lrh__grh__bth__deth;
+	struct {
+		struct ib_local_route_header lrh;
+		struct ib_base_transport_header bth;
+		struct ib_datagram_extended_transport_header deth;
+	} __attribute__ (( packed )) lrh__bth__deth;
+} __attribute__ (( packed ));
+
+/** Maximum size required for IB headers */
+#define IB_MAX_HEADER_SIZE sizeof ( union ib_headers )
+
+extern int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
+		     struct ib_queue_pair *qp, size_t payload_len,
+		     const struct ib_address_vector *av );
+extern int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
+		     struct ib_queue_pair **qp, size_t *payload_len,
+		     struct ib_address_vector *av );
+
 #endif /* _GPXE_IB_PACKET_H */