[infiniband] Create a general management agent

Generalise the subnet management agent into a general management agent capable of sending and responding to MADs, including support for retransmissions as necessary.
ipxe · Jul 17, 2009 · 1d8c85d · 1d8c85d
1 parent 365b8db
commit 1d8c85d
Show file tree

Hide file tree

Showing 6 changed files with 534 additions and 7 deletions.
diff --git a/src/include/gpxe/errfile.h b/src/include/gpxe/errfile.h
@@ -144,6 +144,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_ib_packet		( ERRFILE_NET | 0x00180000 )
 #define ERRFILE_icmp			( ERRFILE_NET | 0x00190000 )
 #define ERRFILE_ib_qset			( ERRFILE_NET | 0x001a0000 )
+#define ERRFILE_ib_gma			( ERRFILE_NET | 0x001b0000 )
 
 #define ERRFILE_image		      ( ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_elf		      ( ERRFILE_IMAGE | 0x00010000 )

diff --git a/src/include/gpxe/ib_gma.h b/src/include/gpxe/ib_gma.h
@@ -0,0 +1,71 @@
+#ifndef _GPXE_IB_GMA_H
+#define _GPXE_IB_GMA_H
+
+/** @file
+ *
+ * Infiniband General Management Agent
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include <gpxe/list.h>
+#include <gpxe/retry.h>
+#include <gpxe/tables.h>
+
+struct ib_device;
+struct ib_completion_queue;
+struct ib_queue_pair;
+union ib_mad;
+
+/** A MAD attribute handler */
+struct ib_mad_handler {
+	/** Management class */
+	uint8_t mgmt_class;
+	/** Class version */
+	uint8_t class_version;
+	/** Method */
+	uint8_t method;
+	/** Response method, or zero */
+	uint8_t resp_method;
+	/** Attribute (in network byte order) */
+	uint16_t attr_id;
+	/** Handle attribute
+	 *
+	 * @v ibdev	Infiniband device
+	 * @v mad	MAD
+	 * @ret rc	Return status code
+	 *
+	 * The handler should modify the MAD as applicable.  If the
+	 * handler returns with a non-zero value in the MAD's @c
+	 * method field, it will be sent as a response.
+	 */
+	int ( * handle ) ( struct ib_device *ibdev, union ib_mad *mad );
+};
+
+/** MAD attribute handlers */
+#define IB_MAD_HANDLERS __table ( struct ib_mad_handler, "ib_mad_handlers" )
+
+/** Declare a MAD attribute handler */
+#define __ib_mad_handler __table_entry ( IB_MAD_HANDLERS, 01 )
+
+/** An Infiniband General Management Agent */
+struct ib_gma {
+	/** Infiniband device */
+	struct ib_device *ibdev;
+	/** Completion queue */
+	struct ib_completion_queue *cq;
+	/** Queue pair */
+	struct ib_queue_pair *qp;
+
+	/** List of outstanding MAD requests */
+	struct list_head requests;
+};
+
+extern int ib_gma_request ( struct ib_gma *gma, union ib_mad *mad,
+			    struct ib_address_vector *av );
+extern int ib_create_gma ( struct ib_gma *gma, struct ib_device *ibdev,
+			   unsigned long qkey );
+extern void ib_destroy_gma ( struct ib_gma *gma );
+
+#endif /* _GPXE_IB_GMA_H */
diff --git a/src/include/gpxe/ib_mad.h b/src/include/gpxe/ib_mad.h
@@ -201,6 +201,8 @@ struct ib_smp_class_specific {
  *****************************************************************************
  */
 
+#define IB_SA_CLASS_VERSION			2
+
 struct ib_rmpp_hdr {
 	uint32_t raw[3];
 } __attribute__ (( packed ));

diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
@@ -14,6 +14,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <gpxe/device.h>
 #include <gpxe/ib_packet.h>
 #include <gpxe/ib_mad.h>
+#include <gpxe/ib_gma.h>
 
 /** Subnet management QPN */
 #define IB_QPN_SMA 0
@@ -133,6 +134,19 @@ struct ib_address_vector {
 	struct ib_gid gid;
 };
 
+/** Infiniband transmission rates */
+enum ib_rate {
+	IB_RATE_2_5 = 2,
+	IB_RATE_10 = 3,
+	IB_RATE_30 = 4,
+	IB_RATE_5 = 5,
+	IB_RATE_20 = 6,
+	IB_RATE_40 = 7,
+	IB_RATE_60 = 8,
+	IB_RATE_80 = 9,
+	IB_RATE_120 = 10,
+};
+
 /** Infiniband completion queue operations */
 struct ib_completion_queue_operations {
 	/**
@@ -354,6 +368,9 @@ struct ib_device {
 	/** Outbound packet sequence number */
 	uint32_t psn;
 
+	/** General management agent */
+	struct ib_gma gma;
+
 	/** Driver private data */
 	void *drv_priv;
 	/** Owner private data */

diff --git a/src/net/infiniband.c b/src/net/infiniband.c
@@ -485,16 +485,36 @@ void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
 int ib_open ( struct ib_device *ibdev ) {
 	int rc;
 
-	/* Open device if this is the first requested opening */
-	if ( ibdev->open_count == 0 ) {
-		if ( ( rc = ibdev->op->open ( ibdev ) ) != 0 )
-			return rc;
+	/* Increment device open request counter */
+	if ( ibdev->open_count++ > 0 ) {
+		/* Device was already open; do nothing */
+		return 0;
 	}
 
-	/* Increment device open request counter */
-	ibdev->open_count++;
+	/* Open device */
+	if ( ( rc = ibdev->op->open ( ibdev ) ) != 0 ) {
+		DBGC ( ibdev, "IBDEV %p could not open: %s\n",
+		       ibdev, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Create general management agent */
+	if ( ( rc = ib_create_gma ( &ibdev->gma, ibdev, IB_QKEY_GMA ) ) != 0 ){
+		DBGC ( ibdev, "IBDEV %p could not create GMA: %s\n",
+		       ibdev, strerror ( rc ) );
+		goto err_create_gma;
+	}
 
+	assert ( ibdev->open_count == 1 );
 	return 0;
+
+	ib_destroy_gma ( &ibdev->gma );
+ err_create_gma:
+	ibdev->op->close ( ibdev );
+ err_open:
+	assert ( ibdev->open_count == 1 );
+	ibdev->open_count = 0;
+	return rc;
 }
 
 /**
@@ -508,8 +528,10 @@ void ib_close ( struct ib_device *ibdev ) {
 	ibdev->open_count--;
 
 	/* Close device if this was the last remaining requested opening */
-	if ( ibdev->open_count == 0 )
+	if ( ibdev->open_count == 0 ) {
+		ib_destroy_gma ( &ibdev->gma );
 		ibdev->op->close ( ibdev );
+	}
 }
 
 /***************************************************************************