Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[infiniband] Update all other MAD users to use a management interface
  • Loading branch information
Michael Brown committed Aug 8, 2009
1 parent 44251eb commit 34bfc04
Show file tree
Hide file tree
Showing 12 changed files with 747 additions and 992 deletions.
32 changes: 28 additions & 4 deletions src/drivers/net/ipoib.c
Expand Up @@ -58,12 +58,14 @@ struct ipoib_device {
struct ib_queue_pair *qp;
/** Broadcast MAC */
struct ipoib_mac broadcast;
/** Joined to multicast group
/** Joined to IPv4 broadcast multicast group
*
* This flag indicates whether or not we have initiated the
* join to the IPv4 multicast group.
* join to the IPv4 broadcast multicast group.
*/
int broadcast_joined;
/** IPv4 broadcast multicast group membership */
struct ib_mc_membership broadcast_membership;
};

/** Broadcast IPoIB address */
Expand Down Expand Up @@ -456,6 +458,26 @@ static void ipoib_irq ( struct net_device *netdev __unused,
/* No implementation */
}

/**
* Handle IPv4 broadcast multicast group join completion
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v membership Multicast group membership
* @v rc Status code
* @v mad Response MAD (or NULL on error)
*/
void ipoib_join_complete ( struct ib_device *ibdev __unused,
struct ib_queue_pair *qp __unused,
struct ib_mc_membership *membership, int rc,
union ib_mad *mad __unused ) {
struct ipoib_device *ipoib = container_of ( membership,
struct ipoib_device, broadcast_membership );

/* Record join status as link status */
netdev_link_err ( ipoib->netdev, rc );
}

/**
* Join IPv4 broadcast multicast group
*
Expand All @@ -466,7 +488,9 @@ static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
int rc;

if ( ( rc = ib_mcast_join ( ipoib->ibdev, ipoib->qp,
&ipoib->broadcast.gid ) ) != 0 ) {
&ipoib->broadcast_membership,
&ipoib->broadcast.gid,
ipoib_join_complete ) ) != 0 ) {
DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n",
ipoib, strerror ( rc ) );
return rc;
Expand All @@ -485,7 +509,7 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {

if ( ipoib->broadcast_joined ) {
ib_mcast_leave ( ipoib->ibdev, ipoib->qp,
&ipoib->broadcast.gid );
&ipoib->broadcast_membership );
ipoib->broadcast_joined = 0;
}
}
Expand Down
60 changes: 54 additions & 6 deletions src/include/gpxe/ib_cm.h
Expand Up @@ -10,12 +10,60 @@
FILE_LICENCE ( GPL2_OR_LATER );

#include <gpxe/infiniband.h>
#include <gpxe/retry.h>

extern int ib_cm_connect ( struct ib_queue_pair *qp, struct ib_gid *dgid,
struct ib_gid_half *service_id,
void *private_data, size_t private_data_len,
void ( * notify ) ( struct ib_queue_pair *qp,
int rc, void *private_data,
size_t private_data_len ) );
struct ib_mad_transaction;
struct ib_connection;

/** Infiniband connection operations */
struct ib_connection_operations {
/** Handle change of connection status
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v conn Connection
* @v rc Connection status code
* @v private_data Private data, if available
* @v private_data_len Length of private data
*/
void ( * changed ) ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_connection *conn, int rc,
void *private_data, size_t private_data_len );
};

/** An Infiniband connection */
struct ib_connection {
/** Infiniband device */
struct ib_device *ibdev;
/** Queue pair */
struct ib_queue_pair *qp;
/** Local communication ID */
uint32_t local_id;
/** Remote communication ID */
uint32_t remote_id;
/** Target service ID */
struct ib_gid_half service_id;
/** Connection operations */
struct ib_connection_operations *op;

/** Path to target */
struct ib_path *path;
/** Connection request management transaction */
struct ib_mad_transaction *madx;

/** Length of connection request private data */
size_t private_data_len;
/** Connection request private data */
uint8_t private_data[0];
};

extern struct ib_connection *
ib_create_conn ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *dgid, struct ib_gid_half *service_id,
void *req_private_data, size_t req_private_data_len,
struct ib_connection_operations *op );
extern void ib_destroy_conn ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_connection *conn );

#endif /* _GPXE_IB_CM_H */
65 changes: 0 additions & 65 deletions src/include/gpxe/ib_gma.h

This file was deleted.

11 changes: 11 additions & 0 deletions src/include/gpxe/ib_mad.h
Expand Up @@ -303,6 +303,16 @@ union ib_sa_data {
#define IB_CM_ATTR_LOAD_ALTERNATE_PATH 0x0019
#define IB_CM_ATTR_ALTERNATE_PATH_RESPONSE 0x001a

/** Communication management common fields */
struct ib_cm_common {
/** Local communication ID */
uint32_t local_id;
/** Remote communication ID */
uint32_t remote_id;
/** Reserved */
uint8_t reserved[224];
} __attribute__ (( packed ));

/** A communication management path */
struct ib_cm_path {
/** Local port LID */
Expand Down Expand Up @@ -438,6 +448,7 @@ struct ib_cm_ready_to_use {

/** A communication management attribute */
union ib_cm_data {
struct ib_cm_common common;
struct ib_cm_connect_request connect_request;
struct ib_cm_connect_reject connect_reject;
struct ib_cm_connect_reply connect_reply;
Expand Down
33 changes: 31 additions & 2 deletions src/include/gpxe/ib_mcast.h
Expand Up @@ -11,9 +11,38 @@ FILE_LICENCE ( GPL2_OR_LATER );

#include <gpxe/infiniband.h>

struct ib_mad_transaction;

/** An Infiniband multicast group membership */
struct ib_mc_membership {
/** Queue pair */
struct ib_queue_pair *qp;
/** Multicast GID */
struct ib_gid gid;
/** Multicast group join transaction */
struct ib_mad_transaction *madx;
/** Handle join success/failure
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v membership Multicast group membership
* @v rc Status code
* @v mad Response MAD (or NULL on error)
*/
void ( * complete ) ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_mc_membership *membership, int rc,
union ib_mad *mad );
};

extern int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *gid );
struct ib_mc_membership *membership,
struct ib_gid *gid,
void ( * joined ) ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_mc_membership *memb,
int rc, union ib_mad *mad ) );

extern void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *gid );
struct ib_mc_membership *membership );

#endif /* _GPXE_IB_MCAST_H */
59 changes: 59 additions & 0 deletions src/include/gpxe/ib_pathrec.h
Expand Up @@ -11,6 +11,65 @@ FILE_LICENCE ( GPL2_OR_LATER );

#include <gpxe/infiniband.h>

struct ib_mad_transaction;
struct ib_path;

/** Infiniband path operations */
struct ib_path_operations {
/** Handle path transaction completion
*
* @v ibdev Infiniband device
* @v path Path
* @v rc Status code
* @v av Address vector, or NULL on error
*/
void ( * complete ) ( struct ib_device *ibdev,
struct ib_path *path, int rc,
struct ib_address_vector *av );
};

/** An Infiniband path */
struct ib_path {
/** Infiniband device */
struct ib_device *ibdev;
/** Address vector */
struct ib_address_vector av;
/** Management transaction */
struct ib_mad_transaction *madx;
/** Path operations */
struct ib_path_operations *op;
/** Owner private data */
void *owner_priv;
};

/**
* Set Infiniband path owner-private data
*
* @v path Path
* @v priv Private data
*/
static inline __always_inline void
ib_path_set_ownerdata ( struct ib_path *path, void *priv ) {
path->owner_priv = priv;
}

/**
* Get Infiniband path owner-private data
*
* @v path Path
* @ret priv Private data
*/
static inline __always_inline void *
ib_path_get_ownerdata ( struct ib_path *path ) {
return path->owner_priv;
}

extern struct ib_path *
ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av,
struct ib_path_operations *op );
extern void ib_destroy_path ( struct ib_device *ibdev,
struct ib_path *path );

extern int ib_resolve_path ( struct ib_device *ibdev,
struct ib_address_vector *av );

Expand Down
5 changes: 2 additions & 3 deletions src/include/gpxe/infiniband.h
Expand Up @@ -46,7 +46,6 @@ struct ib_queue_pair;
struct ib_address_vector;
struct ib_completion_queue;
struct ib_mad_interface;
struct ib_gma;

/** Infiniband transmission rates */
enum ib_rate {
Expand Down Expand Up @@ -416,8 +415,8 @@ struct ib_device {

/** Subnet management interface */
struct ib_mad_interface *smi;
/** General management agent */
struct ib_gma *gma;
/** General services interface */
struct ib_mad_interface *gsi;

/** Driver private data */
void *drv_priv;
Expand Down
17 changes: 8 additions & 9 deletions src/net/infiniband.c
Expand Up @@ -35,7 +35,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <gpxe/infiniband.h>
#include <gpxe/ib_mi.h>
#include <gpxe/ib_sma.h>
#include <gpxe/ib_gma.h>

/** @file
*
Expand Down Expand Up @@ -551,12 +550,12 @@ int ib_open ( struct ib_device *ibdev ) {
goto err_create_sma;
}

/* Create general management agent */
ibdev->gma = ib_create_gma ( ibdev, IB_QPT_GSI );
if ( ! ibdev->gma ) {
DBGC ( ibdev, "IBDEV %p could not create GMA\n", ibdev );
/* Create general services interface */
ibdev->gsi = ib_create_mi ( ibdev, IB_QPT_GSI );
if ( ! ibdev->gsi ) {
DBGC ( ibdev, "IBDEV %p could not create GSI\n", ibdev );
rc = -ENOMEM;
goto err_create_gma;
goto err_create_gsi;
}

/* Open device */
Expand All @@ -571,8 +570,8 @@ int ib_open ( struct ib_device *ibdev ) {

ibdev->op->close ( ibdev );
err_open:
ib_destroy_gma ( ibdev->gma );
err_create_gma:
ib_destroy_mi ( ibdev, ibdev->gsi );
err_create_gsi:
ib_destroy_sma ( ibdev, ibdev->smi );
err_create_sma:
ib_destroy_mi ( ibdev, ibdev->smi );
Expand All @@ -594,7 +593,7 @@ void ib_close ( struct ib_device *ibdev ) {

/* Close device if this was the last remaining requested opening */
if ( ibdev->open_count == 0 ) {
ib_destroy_gma ( ibdev->gma );
ib_destroy_mi ( ibdev, ibdev->gsi );
ib_destroy_sma ( ibdev, ibdev->smi );
ib_destroy_mi ( ibdev, ibdev->smi );
ibdev->op->close ( ibdev );
Expand Down

0 comments on commit 34bfc04

Please sign in to comment.