Skip to content

Commit

Permalink
[infiniband] Split queue set functionality out of ipoib.c to ib_qset.c
Browse files Browse the repository at this point in the history
  • Loading branch information
Michael Brown committed Jul 17, 2009
1 parent 8868956 commit b25a4b6
Show file tree
Hide file tree
Showing 4 changed files with 204 additions and 137 deletions.
169 changes: 32 additions & 137 deletions src/drivers/net/ipoib.c
View file
Expand Up @@ -28,6 +28,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <gpxe/iobuf.h>
#include <gpxe/netdevice.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_qset.h>
#include <gpxe/ipoib.h>

/** @file
Expand All @@ -53,26 +54,16 @@ FILE_LICENCE ( GPL2_OR_LATER );
/** Number of IPoIB metadata completion entries */
#define IPOIB_META_NUM_CQES 8

/** An IPoIB queue set */
struct ipoib_queue_set {
/** Completion queue */
struct ib_completion_queue *cq;
/** Queue pair */
struct ib_queue_pair *qp;
/** Receive work queue maximum fill level */
unsigned int recv_max_fill;
};

/** An IPoIB device */
struct ipoib_device {
/** Network device */
struct net_device *netdev;
/** Underlying Infiniband device */
struct ib_device *ibdev;
/** Data queue set */
struct ipoib_queue_set data;
struct ib_queue_set data;
/** Data queue set */
struct ipoib_queue_set meta;
struct ib_queue_set meta;
/** Broadcast GID */
struct ib_gid broadcast_gid;
/** Broadcast LID */
Expand Down Expand Up @@ -361,79 +352,6 @@ struct ll_protocol ipoib_protocol __ll_protocol = {
****************************************************************************
*/

/**
* Destroy queue set
*
* @v ipoib IPoIB device
* @v qset Queue set
*/
static void ipoib_destroy_qset ( struct ipoib_device *ipoib,
struct ipoib_queue_set *qset ) {
struct ib_device *ibdev = ipoib->ibdev;

if ( qset->qp )
ib_destroy_qp ( ibdev, qset->qp );
if ( qset->cq )
ib_destroy_cq ( ibdev, qset->cq );
memset ( qset, 0, sizeof ( *qset ) );
}

/**
* Create queue set
*
* @v ipoib IPoIB device
* @v qset Queue set
* @v num_cqes Number of completion queue entries
* @v cq_op Completion queue operations
* @v num_send_wqes Number of send work queue entries
* @v num_recv_wqes Number of receive work queue entries
* @v qkey Queue key
* @ret rc Return status code
*/
static int ipoib_create_qset ( struct ipoib_device *ipoib,
struct ipoib_queue_set *qset,
unsigned int num_cqes,
struct ib_completion_queue_operations *cq_op,
unsigned int num_send_wqes,
unsigned int num_recv_wqes,
unsigned long qkey ) {
struct ib_device *ibdev = ipoib->ibdev;
int rc;

/* Sanity check */
assert ( qset->cq == NULL );
assert ( qset->qp == NULL );

/* Store queue parameters */
qset->recv_max_fill = num_recv_wqes;

/* Allocate completion queue */
qset->cq = ib_create_cq ( ibdev, num_cqes, cq_op );
if ( ! qset->cq ) {
DBGC ( ipoib, "IPoIB %p could not allocate completion queue\n",
ipoib );
rc = -ENOMEM;
goto err;
}

/* Allocate queue pair */
qset->qp = ib_create_qp ( ibdev, num_send_wqes, qset->cq,
num_recv_wqes, qset->cq, qkey );
if ( ! qset->qp ) {
DBGC ( ipoib, "IPoIB %p could not allocate queue pair\n",
ipoib );
rc = -ENOMEM;
goto err;
}
ib_qp_set_ownerdata ( qset->qp, ipoib->netdev );

return 0;

err:
ipoib_destroy_qset ( ipoib, qset );
return rc;
}

/**
* Transmit path record request
*
Expand Down Expand Up @@ -625,9 +543,9 @@ static int ipoib_transmit ( struct net_device *netdev,
static void ipoib_data_complete_send ( struct ib_device *ibdev __unused,
struct ib_queue_pair *qp,
struct io_buffer *iobuf, int rc ) {
struct net_device *netdev = ib_qp_get_ownerdata ( qp );
struct ipoib_device *ipoib = ib_qp_get_ownerdata ( qp );

netdev_tx_complete_err ( netdev, iobuf, rc );
netdev_tx_complete_err ( ipoib->netdev, iobuf, rc );
}

/**
Expand All @@ -643,8 +561,8 @@ static void ipoib_data_complete_recv ( struct ib_device *ibdev __unused,
struct ib_queue_pair *qp,
struct ib_address_vector *av,
struct io_buffer *iobuf, int rc ) {
struct net_device *netdev = ib_qp_get_ownerdata ( qp );
struct ipoib_device *ipoib = netdev->priv;
struct ipoib_device *ipoib = ib_qp_get_ownerdata ( qp );
struct net_device *netdev = ipoib->netdev;
struct ipoib_hdr *ipoib_hdr;
struct ipoib_peer *src;

Expand Down Expand Up @@ -690,8 +608,7 @@ static struct ib_completion_queue_operations ipoib_data_cq_op = {
static void ipoib_meta_complete_send ( struct ib_device *ibdev __unused,
struct ib_queue_pair *qp,
struct io_buffer *iobuf, int rc ) {
struct net_device *netdev = ib_qp_get_ownerdata ( qp );
struct ipoib_device *ipoib = netdev->priv;
struct ipoib_device *ipoib = ib_qp_get_ownerdata ( qp );

if ( rc != 0 ) {
DBGC ( ipoib, "IPoIB %p metadata TX completion error: %s\n",
Expand Down Expand Up @@ -769,8 +686,7 @@ ipoib_meta_complete_recv ( struct ib_device *ibdev __unused,
struct ib_queue_pair *qp,
struct ib_address_vector *av __unused,
struct io_buffer *iobuf, int rc ) {
struct net_device *netdev = ib_qp_get_ownerdata ( qp );
struct ipoib_device *ipoib = netdev->priv;
struct ipoib_device *ipoib = ib_qp_get_ownerdata ( qp );
struct ib_mad_sa *sa;

if ( rc != 0 ) {
Expand Down Expand Up @@ -818,28 +734,6 @@ static struct ib_completion_queue_operations ipoib_meta_cq_op = {
.complete_recv = ipoib_meta_complete_recv,
};

/**
* Refill IPoIB receive ring
*
* @v ipoib IPoIB device
*/
static void ipoib_refill_recv ( struct ipoib_device *ipoib,
struct ipoib_queue_set *qset ) {
struct ib_device *ibdev = ipoib->ibdev;
struct io_buffer *iobuf;
int rc;

while ( qset->qp->recv.fill < qset->recv_max_fill ) {
iobuf = alloc_iob ( IPOIB_PKT_LEN );
if ( ! iobuf )
break;
if ( ( rc = ib_post_recv ( ibdev, qset->qp, iobuf ) ) != 0 ) {
free_iob ( iobuf );
break;
}
}
}

/**
* Poll IPoIB network device
*
Expand All @@ -851,8 +745,8 @@ static void ipoib_poll ( struct net_device *netdev ) {

ib_poll_cq ( ibdev, ipoib->meta.cq );
ib_poll_cq ( ibdev, ipoib->data.cq );
ipoib_refill_recv ( ipoib, &ipoib->meta );
ipoib_refill_recv ( ipoib, &ipoib->data );
ib_qset_refill_recv ( ibdev, &ipoib->meta );
ib_qset_refill_recv ( ibdev, &ipoib->data );
}

/**
Expand Down Expand Up @@ -928,46 +822,47 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
*/
static int ipoib_open ( struct net_device *netdev ) {
struct ipoib_device *ipoib = netdev->priv;
struct ib_device *ibdev = ipoib->ibdev;
struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
int rc;

/* Open IB device */
if ( ( rc = ib_open ( ipoib->ibdev ) ) != 0 ) {
if ( ( rc = ib_open ( ibdev ) ) != 0 ) {
DBGC ( ipoib, "IPoIB %p could not open device: %s\n",
ipoib, strerror ( rc ) );
goto err_ib_open;
}

/* Allocate metadata queue set */
if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta,
IPOIB_META_NUM_CQES,
&ipoib_meta_cq_op,
IPOIB_META_NUM_SEND_WQES,
IPOIB_META_NUM_RECV_WQES,
IB_GLOBAL_QKEY ) ) != 0 ) {
if ( ( rc = ib_create_qset ( ibdev, &ipoib->meta,
IPOIB_META_NUM_CQES, &ipoib_meta_cq_op,
IPOIB_META_NUM_SEND_WQES,
IPOIB_META_NUM_RECV_WQES,
IPOIB_PKT_LEN, IB_GLOBAL_QKEY ) ) != 0 ) {
DBGC ( ipoib, "IPoIB %p could not allocate metadata QP: %s\n",
ipoib, strerror ( rc ) );
goto err_create_meta_qset;
}
ib_qp_set_ownerdata ( ipoib->meta.qp, ipoib );

/* Allocate data queue set */
if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->data,
IPOIB_DATA_NUM_CQES,
&ipoib_data_cq_op,
IPOIB_DATA_NUM_SEND_WQES,
IPOIB_DATA_NUM_RECV_WQES,
IB_GLOBAL_QKEY ) ) != 0 ) {
if ( ( rc = ib_create_qset ( ibdev, &ipoib->data,
IPOIB_DATA_NUM_CQES, &ipoib_data_cq_op,
IPOIB_DATA_NUM_SEND_WQES,
IPOIB_DATA_NUM_RECV_WQES,
IPOIB_PKT_LEN, IB_GLOBAL_QKEY ) ) != 0 ) {
DBGC ( ipoib, "IPoIB %p could not allocate data QP: %s\n",
ipoib, strerror ( rc ) );
goto err_create_data_qset;
}
ib_qp_set_ownerdata ( ipoib->data.qp, ipoib );

/* Update MAC address with data QPN */
mac->qpn = htonl ( ipoib->data.qp->qpn );

/* Fill receive rings */
ipoib_refill_recv ( ipoib, &ipoib->meta );
ipoib_refill_recv ( ipoib, &ipoib->data );
ib_qset_refill_recv ( ibdev, &ipoib->meta );
ib_qset_refill_recv ( ibdev, &ipoib->data );

/* Join broadcast group */
if ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) {
Expand All @@ -979,11 +874,11 @@ static int ipoib_open ( struct net_device *netdev ) {
return 0;

err_join_broadcast:
ipoib_destroy_qset ( ipoib, &ipoib->data );
ib_destroy_qset ( ibdev, &ipoib->data );
err_create_data_qset:
ipoib_destroy_qset ( ipoib, &ipoib->meta );
ib_destroy_qset ( ibdev, &ipoib->meta );
err_create_meta_qset:
ib_close ( ipoib->ibdev );
ib_close ( ibdev );
err_ib_open:
return rc;
}
Expand All @@ -1004,8 +899,8 @@ static void ipoib_close ( struct net_device *netdev ) {
mac->qpn = 0;

/* Tear down the queues */
ipoib_destroy_qset ( ipoib, &ipoib->data );
ipoib_destroy_qset ( ipoib, &ipoib->meta );
ib_destroy_qset ( ipoib->ibdev, &ipoib->data );
ib_destroy_qset ( ipoib->ibdev, &ipoib->meta );

/* Close IB device */
ib_close ( ipoib->ibdev );
Expand Down
1 change: 1 addition & 0 deletions src/include/gpxe/errfile.h
View file
Expand Up @@ -143,6 +143,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
#define ERRFILE_ib_sma ( ERRFILE_NET | 0x00170000 )
#define ERRFILE_ib_packet ( ERRFILE_NET | 0x00180000 )
#define ERRFILE_icmp ( ERRFILE_NET | 0x00190000 )
#define ERRFILE_ib_qset ( ERRFILE_NET | 0x001a0000 )

#define ERRFILE_image ( ERRFILE_IMAGE | 0x00000000 )
#define ERRFILE_elf ( ERRFILE_IMAGE | 0x00010000 )
Expand Down
38 changes: 38 additions & 0 deletions src/include/gpxe/ib_qset.h
View file
@@ -0,0 +1,38 @@
#ifndef _GPXE_IB_QSET_H
#define _GPXE_IB_QSET_H

/** @file
*
* Infiniband queue sets
*
*/

FILE_LICENCE ( GPL2_OR_LATER );

#include <stdint.h>
#include <gpxe/infiniband.h>

/** An Infiniband queue set */
struct ib_queue_set {
/** Completion queue */
struct ib_completion_queue *cq;
/** Queue pair */
struct ib_queue_pair *qp;
/** Receive work queue maximum fill level */
unsigned int recv_max_fill;
/** Receive packet length */
size_t recv_pkt_len;
};

extern int ib_create_qset ( struct ib_device *ibdev,
struct ib_queue_set *qset, unsigned int num_cqes,
struct ib_completion_queue_operations *cq_op,
unsigned int num_send_wqes,
unsigned int num_recv_wqes, size_t recv_pkt_len,
unsigned long qkey );
extern void ib_qset_refill_recv ( struct ib_device *ibdev,
struct ib_queue_set *qset );
extern void ib_destroy_qset ( struct ib_device *ibdev,
struct ib_queue_set *qset );

#endif /* _GPXE_IB_QSET_H */

0 comments on commit b25a4b6

Please sign in to comment.