Skip to content

Commit

Permalink
[dma] Move I/O buffer DMA operations to iobuf.h
Browse files Browse the repository at this point in the history 
Include a potential DMA mapping within the definition of an I/O
buffer, and move all I/O buffer DMA mapping functions from dma.h to
iobuf.h.  This avoids the need for drivers to maintain a separate list
of DMA mappings for each I/O buffer that they may handle.

Network device drivers typically do not keep track of transmit I/O
buffers, since the network device core already maintains a transmit
queue.  Drivers will typically call netdev_tx_complete_next() to
complete a transmission without first obtaining the relevant I/O
buffer pointer (and will rely on the network device core automatically
cancelling any pending transmissions when the device is closed).

To allow this driver design approach to be retained, update the
netdev_tx_complete() family of functions to automatically perform the
DMA unmapping operation if required.  For symmetry, also update the
netdev_rx() family of functions to behave the same way.

As a further convenience for drivers, allow the network device core to
automatically perform DMA mapping on the transmit datapath before
calling the driver's transmit() method.  This avoids the need to
introduce a mapping error handling code path into the typically
error-free transmit methods.

With these changes, the modifications required to update a typical
network device driver to use the new DMA API are fairly minimal:

- Allocate and free descriptor rings and similar coherent structures
  using dma_alloc()/dma_free() rather than malloc_phys()/free_phys()

- Allocate and free receive buffers using alloc_rx_iob()/free_rx_iob()
  rather than alloc_iob()/free_iob()

- Calculate DMA addresses using dma() or iob_dma() rather than
  virt_to_bus()

- Set a 64-bit DMA mask if needed using dma_set_mask_64bit() and
  thereafter eliminate checks on DMA address ranges

- Either record the DMA device in netdev->dma, or call iob_map_tx() as
  part of the transmit() method

- Ensure that debug messages use virt_to_phys() when displaying
  "hardware" addresses

Signed-off-by: Michael Brown <mcb30@ipxe.org>
  • Loading branch information
@mcb30
mcb30 committed Nov 28, 2020
1 parent 70e6e83 commit 8d337ec
Show file tree
Hide file tree
Showing 16 changed files with 342 additions and 391 deletions.
41 changes: 0 additions & 41 deletions src/core/dma.c
View file
Expand Up @@ -25,7 +25,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );

#include <assert.h>
#include <errno.h>
#include <ipxe/iobuf.h>
#include <ipxe/dma.h>

/** @file
Expand Down Expand Up @@ -139,43 +138,3 @@ PROVIDE_DMAAPI ( op, dma_alloc, dma_op_alloc );
PROVIDE_DMAAPI ( op, dma_free, dma_op_free );
PROVIDE_DMAAPI ( op, dma_set_mask, dma_op_set_mask );
PROVIDE_DMAAPI_INLINE ( op, dma_phys );

/******************************************************************************
*
* Utility functions
*
******************************************************************************
*/

/**
* Allocate and map I/O buffer for receiving data from device
*
* @v dma DMA device
* @v map DMA mapping to fill in
* @v len Length of I/O buffer
* @ret iobuf I/O buffer, or NULL on error
*/
struct io_buffer * dma_alloc_rx_iob ( struct dma_device *dma,
struct dma_mapping *map,
size_t len ) {
struct io_buffer *iobuf;
int rc;

/* Allocate I/O buffer */
iobuf = alloc_iob ( len );
if ( ! iobuf )
goto err_alloc;

/* Map I/O buffer */
if ( ( rc = dma_map ( dma, map, virt_to_phys ( iobuf->data ),
len, DMA_RX ) ) != 0 )
goto err_map;

return iobuf;

dma_unmap ( map );
err_map:
free_iob ( iobuf );
err_alloc:
return NULL;
}
45 changes: 45 additions & 0 deletions src/core/iobuf.c
View file
Expand Up @@ -110,6 +110,7 @@ struct io_buffer * alloc_iob_raw ( size_t len, size_t align, size_t offset ) {
}

/* Populate descriptor */
memset ( &iobuf->map, 0, sizeof ( iobuf->map ) );
iobuf->head = iobuf->data = iobuf->tail = data;
iobuf->end = ( data + len );

Expand Down Expand Up @@ -153,6 +154,7 @@ void free_iob ( struct io_buffer *iobuf ) {
assert ( iobuf->head <= iobuf->data );
assert ( iobuf->data <= iobuf->tail );
assert ( iobuf->tail <= iobuf->end );
assert ( ! dma_mapped ( &iobuf->map ) );

/* Free buffer */
len = ( iobuf->end - iobuf->head );
Expand All @@ -169,6 +171,49 @@ void free_iob ( struct io_buffer *iobuf ) {
}
}

/**
* Allocate and map I/O buffer for receive DMA
*
* @v len Length of I/O buffer
* @v dma DMA device
* @ret iobuf I/O buffer, or NULL on error
*/
struct io_buffer * alloc_rx_iob ( size_t len, struct dma_device *dma ) {
struct io_buffer *iobuf;
int rc;

/* Allocate I/O buffer */
iobuf = alloc_iob ( len );
if ( ! iobuf )
goto err_alloc;

/* Map I/O buffer */
if ( ( rc = iob_map_rx ( iobuf, dma ) ) != 0 )
goto err_map;

return iobuf;

iob_unmap ( iobuf );
err_map:
free_iob ( iobuf );
err_alloc:
return NULL;
}

/**
* Unmap and free I/O buffer for receive DMA
*
* @v iobuf I/O buffer
*/
void free_rx_iob ( struct io_buffer *iobuf ) {

/* Unmap I/O buffer */
iob_unmap ( iobuf );

/* Free I/O buffer */
free_iob ( iobuf );
}

/**
* Ensure I/O buffer has sufficient headroom
*
Expand Down
120 changes: 44 additions & 76 deletions src/drivers/net/intel.c
View file
Expand Up @@ -568,34 +568,30 @@ void intel_destroy_ring ( struct intel_nic *intel, struct intel_ring *ring ) {
void intel_refill_rx ( struct intel_nic *intel ) {
struct intel_descriptor *rx;
struct io_buffer *iobuf;
struct dma_mapping *map;
unsigned int rx_idx;
unsigned int rx_tail;
unsigned int refilled = 0;

/* Refill ring */
while ( ( intel->rx.ring.prod -
intel->rx.ring.cons ) < INTEL_RX_FILL ) {

/* Get next receive descriptor */
rx_idx = ( intel->rx.ring.prod % INTEL_NUM_RX_DESC );
rx = &intel->rx.ring.desc[rx_idx];
map = &intel->rx.map[rx_idx];
assert ( intel->rx.iobuf[rx_idx] == NULL );
while ( ( intel->rx.prod - intel->rx.cons ) < INTEL_RX_FILL ) {

/* Allocate I/O buffer */
iobuf = dma_alloc_rx_iob ( intel->dma, map, INTEL_RX_MAX_LEN );
iobuf = alloc_rx_iob ( INTEL_RX_MAX_LEN, intel->dma );
if ( ! iobuf ) {
/* Wait for next refill */
break;
}
intel->rx.iobuf[rx_idx] = iobuf;

/* Update producer index */
intel->rx.ring.prod++;
/* Get next receive descriptor */
rx_idx = ( intel->rx.prod++ % INTEL_NUM_RX_DESC );
rx = &intel->rx.desc[rx_idx];

/* Populate receive descriptor */
intel->rx.ring.describe ( rx, dma ( map, iobuf->data ), 0 );
intel->rx.describe ( rx, iob_dma ( iobuf ), 0 );

/* Record I/O buffer */
assert ( intel->rx_iobuf[rx_idx] == NULL );
intel->rx_iobuf[rx_idx] = iobuf;

DBGC2 ( intel, "INTEL %p RX %d is [%lx,%lx)\n",
intel, rx_idx, virt_to_phys ( iobuf->data ),
Expand All @@ -606,40 +602,27 @@ void intel_refill_rx ( struct intel_nic *intel ) {
/* Push descriptors to card, if applicable */
if ( refilled ) {
wmb();
rx_tail = ( intel->rx.ring.prod % INTEL_NUM_RX_DESC );
rx_tail = ( intel->rx.prod % INTEL_NUM_RX_DESC );
profile_start ( &intel_vm_refill_profiler );
writel ( rx_tail,
intel->regs + intel->rx.ring.reg + INTEL_xDT );
writel ( rx_tail, intel->regs + intel->rx.reg + INTEL_xDT );
profile_stop ( &intel_vm_refill_profiler );
profile_exclude ( &intel_vm_refill_profiler );
}
}

/**
* Flush unused I/O buffers
* Discard unused receive I/O buffers
*
* @v intel Intel device
*
* Discard any unused receive I/O buffers and unmap any incomplete
* transmit I/O buffers.
*/
void intel_flush ( struct intel_nic *intel ) {
void intel_empty_rx ( struct intel_nic *intel ) {
unsigned int i;
unsigned int tx_idx;

/* Discard unused receive buffers */
for ( i = 0 ; i < INTEL_NUM_RX_DESC ; i++ ) {
if ( intel->rx.iobuf[i] ) {
dma_unmap ( &intel->rx.map[i] );
free_iob ( intel->rx.iobuf[i] );
}
intel->rx.iobuf[i] = NULL;
}

/* Unmap incomplete transmit buffers */
for ( i = intel->tx.ring.cons ; i != intel->tx.ring.prod ; i++ ) {
tx_idx = ( i % INTEL_NUM_TX_DESC );
dma_unmap ( &intel->tx.map[tx_idx] );
if ( intel->rx_iobuf[i] )
free_rx_iob ( intel->rx_iobuf[i] );
intel->rx_iobuf[i] = NULL;
}
}

Expand Down Expand Up @@ -670,11 +653,11 @@ static int intel_open ( struct net_device *netdev ) {
}

/* Create transmit descriptor ring */
if ( ( rc = intel_create_ring ( intel, &intel->tx.ring ) ) != 0 )
if ( ( rc = intel_create_ring ( intel, &intel->tx ) ) != 0 )
goto err_create_tx;

/* Create receive descriptor ring */
if ( ( rc = intel_create_ring ( intel, &intel->rx.ring ) ) != 0 )
if ( ( rc = intel_create_ring ( intel, &intel->rx ) ) != 0 )
goto err_create_rx;

/* Program MAC address */
Expand Down Expand Up @@ -713,9 +696,9 @@ static int intel_open ( struct net_device *netdev ) {

return 0;

intel_destroy_ring ( intel, &intel->rx.ring );
intel_destroy_ring ( intel, &intel->rx );
err_create_rx:
intel_destroy_ring ( intel, &intel->tx.ring );
intel_destroy_ring ( intel, &intel->tx );
err_create_tx:
return rc;
}
Expand All @@ -735,13 +718,13 @@ static void intel_close ( struct net_device *netdev ) {
writel ( 0, intel->regs + INTEL_TCTL );

/* Destroy receive descriptor ring */
intel_destroy_ring ( intel, &intel->rx.ring );
intel_destroy_ring ( intel, &intel->rx );

/* Flush unused buffers */
intel_flush ( intel );
/* Discard any unused receive buffers */
intel_empty_rx ( intel );

/* Destroy transmit descriptor ring */
intel_destroy_ring ( intel, &intel->tx.ring );
intel_destroy_ring ( intel, &intel->tx );

/* Reset the NIC, to flush the transmit and receive FIFOs */
intel_reset ( intel );
Expand All @@ -757,37 +740,27 @@ static void intel_close ( struct net_device *netdev ) {
int intel_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
struct intel_nic *intel = netdev->priv;
struct intel_descriptor *tx;
struct dma_mapping *map;
unsigned int tx_idx;
unsigned int tx_tail;
size_t len;
int rc;

/* Get next transmit descriptor */
if ( ( intel->tx.ring.prod - intel->tx.ring.cons ) >= INTEL_TX_FILL ) {
if ( ( intel->tx.prod - intel->tx.cons ) >= INTEL_TX_FILL ) {
DBGC ( intel, "INTEL %p out of transmit descriptors\n", intel );
return -ENOBUFS;
}
tx_idx = ( intel->tx.ring.prod % INTEL_NUM_TX_DESC );
tx = &intel->tx.ring.desc[tx_idx];
map = &intel->tx.map[tx_idx];

/* Map I/O buffer */
if ( ( rc = dma_map_tx_iob ( intel->dma, map, iobuf ) ) != 0 )
return rc;

/* Update producer index */
intel->tx.ring.prod++;
tx_idx = ( intel->tx.prod++ % INTEL_NUM_TX_DESC );
tx_tail = ( intel->tx.prod % INTEL_NUM_TX_DESC );
tx = &intel->tx.desc[tx_idx];

/* Populate transmit descriptor */
len = iob_len ( iobuf );
intel->tx.ring.describe ( tx, dma ( map, iobuf->data ), len );
intel->tx.describe ( tx, iob_dma ( iobuf ), len );
wmb();

/* Notify card that there are packets ready to transmit */
profile_start ( &intel_vm_tx_profiler );
tx_tail = ( intel->tx.ring.prod % INTEL_NUM_TX_DESC );
writel ( tx_tail, intel->regs + intel->tx.ring.reg + INTEL_xDT );
writel ( tx_tail, intel->regs + intel->tx.reg + INTEL_xDT );
profile_stop ( &intel_vm_tx_profiler );
profile_exclude ( &intel_vm_tx_profiler );

Expand All @@ -809,24 +782,21 @@ void intel_poll_tx ( struct net_device *netdev ) {
unsigned int tx_idx;

/* Check for completed packets */
while ( intel->tx.ring.cons != intel->tx.ring.prod ) {
while ( intel->tx.cons != intel->tx.prod ) {

/* Get next transmit descriptor */
tx_idx = ( intel->tx.ring.cons % INTEL_NUM_TX_DESC );
tx = &intel->tx.ring.desc[tx_idx];
tx_idx = ( intel->tx.cons % INTEL_NUM_TX_DESC );
tx = &intel->tx.desc[tx_idx];

/* Stop if descriptor is still in use */
if ( ! ( tx->status & cpu_to_le32 ( INTEL_DESC_STATUS_DD ) ) )
return;

DBGC2 ( intel, "INTEL %p TX %d complete\n", intel, tx_idx );

/* Unmap I/O buffer */
dma_unmap ( &intel->tx.map[tx_idx] );

/* Complete TX descriptor */
netdev_tx_complete_next ( netdev );
intel->tx.ring.cons++;
intel->tx.cons++;
}
}

Expand All @@ -843,22 +813,19 @@ void intel_poll_rx ( struct net_device *netdev ) {
size_t len;

/* Check for received packets */
while ( intel->rx.ring.cons != intel->rx.ring.prod ) {
while ( intel->rx.cons != intel->rx.prod ) {

/* Get next receive descriptor */
rx_idx = ( intel->rx.ring.cons % INTEL_NUM_RX_DESC );
rx = &intel->rx.ring.desc[rx_idx];
rx_idx = ( intel->rx.cons % INTEL_NUM_RX_DESC );
rx = &intel->rx.desc[rx_idx];

/* Stop if descriptor is still in use */
if ( ! ( rx->status & cpu_to_le32 ( INTEL_DESC_STATUS_DD ) ) )
return;

/* Unmap I/O buffer */
dma_unmap ( &intel->rx.map[rx_idx] );

/* Populate I/O buffer */
iobuf = intel->rx.iobuf[rx_idx];
intel->rx.iobuf[rx_idx] = NULL;
iobuf = intel->rx_iobuf[rx_idx];
intel->rx_iobuf[rx_idx] = NULL;
len = le16_to_cpu ( rx->length );
iob_put ( iobuf, len );

Expand All @@ -873,7 +840,7 @@ void intel_poll_rx ( struct net_device *netdev ) {
intel, rx_idx, len );
netdev_rx ( netdev, iobuf );
}
intel->rx.ring.cons++;
intel->rx.cons++;
}
}

Expand Down Expand Up @@ -981,9 +948,9 @@ static int intel_probe ( struct pci_device *pci ) {
memset ( intel, 0, sizeof ( *intel ) );
intel->port = PCI_FUNC ( pci->busdevfn );
intel->flags = pci->id->driver_data;
intel_init_ring ( &intel->tx.ring, INTEL_NUM_TX_DESC, INTEL_TD,
intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTEL_TD,
intel_describe_tx );
intel_init_ring ( &intel->rx.ring, INTEL_NUM_RX_DESC, INTEL_RD,
intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTEL_RD,
intel_describe_rx );

/* Fix up PCI device */
Expand All @@ -999,6 +966,7 @@ static int intel_probe ( struct pci_device *pci ) {
/* Configure DMA */
intel->dma = &pci->dma;
dma_set_mask_64bit ( intel->dma );
netdev->dma = intel->dma;

/* Reset the NIC */
if ( ( rc = intel_reset ( intel ) ) != 0 )
Expand Down

0 comments on commit 8d337ec

Please sign in to comment.