Skip to content

Commit

Permalink
[intelxl] Choose to operate in non-PXE mode
Browse files Browse the repository at this point in the history 
The physical function defaults to operating in "PXE mode" after a
power-on reset.  In this mode, receive descriptors are fetched and
written back as single descriptors.  In normal (non-PXE mode)
operation, receive descriptors are fetched and written back only as
complete cachelines unless an interrupt is raised.

There is no way to return to PXE mode from non-PXE mode, and there is
no way for the virtual function driver to operate in PXE mode.

Choose to operate in non-PXE mode.  This requires us to trick the
hardware into believing that it is raising an interrupt, so that it
will not defer writing back receive descriptors until a complete
cacheline (i.e. four packets) have been consumed.  We do so by
configuring the hardware to use MSI-X with a dummy target location in
place of the usual APIC register.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
  • Loading branch information
@mcb30
mcb30 committed Apr 27, 2019
1 parent 3078a95 commit 92b46b7
Show file tree
Hide file tree
Showing 2 changed files with 155 additions and 28 deletions.
133 changes: 113 additions & 20 deletions src/drivers/net/intelxl.c
View file
Expand Up @@ -116,6 +116,56 @@ static int intelxl_fetch_mac ( struct intelxl_nic *intelxl,
return 0;
}

/******************************************************************************
*
* MSI-X interrupts
*
******************************************************************************
*/

/**
* Enable MSI-X dummy interrupt
*
* @v intelxl Intel device
* @v pci PCI device
* @ret rc Return status code
*/
int intelxl_msix_enable ( struct intelxl_nic *intelxl,
struct pci_device *pci ) {
int rc;

/* Enable MSI-X capability */
if ( ( rc = pci_msix_enable ( pci, &intelxl->msix ) ) != 0 ) {
DBGC ( intelxl, "INTELXL %p could not enable MSI-X: %s\n",
intelxl, strerror ( rc ) );
return rc;
}

/* Configure interrupt zero to write to dummy location */
pci_msix_map ( &intelxl->msix, 0, virt_to_bus ( &intelxl->msg ), 0 );

/* Enable dummy interrupt zero */
pci_msix_unmask ( &intelxl->msix, 0 );

return 0;
}

/**
* Disable MSI-X dummy interrupt
*
* @v intelxl Intel device
* @v pci PCI device
*/
void intelxl_msix_disable ( struct intelxl_nic *intelxl,
struct pci_device *pci ) {

/* Disable dummy interrupt zero */
pci_msix_mask ( &intelxl->msix, 0 );

/* Disable MSI-X capability */
pci_msix_disable ( pci, &intelxl->msix );
}

/******************************************************************************
*
* Admin queue
Expand Down Expand Up @@ -480,6 +530,39 @@ static int intelxl_admin_shutdown ( struct intelxl_nic *intelxl ) {
return 0;
}

/**
* Clear PXE mode
*
* @v intelxl Intel device
* @ret rc Return status code
*/
static int intelxl_admin_clear_pxe ( struct intelxl_nic *intelxl ) {
struct intelxl_admin_descriptor *cmd;
struct intelxl_admin_clear_pxe_params *pxe;
uint32_t gllan_rctl_0;
int rc;

/* Do nothing if device is already out of PXE mode */
gllan_rctl_0 = readl ( intelxl->regs + INTELXL_GLLAN_RCTL_0 );
if ( ! ( gllan_rctl_0 & INTELXL_GLLAN_RCTL_0_PXE_MODE ) ) {
DBGC2 ( intelxl, "INTELXL %p already in non-PXE mode\n",
intelxl );
return 0;
}

/* Populate descriptor */
cmd = intelxl_admin_command_descriptor ( intelxl );
cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_CLEAR_PXE );
pxe = &cmd->params.pxe;
pxe->magic = INTELXL_ADMIN_CLEAR_PXE_MAGIC;

/* Issue command */
if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
return rc;

return 0;
}

/**
* Get switch configuration
*
Expand Down Expand Up @@ -1504,13 +1587,6 @@ static void intelxl_poll_rx ( struct net_device *netdev ) {
void intelxl_poll ( struct net_device *netdev ) {
struct intelxl_nic *intelxl = netdev->priv;

/* Acknowledge interrupts, if applicable */
if ( netdev_irq_enabled ( netdev ) ) {
writel ( ( INTELXL_INT_DYN_CTL_CLEARPBA |
INTELXL_INT_DYN_CTL_INTENA_MASK ),
( intelxl->regs + intelxl->intr ) );
}

/* Poll for completed packets */
intelxl_poll_tx ( netdev );

Expand All @@ -1522,19 +1598,23 @@ void intelxl_poll ( struct net_device *netdev ) {

/* Refill RX ring */
intelxl_refill_rx ( intelxl );
}

/**
* Enable or disable interrupts
*
* @v netdev Network device
* @v enable Interrupts should be enabled
*/
static void intelxl_irq ( struct net_device *netdev, int enable ) {
struct intelxl_nic *intelxl = netdev->priv;

writel ( ( enable ? INTELXL_INT_DYN_CTL_INTENA : 0 ),
( intelxl->regs + intelxl->intr ) );
/* Rearm interrupt, since otherwise receive descriptors will
* be written back only after a complete cacheline (four
* packets) have been received.
*
* There is unfortunately no efficient way to determine
* whether or not rearming the interrupt is necessary. If we
* are running inside a hypervisor (e.g. using a VF or PF as a
* passed-through PCI device), then the MSI-X write is
* redirected by the hypervisor to the real host APIC and the
* host ISR then raises an interrupt within the guest. We
* therefore cannot poll the nominal MSI-X target location to
* watch for the value being written. We could read from the
* INT_DYN_CTL register, but this is even less efficient than
* just unconditionally rearming the interrupt.
*/
writel ( INTELXL_INT_DYN_CTL_INTENA, intelxl->regs + intelxl->intr );
}

/** Network device operations */
Expand All @@ -1543,7 +1623,6 @@ static struct net_device_operations intelxl_operations = {
.close = intelxl_close,
.transmit = intelxl_transmit,
.poll = intelxl_poll,
.irq = intelxl_irq,
};

/******************************************************************************
Expand Down Expand Up @@ -1617,10 +1696,18 @@ static int intelxl_probe ( struct pci_device *pci ) {
if ( ( rc = intelxl_fetch_mac ( intelxl, netdev ) ) != 0 )
goto err_fetch_mac;

/* Enable MSI-X dummy interrupt */
if ( ( rc = intelxl_msix_enable ( intelxl, pci ) ) != 0 )
goto err_msix;

/* Open admin queues */
if ( ( rc = intelxl_open_admin ( intelxl ) ) != 0 )
goto err_open_admin;

/* Clear PXE mode */
if ( ( rc = intelxl_admin_clear_pxe ( intelxl ) ) != 0 )
goto err_admin_clear_pxe;

/* Get switch configuration */
if ( ( rc = intelxl_admin_switch ( intelxl ) ) != 0 )
goto err_admin_switch;
Expand Down Expand Up @@ -1667,8 +1754,11 @@ static int intelxl_probe ( struct pci_device *pci ) {
err_admin_promisc:
err_admin_vsi:
err_admin_switch:
err_admin_clear_pxe:
intelxl_close_admin ( intelxl );
err_open_admin:
intelxl_msix_disable ( intelxl, pci );
err_msix:
err_fetch_mac:
intelxl_reset ( intelxl );
err_reset:
Expand All @@ -1695,6 +1785,9 @@ static void intelxl_remove ( struct pci_device *pci ) {
/* Close admin queues */
intelxl_close_admin ( intelxl );

/* Disable MSI-X dummy interrupt */
intelxl_msix_disable ( intelxl, pci );

/* Reset the NIC */
intelxl_reset ( intelxl );

Expand Down
50 changes: 42 additions & 8 deletions src/drivers/net/intelxl.h
View file
Expand Up @@ -11,6 +11,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );

#include <stdint.h>
#include <ipxe/if_ether.h>
#include <ipxe/pcimsix.h>

struct intelxl_nic;

Expand Down Expand Up @@ -143,6 +144,20 @@ struct intelxl_admin_shutdown_params {
/** Driver is unloading */
#define INTELXL_ADMIN_SHUTDOWN_UNLOADING 0x01

/** Admin queue Clear PXE Mode command */
#define INTELXL_ADMIN_CLEAR_PXE 0x0110

/** Admin queue Clear PXE Mode command parameters */
struct intelxl_admin_clear_pxe_params {
/** Magic value */
uint8_t magic;
/** Reserved */
uint8_t reserved[15];
} __attribute__ (( packed ));

/** Clear PXE Mode magic value */
#define INTELXL_ADMIN_CLEAR_PXE_MAGIC 0x02

/** Admin queue Get Switch Configuration command */
#define INTELXL_ADMIN_SWITCH 0x0200

Expand Down Expand Up @@ -305,6 +320,8 @@ union intelxl_admin_params {
struct intelxl_admin_driver_params driver;
/** Shutdown command parameters */
struct intelxl_admin_shutdown_params shutdown;
/** Clear PXE Mode command parameters */
struct intelxl_admin_clear_pxe_params pxe;
/** Get Switch Configuration command parameters */
struct intelxl_admin_switch_params sw;
/** Get VSI Parameters command parameters */
Expand Down Expand Up @@ -563,6 +580,10 @@ struct intelxl_context_rx {
/** Queue Tail Pointer Register (offset) */
#define INTELXL_QXX_TAIL 0x8000

/** Global RLAN Control 0 register */
#define INTELXL_GLLAN_RCTL_0 0x12a500
#define INTELXL_GLLAN_RCTL_0_PXE_MODE 0x00000001UL /**< PXE mode */

/** Transmit data descriptor */
struct intelxl_tx_data_descriptor {
/** Buffer address */
Expand Down Expand Up @@ -709,22 +730,27 @@ intelxl_init_ring ( struct intelxl_ring *ring, unsigned int count, size_t len,
ring->context = context;
}

/** Number of transmit descriptors */
#define INTELXL_TX_NUM_DESC 16
/** Number of transmit descriptors
*
* Chosen to exceed the receive ring fill level, in order to avoid
* running out of transmit descriptors when sending TCP ACKs.
*/
#define INTELXL_TX_NUM_DESC 64

/** Transmit descriptor ring maximum fill level */
#define INTELXL_TX_FILL ( INTELXL_TX_NUM_DESC - 1 )

/** Number of receive descriptors
*
* In PXE mode (i.e. able to post single receive descriptors), 8
* descriptors is the only permitted value covering all possible
* numbers of PFs.
* Must be a multiple of 32.
*/
#define INTELXL_RX_NUM_DESC 8
#define INTELXL_RX_NUM_DESC 32

/** Receive descriptor ring fill level */
#define INTELXL_RX_FILL ( INTELXL_RX_NUM_DESC - 1 )
/** Receive descriptor ring fill level
*
* Must be a multiple of 8 and greater than 8.
*/
#define INTELXL_RX_FILL 16

/******************************************************************************
*
Expand Down Expand Up @@ -837,6 +863,10 @@ struct intelxl_nic {
unsigned int qset;
/** Interrupt control register */
unsigned int intr;
/** MSI-X capability */
struct pci_msix msix;
/** MSI-X dummy interrupt target */
uint32_t msg;

/** Admin command queue */
struct intelxl_admin command;
Expand All @@ -851,6 +881,10 @@ struct intelxl_nic {
struct io_buffer *rx_iobuf[INTELXL_RX_NUM_DESC];
};

extern int intelxl_msix_enable ( struct intelxl_nic *intelxl,
struct pci_device *pci );
extern void intelxl_msix_disable ( struct intelxl_nic *intelxl,
struct pci_device *pci );
extern struct intelxl_admin_descriptor *
intelxl_admin_command_descriptor ( struct intelxl_nic *intelxl );
extern union intelxl_admin_buffer *
Expand Down

0 comments on commit 92b46b7

Please sign in to comment.