-- MichaelReese - 14 Aug 2023

Device Driver Tutorial EB-slave WB-master over PCIe

The goal is to write an Etherbone slave that controls a wishbone master on the FPGA over PCIe. An archive with the code and the text in markdown is attached.

This driver does allow to start saftlib, but ECA does not work, possibly because the driver uses the Direct Access Mode of the PCIe-wishbone bridge and this does not support control of the cycle line.

The code shown here compiles for kernel version 6.4. For kernel 6.3 the function class_create needs an additional parameter to compile "class_create(THIS_MODULE, ...)".

Chapter 1: Setup the build system, insert and remove a trivial kernel module

Requirements

In Arch-Linux, the packages "base-devel" and "linux-headers" are needed. At the time of writing, the default kernel version used in Arch-Linux was 6.4.4.

Source code and Makefile

After installing these packages, the system contains a Makefile

/lib/modules/6.4.4-arch1-1/build/Makefile

which can be used to compile kernel modules (.ko files) for this kernel version. The simplest kernel Module contains only two functions, an "init" and an "exit" functions. These functions need to be registered with the "module_init" and "module_exit" macros. The relevant code [ebs_wbm_pcie.c](./chapter1/ebs_wbm_pcie.c) looks like this

static int __init ebs_wbm_pcie_init(void)
{
    printk(KERN_INFO "Simple kernel module doing nothing Inserted successfully\n");
    return 0;
}
static void __exit ebs_wbm_pcie_exit(void)
{
    printk(KERN_INFO "Simple kernel module removed\n");
}
module_init(ebs_wbm_pcie_init);
module_exit(ebs_wbm_pcie_exit);

and it can be compiled by calling the above mentioned Makefile which is used from within a local Makefile like the following:

obj-m += ebs_wbm_pcie.o
 
KDIR = /lib/modules/$(shell uname -r)/build

all:
   make -C $(KDIR)  M=$(shell pwd) modules

It only does 2 things

• it adds the object file to the "obj-m" variable
• it calls the Makfile from the kernel "/build/" directory (using the -C switch) with the variable "M" set to the directory of our kernel module source code

Building, inserting, removing the kernel module

The kernel module can be build by typing "make". The result is a file "ebs_wbm_pcie.ko" which can be inserted into the running kernel. Calling "dmesg" shows that the module was actually loaded, the output from the printk functions is visible.

[michael@acopc064 chapter1]$ make
make -C /lib/modules/6.4.4-arch1-1/build  M=/home/michael/work/etherbone_pcie_device_driver_tutorial/chapter1 modules
  CC [M]  /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter1/ebs_wbm_pcie.o
  MODPOST /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter1/Module.symvers
  CC [M]  /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter1/ebs_wbm_pcie.mod.o
  LD [M]  /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter1/ebs_wbm_pcie.ko
  BTF [M] /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter1/ebs_wbm_pcie.ko
[michael@acopc064 chapter1]$ sudo dmesg -C
[michael@acopc064 chapter1]$ sudo insmod ebs_wbm_pcie.ko 
[michael@acopc064 chapter1]$ sudo dmesg
[281021.335150] Simple kernel module doing nothing inserted successfully
[michael@acopc064 chapter1]$ sudo rmmod ebs_wbm_pcie
[michael@acopc064 chapter1]$ sudo dmesg
[281021.335150] Simple kernel module doing nothing inserted successfully
[281030.894961] Simple kernel module removed

Chapter 2: Connect the file operations and create a device file

File Operations

System calls of user space programs that operate on file descriptors (open/close/read/write) have a counter part in the device driver. In the driver source, the function addresses are stored in the file_operations struct. For now the four functions (open/close/read/write) will be registered. The following code has to be added to the source file

#include <linux/fs.h>
#include <linux/cdev.h>

static int ebs_wbm_pcie_open(struct inode *inode, struct file *file)
{
        printk(KERN_INFO "%s\n", __func__);
        return 0;
}
static int ebs_wbm_pcie_release(struct inode *inode, struct file *file)
{
        printk(KERN_INFO "%s\n", __func__);
        return 0;
}
static ssize_t ebs_wbm_pcie_read(struct file *filp, char __user *buf, size_t len, loff_t *off)
{
        printk(KERN_INFO "%s\n", __func__);
        return 0;
}
static ssize_t ebs_wbm_pcie_write(struct file *filp, const char __user *buf, size_t len, loff_t *off)
{
        printk(KERN_INFO "%s\n", __func__);
        return len;
}

static struct file_operations fops =
{
    .owner      = THIS_MODULE,
    .read       = ebs_wbm_pcie_read,
    .write      = ebs_wbm_pcie_write,
    .open       = ebs_wbm_pcie_open,
    .release    = ebs_wbm_pcie_release,
};

static struct cdev cdev;

And in the init function the file operations have to be linked to the cdev instance;

static int __init ebs_wbm_pcie_init(void)
{
    cdev_init(&cdev,&fops);
    /* same as before */
}

Device File

In addition to the implementation of the file access functions a device file must be created that user space application can use to open and read/write to. Three more include files are needed

#include <linux/kdev_t.h>
#include <linux/err.h>
#include <linux/device.h>

And the init function has to call some functions to register the device class "ebs_wbm_pcie_class" and create the device file "ebs_wbm_pcie_device" under "/dev/ebs_wbm_pcie_device". The cleanup code in case of errors is at the end of the functions.

dev_t dev = 0;
static struct class *dev_class;
 
static int __init ebs_wbm_pcie_init(void)
{
    cdev_init(&cdev,&fops);
    if ((alloc_chrdev_region(&dev, 0, 1, "ebs_wbm_pcie_dev")) <0) {
        return -1;
    }
    if((cdev_add(&cdev,dev,1)) < 0){
        goto r_class;
    }
    dev_class = class_create("ebs_wbm_pcie_class");
    if(IS_ERR(dev_class)){
        goto r_class;
    }
    if(IS_ERR(device_create(dev_class,NULL,dev,NULL,"ebs_wbm_pcie_device"))){
        goto r_device;
    }

    printk(KERN_INFO "Simple kernel module inserted successfully\n");
    return 0;
 
r_device:
    class_destroy(dev_class);
r_class:
    unregister_chrdev_region(dev,1);
    return -1;
}

Cleanup code must also be added to the exit function to free the kernel resources after the kernel module was removed.

static void __exit ebs_wbm_pcie_exit(void)
{
    device_destroy(dev_class,dev);
    class_destroy(dev_class);
    unregister_chrdev_region(dev, 1);
    printk(KERN_INFO "Simple kernel module removed\n");
}

When the driver is loaded, a device file will be created. When userspace programs (echo/cat) operate on this device file the corresponding functions in the kernel module will be called, as can be seen in dmesg output: echo calls "ebs_wbm_pcie_write" and cat calls "ebs_wbm_pcie_read"

[root@acopc064 chapter2]# make
make -C /lib/modules/6.4.4-arch1-1/build  M=/home/michael/work/etherbone_pcie_device_driver_tutorial/chapter2 modules
  CC [M]  /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter2/ebs_wbm_pcie.o
  MODPOST /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter2/Module.symvers
  CC [M]  /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter2/ebs_wbm_pcie.mod.o
  LD [M]  /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter2/ebs_wbm_pcie.ko
  BTF [M] /home/michael/work/etherbone_pcie_device_driver_tutorial/chapter2/ebs_wbm_pcie.ko
[root@acopc064 chapter2]# insmod ebs_wbm_pcie.ko 
[root@acopc064 chapter2]# echo -ne x > /dev/ebs_wbm_pcie_device 
[root@acopc064 chapter2]# dmesg
[292193.587926] Simple kernel module inserted successfully
[292225.966201] ebs_wbm_pcie_open
[292225.966223] ebs_wbm_pcie_write
[292225.966227] ebs_wbm_pcie_release
[root@acopc064 chapter2]# cat /dev/ebs_wbm_pcie_device 
[root@acopc064 chapter2]# dmesg
[292193.587926] Simple kernel module inserted successfully
[292225.966201] ebs_wbm_pcie_open
[292225.966223] ebs_wbm_pcie_write
[292225.966227] ebs_wbm_pcie_release
[292235.991337] ebs_wbm_pcie_open
[292235.991359] ebs_wbm_pcie_read
[292235.991369] ebs_wbm_pcie_release

What actually happens is that udev creates the file "/dev/ebs_wbm_pcie_device". For more details see here: https://fastbitlab.com/creating-device-files/ .

Chapter 3: Use the PCI subsystem from the driver

Linux kernel provides infrastructure to connect to PCI devices. The driver code code needs to include the "linux/pci.h" header and register itself to the pci subsystem of the kernel by calling the function "pci_register_driver". This function needs to know the Vendor and product ID of the PCI hardware that should be controlled with the driver. In addition, it needs two functions "probe" and "remove", which are called when the driver is loaded and the PCI subsystem knows a PCI device with matching IDs. The new "ebs_wbm_pcie_init" and "ebs_wbm_pcie_exit" functions are now only doing the registration/unregistration at the PCI subsystem. The old "ebs_wbm_pcie_init" and "ebs_wbm_pcie_exit" functions are renamed to "install_device" and "remove_device" and they are called from within the "probe" and "remove" functions.

static int install_device(void) /* in chapter2 this function was called ebs_wbm_pcie_init */
/*...*/
static void remove_device(void) /* in chapter2 this was called ebs_wbm_pcie_exit */
/*...*/

#include <linux/pci.h>
 
#define PCI_DRIVER_NAME "ebs_wbm" 
static int probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
    printk(KERN_INFO "probe was called\n");
    return install_device();
}
static void remove(struct pci_dev *pdev)
{
    remove_device();
    printk(KERN_INFO "remove was called\n");
}

#define PCIE_WB_VENDOR_ID 0x10dc
#define PCIE_WB_DEVICE_ID 0x019a 
static struct pci_device_id ids[] = {
    { PCI_DEVICE(PCIE_WB_VENDOR_ID, PCIE_WB_DEVICE_ID), },
    { 0, }
};
MODULE_DEVICE_TABLE(pci, ids);

static struct pci_driver pcie_wb_driver = {
    .name     = PCI_DRIVER_NAME,
    .id_table = ids,
    .probe    = probe,
    .remove   = remove,
};

static int __init ebs_wbm_pcie_init(void)
{
    return pci_register_driver(&pcie_wb_driver);
}
static void __exit ebs_wbm_pcie_exit(void)
{
    pci_unregister_driver(&pcie_wb_driver);
}

After compiling the kernel module, it can be inserted and removed. If a PCIe Timing Receiver (e.g. pexarria5) is connected to the system, the "probe" and remove "functions" will be called.

[root@belpc121 chapter3]# dmesg -C
[root@belpc121 chapter3]# insmod ebs_wbm_pcie.ko 
[root@belpc121 chapter3]# dmesg 
[18502.667063] probe was called
[18502.667205] Simple kernel module inserted successfully
[root@belpc121 chapter3]# rmmod ebs_wbm_pcie
[root@belpc121 chapter3]# dmesg 
[18502.667063] probe was called
[18502.667205] Simple kernel module inserted successfully
[18512.874309] Simple kernel module removed
[18512.874316] remove was called

Chapter 4: Mapping hardware registers into the driver

Until now, the driver does not read or write any registers of the hardware.

Base Address Registers (BARs)

PCI-hardware provides address ranges that can be accessed by the host. These are called Base Address Registers (BARs). On the FPGA of the pexarri5 is a PCI-wishbone bridge with two Base Address Register (BAR). - BAR0 : Contains control registers to set/release the wishbone cycle line, set wishbone sel-bits (and other functions). Register 0x4 in BAR0 is the Direct Access Control Register (DACR) and can be used to put the bridge into Direct Access Mode by writing any value other than -1 into it. In this mode each R/W access to BAR1 will be directly mapped to one complete wishbone R/W cycle. This mode will be used in the driver because it is easier. - BAR1 : Represents read/write address range. - In normal mode each read/write to this address range will cause a wishbone read/write strobe with the settings (sel bits, keep cycle line, address offset) from the BAR0 registers. - In Direct Access Mode each read or write to this address range will open a wishbone cycle, do a read or write strobe at address offset given by the content of the DACR, byte selection sel=0xf, wait for ack and close the cycle.

The driver code to setup the BARs fills a structure with the "start","end","size" of the BAR and "addr" is an address that can later be used to read from or write to the BAR registers.

struct pcie_wb_resource {
    unsigned long start;    /* start addr of BAR */
    unsigned long end;      /* end addr of BAR */
    unsigned long size;     /* size of BAR */
    void *addr;             /* remapped addr */
} bar0, bar1;

static int setup_bar(struct pci_dev* pdev, struct pcie_wb_resource* res, int bar)
{
    res->start = pci_resource_start(pdev, bar);
    res->end   = pci_resource_end(pdev, bar);
    res->size  = res->end - res->start + 1;

    printk(KERN_INFO "setup bar %d  0x%lx - 0x%lx\n", bar, res->start, res->end);

    if ((pci_resource_flags(pdev, 0) & IORESOURCE_MEM) == 0) {
        printk(KERN_INFO "bar %d is not a memory resource\n", bar);
        return -ENOMEM;
    }

    if (!request_mem_region(res->start, res->size, PCI_DRIVER_NAME)) {
        printk(KERN_INFO "bar %d: request_mem_region failed\n", bar);
        return -ENOMEM;
    }

    res->addr = ioremap_cache(res->start, res->size);
    printk(KERN_INFO "bar %d: ioremap to %lx\n", bar, (unsigned long)res->addr);

    return 0;
}

static void destroy_bar(struct pcie_wb_resource* res)
{
    printk(KERN_INFO "released io 0x%lx\n", res->start);

    iounmap(res->addr);
    release_mem_region(res->start, res->size);
}

The functions "setup_bar" and "destroy_bar" can be called in the "probe" and "remove" functions to get access to the two BARs provided by the hardware.

static int probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
    printk(KERN_INFO "probe was called\n");
    if (pci_enable_device(pdev) < 0)   goto r_out;
    if (setup_bar(pdev, &bar0, 0) < 0) goto r_disable;
    if (setup_bar(pdev, &bar1, 1) < 0) goto r_bar0;
    if (install_device() < 0)          goto r_bar1;
    return 0;
r_bar1:
    destroy_bar(&bar1);
r_bar0: 
    destroy_bar(&bar0);
r_disable:
    pci_disable_device(pdev);
r_out:
    return -EIO;
}
static void remove(struct pci_dev *pdev)
{
    remove_device();
    destroy_bar(&bar1);
    destroy_bar(&bar0);
    pci_disable_device(pdev);
    printk(KERN_INFO "remove was called\n");
}

After compiling, inserting and removing the kernel module, one can see the bar0 and bar1 address ranges with dmesg:

[root@belpc121 chapter4]# insmod ebs_wbm_pcie.ko 
[root@belpc121 chapter4]# dmesg 
[21370.007329] probe was called
[21370.007419] setup bar 0  0xfb000000 - 0xfb00007f
[21370.007443] bar 0: ioremap to ffffb9ac0008f000
[21370.007445] setup bar 1  0xfa000000 - 0xfaffffff
[21370.007455] bar 1: ioremap to ffffb9ac09000000
[21370.007653] Simple kernel module inserted successfully
[root@belpc121 chapter4]# rmmod ebs_wbm_pcie.ko 
[root@belpc121 chapter4]# dmesg 
[21370.007329] probe was called
[21370.007419] setup bar 0  0xfb000000 - 0xfb00007f
[21370.007443] bar 0: ioremap to ffffb9ac0008f000
[21370.007445] setup bar 1  0xfa000000 - 0xfaffffff
[21370.007455] bar 1: ioremap to ffffb9ac09000000
[21370.007653] Simple kernel module inserted successfully
[21380.854106] Simple kernel module removed
[21380.854112] released io 0xfa000000
[21380.854126] released io 0xfb000000
[21380.854219] remove was called

Chapter 5: Writing to and reading from hardware

Now everything is set up to read and write hardware registers. In the Direct Access Mode reading and writing can be done by first putting the wishbone target address into the DACR of BAR0 and then read/write the BAR1 address 0. Register access is done with the "ioread32" and "iowrite32" functions. As a first step the driver will only read and write from a single hard coded address: The first word of LM32 user ram, which is (at the time of this writing) at address 0x4060000 on the pexarria5. The definition of the global variables "bar0" and "bar1" has to be moved to the beginning of the file so that they can be accessed by the "ebs_wbm_pcie_read" and "ebs_wbm_pcie_write" functions.

static ssize_t ebs_wbm_pcie_read(struct file *filp, char __user *buf, size_t len, loff_t *off)
{
    int value;
    printk(KERN_INFO "%s\n", __func__);
    if (len < sizeof(value)) return 0;
    /* write hard coded address into the Direct Access Control Register (DACR) */
    iowrite32(0x4060000, bar0.addr + 0x4); 
    /* make a wishbone read cycle in the hardware */
    value = ioread32(bar1.addr);
    if (copy_to_user(buf, &value, sizeof(value))) {
        return 0;
    }
    return sizeof(value);
}
static ssize_t ebs_wbm_pcie_write(struct file *filp, const char __user *buf, size_t len, loff_t *off)
{
    int value;
    printk(KERN_INFO "%s\n", __func__);
    if (len < sizeof(value)) return 0;
    if (copy_from_user(&value, buf, sizeof(value))) {
        return 0;
    }
    /* write hard coded address into the Direct Access Control Register (DACR) */
    iowrite32(0x4060000, bar0.addr + 0x4); 
    /* make a wishbone write cycle in the hardware */
    iowrite32(value, bar1.addr);
    return sizeof(value);
}

A small user space program (userspace.c) is needed to test the driver read/write functionality.

#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
int main(int argc, char *argv[]) {
    int fd = open("/dev/ebs_wbm_pcie_device", O_RDWR);
    int value;
    if (argc == 2) {
        sscanf(argv[1],"%x",&value);
        write(fd, &value, sizeof(value));
    }
    read(fd, &value, sizeof(value));
    printf("value = %08x\n",value);
    return 0;
}

A small test run looks like this. In this example an additional connection to the hardware via USB is used to verify that the hardware RAM was really modified.

[root@belpc121 chapter5]# insmod ebs_wbm_pcie.ko 
[root@belpc121 chapter5]# dmesg
[25799.311437] probe was called
[25799.311509] setup bar 0  0xfb000000 - 0xfb00007f
[25799.311529] bar 0: ioremap to ffffb9ac0008f000
[25799.311531] setup bar 1  0xfa000000 - 0xfaffffff
[25799.311537] bar 1: ioremap to ffffb9ac09000000
[25799.311655] Simple kernel module inserted successfully
[root@belpc121 chapter5]# gcc -o userspace userspace.c 
[root@belpc121 chapter5]# ./userspace 
value = 90c00000
[root@belpc121 chapter5]# eb-read dev/ttyUSB0 0x4060000/4
90c00000
[root@belpc121 chapter5]# ./userspace 0x12345678
value = 12345678
[root@belpc121 chapter5]# eb-read dev/ttyUSB0 0x4060000/4
12345678
[root@belpc121 chapter5]# dmesg
[25799.311437] probe was called
[25799.311509] setup bar 0  0xfb000000 - 0xfb00007f
[25799.311529] bar 0: ioremap to ffffb9ac0008f000
[25799.311531] setup bar 1  0xfa000000 - 0xfaffffff
[25799.311537] bar 1: ioremap to ffffb9ac09000000
[25799.311655] Simple kernel module inserted successfully
[25829.772748] ebs_wbm_pcie_open
[25829.772757] ebs_wbm_pcie_read
[25829.772844] ebs_wbm_pcie_release
[25856.396612] ebs_wbm_pcie_open
[25856.396649] ebs_wbm_pcie_write
[25856.396653] ebs_wbm_pcie_read
[25856.396769] ebs_wbm_pcie_release

Chapter 6: Etherbone slave implementation

The driver can at the moment only write single words. All eb-tools expect to talk to an etherbone slave device. They write etherbone header and etherbone records into the device file and read back the result. Details of the etherbone protocol are described in the [etherbone specifications](https://www-acc.gsi.de/wiki/pub/FESA/SCUTesting/Wishbone-core-api.pdf) in chapter 5. For example, the communication between eb-tool and etherbone slave when reading a value (from the Mailbox in this case) by executing "eb-read -p 0x800/4" looks like this:

eb-master-out -> eb-slave-out
-----------------------------
0x4e6f11ff    -> 0x4e6f1644    Etherbone Header
0x00000086    -> 0x00000086    Etherbone Header
0xa00f0001    -> 0x060f0100    EB-Record: master sends Rcount=1, slave responds Wcount=1
0x00008000    -> 0x00008000    EB-Record: master sends base return address, slave repeats it
0x00000800    -> 0xffffffff    EB-Record: master sends read address, slave sends data at this address
0xe80f0001    -> 0x0e0f0100    EB-Record: master reads config space 
0x00008001    -> 0x00008001    EB-Record: master sends base return address, slave repeats it
0x00000004    -> 0x00000000    EB-Record: master sends 4 (address of error shift register)

The etherbone master sends a 32-bit word and the etherbone slave responds to it. If eb-tools should be used with the driver, it needs to implement etherbone slave behavior. The etherbone slave logic is implemented in a separate file "ebs.c" with a header file "ebs.h". It implements the behavior as described in [etherbone specifications](https://www-acc.gsi.de/wiki/pub/FESA/SCUTesting/Wishbone-core-api.pdf) and will be not discussed in detail here. It must be used with an instance of the EbSlaveState structure which must be configured with three function pointers - hw_write(addr,data): a function that executes a wishbone write at the given address in hardware. - hw_read(addr): a function that executes a wishbone read from the given address and returns the result. - hv_cfg_read(addr): a function that provides the etherbone configuration space registers, mainly the error shift registers. Whenever the device file is opened by a process, the EbSlaveState structure must be reset with the eb_slave_reset function. Whenever a 32-bit word is written to the device file, it can be processes with the eb_slave_process function, which interprets the word, executes the requested action on the hardware (call to hw_read/hw_write/hw_cfg_read) and provides the etherbone response word. The following code does the setup of the EbSlaveState variable

void hw_write(uint32_t addr, uint32_t data) {
    iowrite32(addr, bar0.addr + 0x4); /* write addr into DACR */
    iowrite32(data, bar1.addr);       /* write the data */
}
uint32_t hw_read(uint32_t addr) {
    iowrite32(addr, bar0.addr + 0x4); /* write addr into DACR */
    return ioread32(bar1.addr);       /* read the data */
}
uint32_t hw_cfg_read(uint32_t addr) {
    /* this function maps the etherbone slave config register addresses (0x0,0x4,0x8,0xc)
       the the register addresses in the PCIe bridge on the FPGA */
    printk(KERN_INFO "hw_cfg_read %08x\n", addr);
    switch(addr) {
        case 0x0: return ioread32(bar0.addr +  8); /* err shiftreg high    */
        case 0x4: return ioread32(bar0.addr + 12); /* err shiftreg low     */
        case 0x8: return ioread32(bar0.addr + 24); /* wishbone addr offset */
        case 0xc: return ioread32(bar0.addr + 28); /* SDB start address    */
    }
    return 0;
}

#include "ebs.h"
EbSlaveState eb = {
    .hw_write    = hw_write,
    .hw_read     = hw_read,
    .hw_cfg_read = hw_cfg_read
};

The ebs_wbm_pcie_read and ebs_wbm_pcie_write functions have to be modified to do etherbone processing instead of reading/writing directly on the hardware. The ebs_wbm_pcie_write function looks at the incoming data as a series of 32-bit words and directly processes them using the eb_slave_process function. The response words are stored in a buffer that is used to provide the data when the etherbone master process reads the response by calling the ebs_wbm_pcie_read function. Note that this is a very simple implementation that assumes that any user code will always read as much data as was written before and that the amount of data is always smaller than BUFFERSIZE. In case the ebs_wbm_pcie_read function is called but there is no data present, the function should do one of two things - if the file descriptor id non-blocking, the function should return -EAGAIN. - if the file descriptor is blocking, the calling process should be put to sleep. This can be done with a wait queue and a call to "wait_event_interruptible". The process should be woken up ("wake_up_interruptible") if data arrived. Ignoring MSIs, the only way that data becomes available is when "ebs_wbm_pcie_write" was called.

The modified functions look like this:

#define BUFFERSIZE 8192
char buffer[BUFFERSIZE];
int write_buffer_idx = 0;
int read_buffer_idx = 0;
DECLARE_WAIT_QUEUE_HEAD(read_waitq);
static ssize_t ebs_wbm_pcie_read(struct file *filp, char __user *buf, size_t len, loff_t *off)
{
    printk(KERN_INFO "%s\n", __func__);
    if (read_buffer_idx == write_buffer_idx) { /* read was called but no data is available */
        if (filp->f_flags & O_NONBLOCK) { /* device is nonblocking -> return -EAGAIN */
            return -EAGAIN;
        } 
        /* device is blocking -> wait_event_interruptible */
        wait_event_interruptible(read_waitq, (read_buffer_idx<write_buffer_idx));
    }
    if (len > (write_buffer_idx-read_buffer_idx)) {
        len = write_buffer_idx-read_buffer_idx;
    }
    if (copy_to_user(buf, buffer+read_buffer_idx, len)) {
        return 0;
    }
    read_buffer_idx += len;
    if (read_buffer_idx == write_buffer_idx) {
        read_buffer_idx  = 0;
        write_buffer_idx = 0;
    }
    return len;
}
static ssize_t ebs_wbm_pcie_write(struct file *filp, const char __user *buf, size_t len, loff_t *off)
{
    printk(KERN_INFO "%s\n", __func__);
    if (len > BUFFERSIZE) len = BUFFERSIZE;
    if (copy_from_user(buffer, buf, len)) {
        return 0;
    }
    write_buffer_idx = len;
    for (int i = 0; i < write_buffer_idx; i += 4) {
        uint32_t request, response;
        request = *(int*)&buffer[i];
        eb_slave_process(&eb, &buffer[i]);
        response = *(int*)&buffer[i];
        printk(KERN_INFO "0x%08x -> 0x%08x\n", be32_to_cpu(request), be32_to_cpu(response));
    }
    wake_up_interruptible(&read_waitq);
    return len;
}

The module now consists of two source files "ebs.c" and "ebs_wbm_pcie.c". This requires a modification of the Makefile to compile the module. The module name will be changed to "ebswbm" because it causes problems when one source file has the module name. The modified makefile looks like this.

obj-m += ebswbm.o 
ebswbm-objs += ebs.o ebs_wbm_pcie.o

In the following test a memory address in the Mailbox device is written an read. Again the USB connection is used to verify that the data was actually written.

t@belpc121 chapter6]# insmod ebswbm.ko 
[root@belpc121 chapter6]# eb-write -p dev/ebs_wbm_pcie_device 0x804/4 0x12345678
[root@belpc121 chapter6]# dmesg
[109826.753448] ebs_wbm_pcie_init
[109826.753480] probe was called
[109826.753546] setup bar 0  0xfb000000 - 0xfb00007f
[109826.753565] bar 0: ioremap to ffffb9ac0008f000
[109826.753567] setup bar 1  0xfa000000 - 0xfaffffff
[109826.753573] bar 1: ioremap to ffffb9ac09000000
[109826.753697] Simple kernel module inserted successfully
[109836.679632] ebs_wbm_pcie_open
[109836.689713] ebs_wbm_pcie_read
[109836.689719] ebs_wbm_pcie_write
[109836.689720] 0x4e6f11ff -> 0x4e6f1644
[109836.689721] 0x00000086 -> 0x00000086
[109836.689731] ebs_wbm_pcie_read
[109836.689740] ebs_wbm_pcie_write
[109836.689741] 0xe80f0101 -> 0x00000000
[109836.689743] 0x00000804 -> 0x00000000
[109836.689744] 0x12345678 -> 0x0e0f0100
[109836.689745] 0x00008001 -> 0x00008001
[109836.689747] 0x00000004 -> 0x00000000
[109836.689751] ebs_wbm_pcie_read
[109836.689755] ebs_wbm_pcie_read
[109836.689757] ebs_wbm_pcie_release
[root@belpc121 chapter6]# eb-read -p dev/ebs_wbm_pcie_device 0x804/4 
12345678
[root@belpc121 chapter6]# dmesg
[109851.976399] ebs_wbm_pcie_open
[109851.986538] ebs_wbm_pcie_read
[109851.986546] ebs_wbm_pcie_write
[109851.986549] 0x4e6f11ff -> 0x4e6f1644
[109851.986551] 0x00000086 -> 0x00000086
[109851.986565] ebs_wbm_pcie_read
[109851.986579] ebs_wbm_pcie_write
[109851.986580] 0xa00f0001 -> 0x060f0100
[109851.986582] 0x00008000 -> 0x00008000
[109851.986586] 0x00000804 -> 0x12345678
[109851.986587] 0xe80f0001 -> 0x0e0f0100
[109851.986589] 0x00008001 -> 0x00008001
[109851.986591] 0x00000004 -> 0x00000000
[109851.986598] ebs_wbm_pcie_read
[109851.986605] ebs_wbm_pcie_read
[109851.986608] ebs_wbm_pcie_release
[root@belpc121 chapter6]# eb-read -p dev/ttyUSB0 0x804/4 
12345678

This driver also works with most other eb-tools like eb-ls or eb-console.

Chapter 7: Support poll system call

User space programs can use the poll system call to check if a file descriptor is ready to read/write. If the driver does not implement the poll function, the poll system call in a user space program will always return immediately. The poll function for the ebswbm driver looks like this.

#include <linux/poll.h>
static unsigned int ebs_wbm_pcie_poll(struct file *filp, struct poll_table_struct *wait)
{
    unsigned int mask = 0;
    printk(KERN_INFO "%s\n", __func__);
    poll_wait(filp, &read_waitq, wait);
    if (read_buffer_idx < write_buffer_idx)  {
        mask |= POLLIN | POLLRDNORM; /* readable if read cursor is left of write cursor */
    }
    if (write_buffer_idx == 0) {
        mask |= POLLOUT | POLLWRNORM; /* always writable */
    }
    return mask;
}
static struct file_operations fops =
{
    .owner      = THIS_MODULE,
    .read       = ebs_wbm_pcie_read,
    .write      = ebs_wbm_pcie_write,
    .poll       = ebs_wbm_pcie_poll,    /* new */
    .open       = ebs_wbm_pcie_open,
    .release    = ebs_wbm_pcie_release,
};

It checks the positions of the buffer read/write index values and sets the flags of the mask to indicate if the device can be written to or read from. It also adds the wait queue "read_waitq", which was used in the read function to implement blocking read. This wait queue has to be added to the "struct poll_table_struct". That allows the kernel to be informed when some changes in the state of the driver occur that change the result of the poll function. In this case, the only place where the state of the driver changes is in the "ebs_wbm_pcie_write" function which calls "wake_up_interruptible" at the end. This causes the read function to unblock, and the poll function to be called again.

The following program accesses the ebswbm driver and checks on two places if there is data to read: - before writing anything. The driver has nothing to respond to and the poll call will wait until the given timeout (1000 ms) is reached. - after writing the etherbone header. In this case poll will return immediately because the etherbone slave response can be read. The writing is done from a separate thread, so that "write" is called while "poll" is waiting.

#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <poll.h>
#include <pthread.h>
int fd;
int request1[2] = {0xff116f4e, 
                   0x86000000};
void *send_etherbone_request(void *data) {
    sleep(2);
    write(fd, request1, sizeof(request1));
}
int main(int argc, char *argv[]) {
    fd = open("/dev/ebs_wbm_pcie_device", O_RDWR);
    struct pollfd pfd;
    pfd.fd = fd;
    pfd.events = POLLIN;
    int response1[2];
    pthread_t thread;
    pthread_create(&thread, NULL, &send_etherbone_request, NULL); // thread will wait 2 seconds before writing etherbone request
    printf("a) pollresult=%d\n",poll(&pfd,1,-1)); // nothing to read yet. will wait until there is data to read.
    pthread_join(thread, NULL);
    printf("b) pollresult=%d\n",poll(&pfd,1,-1)); // response can be read. will return immediately and return 1.
    read(fd, response1, sizeof(response1));
    for (int i = 0; i < 2; ++i ) {
        printf("%08x -> %08x\n", request1[i], response1[i]);
    }
    return 0;
}

Chapter 8: Interrupts

The PCIe bridge on the FPGA can interrupt the host. The driver can register a function that is called by the kernel whenever that interrupt is raised. The following additions to the driver are needed: The first function enables/disables the interrupts on the hardware in the FPGA by setting/deleting flags in the control register (in BAR0).

static void wb_set_interrupt(int on)
{
    static int CONTROL_REGISTER_HIGH  = 0;
    uint32_t value  = 0x10000000;
    if (on)  value |= 0x20000000;
    iowrite32(value, bar0.addr + CONTROL_REGISTER_HIGH);
}

The next function reads three registers in from the hardware that contain address and data of the latest interrupt and a flag that is set if these values are valid MSI data. If the valid_data was set, new values can be popped from the hardware queue.

static int wb_read_msi_data(void)
{
    static int MASTER_CTL_HIGH = 64; 
    static int MASTER_ADR_LOW  = 76;
    static int MASTER_DAT_LOW  = 84;
    uint32_t ctl       = ioread32(bar0.addr + MASTER_CTL_HIGH);
    uint32_t msi_addr  = ioread32(bar0.addr + MASTER_ADR_LOW);
    uint32_t msi_data  = ioread32(bar0.addr + MASTER_DAT_LOW);
    uint32_t msi_valid = (ctl & 0x80000000) != 0;
    if (msi_valid) {
        printk(KERN_INFO "got IRQ data: %08x %08x msi_valid=%d\n", msi_addr, msi_data, msi_valid);
        iowrite32(1, bar0.addr + MASTER_CTL_HIGH); /* pop the data from the hardware queue */
        iowrite32(2, bar0.addr + MASTER_CTL_HIGH); /* send ack */
    }
    return msi_valid;
}

The last function is the actual interrupt handler that will be called by the kenrel.

static irqreturn_t irq_handler(int irq, void *dev_id)
{
    printk(KERN_INFO "%s\n", __func__);
    wb_set_interrupt(0); /* disable IRQ */
    wb_read_msi_data();
    wb_set_interrupt(1); /* enable IRQ */
    return IRQ_HANDLED;
} 

Interrupts are enabled and the irq_handler is registered by adding the following code (and some cleanup code that is not shown here) the probe function.

static int probe(struct pci_dev *pdev, const struct pci_device_id *id)
{ 
    /* ... */
    pci_set_master(pdev); 
    if (pci_enable_msi(pdev) < 0) {
        goto r_bar1;
    }
    if (request_irq(pdev->irq, irq_handler, IRQF_SHARED, "ebs_wbm_pcie", pdev) < 0) {
        goto r_msi;
    }
    while (wb_read_msi_data()); /* empty the queue */
    wb_set_interrupt(1);
    /* ... */

To test if the irq_handler is actually called, the Mailbox device is used. It can be configured to make a wishbone write onto the PCIe bridge's wishbone slave interface. For the pexarria5, the address range of the PICe bridge's slave interface is 0x10000 to 0x1ffff. The address 0x804 of the Mailbox has to be configured with the target address (any value in the PCIe birdge's slave address range) by writing a value to it. The Mailbox can be triggered by writing any value to address 0x800. The Mailbox will then do a wishbone write to the configured address and the data that was written to trigger the Mailbox.

[root@belpc121 chapter8]# insmod ebswbm.ko 
[root@belpc121 chapter8]# eb-write -p dev/ttyUSB0 0x804/4 0x10100
[root@belpc121 chapter8]# eb-write -p dev/ttyUSB0 0x800/4 0x12345678
[root@belpc121 chapter8]# dmesg
[189608.247535] ebs_wbm_pcie_init
[189608.247567] probe was called
[189608.247636] setup bar 0  0xfb000000 - 0xfb00007f
[189608.247657] bar 0: ioremap to ffffb9ac0008f000
[189608.247658] setup bar 1  0xfa000000 - 0xfaffffff
[189608.247665] bar 1: ioremap to ffffb9ac09000000
[189608.247849] Simple kernel module inserted successfully
[189627.999205] irq_handler
[189627.999213] got IRQ data: 00000100 12345678 msi_valid=1

Indeed the irq_handler was called and the value (0x12345678) and address (0x00000100) were read from the registers of the hardware. As it is now, the driver discards the address and data. In the next step these need to be packed into an etherbone write request and must be provided through the ebs_wbm_pcie_read function.

Chapter 9: Redirect the etherbone MSI data to userspace

The driver was written to always provide N words of data to read for N words of data written. This is different for MSI data, which may be suddenly available even if the etherbone master did not ask for it. Provinding the MSI data must be done when the etherbone master does not wait for a write response. As soon as the etherbone master sends an etherbone request, the response to this request has priority over any MSI data.

The poll function must be modified to take availability of MSI data into account.

Another Problem is the concurrent execution of the interrupt handler and the rest of the driver code. The inerrupt handler code can run in parallel to the rest of the driver code and access to shared data must take this into account.

Concept

MSI data consists of a data word, and an address word. They are translated into an etherbone write request (4 byte record header, 4 byte address, 4 byte data). A response to the etherbone write request is not expected.

The interrupt handler does not check if the etherbone data can be send because this would require accessing shared data. Instead, it stores the data into a buffer and modifies an index value (msi_valid) that points to the buffer position of the latest MSI data. If the pointer reaches the end of the buffer, it is wrapped around to 0. There is a second index intor the buffer (msi_handled) points to the latest handled (i.e. copy_to_user in the ebs_wbm_pcie_read function). msi_valid is only written in the irq_handler, msi_handled is only written in the ebs_wbm_pcie_read function.

Both ebs_wbm_pcie_poll and ebs_wbm_pcie_read compare msi_valid and msi_handled to find out if unhandled MSIs are present. If so, the MSI data is only send if no etherbone master request is open (read_buffer_idx < write_buffer_idx).

There is no protection against overflow of the buffer.

Question: Should msi_valid and msi_handled be atomic variables?

Code modifications

The function wb_read_msi_data has to convert the MSI data into etherbone write request data in the msi_buffer and update the msi_valid index. It also has to wake up the read_waitq if MSI data is present. This function is called once in the initialization of the driver to pull out all pending interrupts from the hardware. The function has an argument to control if the read_waitq should be woken up or not.

static void wb_read_msi_data(int wakeup)
{
    char *msi_window;
    int new_msis = 0;
    for(;;) {
        uint32_t msi_ctl  = ioread32(bar0.addr + MASTER_CTL_HIGH);
        uint32_t msi_addr = ioread32(bar0.addr + MASTER_ADR_LOW);
        uint32_t msi_data = ioread32(bar0.addr + MASTER_DAT_LOW);
        if (msi_ctl&0x80000000) { /* valid data */
            msi_window = msi_buffer+msi_valid;
            msi_window[0]=        0xa8; msi_window[1]=        0x0f; msi_window[ 2]=       0x01; msi_window[ 3]=       0x00;
            msi_window[4]=msi_addr>>24; msi_window[5]=msi_addr>>16; msi_window[ 6]=msi_addr>>8; msi_window[ 7]=msi_addr>>0;
            msi_window[8]=msi_data>>24, msi_window[9]=msi_data>>16, msi_window[10]=msi_data>>8, msi_window[11]=msi_data>>0;
            msi_valid = (msi_valid+16)%BUFFERSIZE; 
            iowrite32(1, bar0.addr + MASTER_CTL_HIGH); /* pop the data from the hardware queue */
            iowrite32(2, bar0.addr + MASTER_CTL_HIGH); /* send ack */
            ++new_msis;
        } else {
            break;
        }
    }
    if (wakeup && new_msis) {
        wake_up_interruptible(&read_waitq);
    }
}

The new msi_buffer is defined at the same place where the etherbone buffer is already defined.

uint32_t msi_valid = 0;
uint32_t msi_handled = 0;
char msi_buffer[BUFFERSIZE];

The conditions in the read function must be extended to check if MSI data is present (msi_valid = msi_handled). If response data is present it is prioritized over the MSI data, i.e. MSI data is only delivered if there is no response data present (read_buffer_idx == write_buffer_idx). The index is increased by 16 (not by 12) bytes because the BUFFERSIZE cannot be integer divided by 12, but by 16.

static ssize_t ebs_wbm_pcie_read(struct file *filp, char __user *buf, size_t len, loff_t *off)
{
    ssize_t ndata = len;
    char *msi_window;
    if (read_buffer_idx == write_buffer_idx && (msi_valid == msi_handled)) { /* read was called but no data is available */
        if (filp->f_flags & O_NONBLOCK) { /* device is nonblocking -> return -EAGAIN */
            return -EAGAIN;
        } 
        wait_event_interruptible(read_waitq, ((read_buffer_idx<write_buffer_idx) || (msi_valid != msi_handled)) );
    }
    if ((read_buffer_idx == write_buffer_idx) && (msi_valid != msi_handled)) {
        msi_window = msi_buffer+msi_handled;
        if (copy_to_user(buf, msi_window, 12)) {
            return 0;
        }
        msi_handled = (msi_handled+16)%BUFFERSIZE;
        return 12;
    }
    if (ndata > (write_buffer_idx-read_buffer_idx)) {
        ndata = write_buffer_idx-read_buffer_idx;
    } 
    if (copy_to_user(buf, buffer+read_buffer_idx, ndata)) {
        return 0;
    }
    read_buffer_idx += ndata;
    if (read_buffer_idx == write_buffer_idx) {
        read_buffer_idx  = 0;
        write_buffer_idx = 0;
    }
    return ndata;
}

The poll function must also check for the presence of MSI data similar to the read function.

static unsigned int ebs_wbm_pcie_poll(struct file *filp, struct poll_table_struct *wait)
{
    unsigned int mask = 0;
    poll_wait(filp, &read_waitq, wait);
    if ((read_buffer_idx < write_buffer_idx) || (msi_valid != msi_handled))  {
        mask |= POLLIN | POLLRDNORM; /* readable if read cursor is left of write cursor */
    }
    if (write_buffer_idx == 0) {
        mask |= POLLOUT | POLLWRNORM; 
    }

    return mask;
}

The etherbone configuration space needs some additional registers where the etherbone master can read the MSI address width from. These have to be added to the hw_cfg_read function.

uint32_t hw_cfg_read(uint32_t addr) {
    switch(addr) {
        case 0x0: return ioread32(bar0.addr +  8); /* err shiftreg high    */
        case 0x4: return ioread32(bar0.addr + 12); /* err shiftreg low     */
        case 0x8: return ioread32(bar0.addr + 24); /* wishbone addr offset */
        case 0xc: return ioread32(bar0.addr + 28); /* SDB start address    */
        case 0x28: return       0;                 /* MSI request high     */
        case 0x2c: return       1;                 /* MSI request low      */
        case 0x30: return       0;                 /* MSI granted high     */
        case 0x34: return  0x0000;                 /* MSI granted low      */
        case 0x38: return       0;                 /* MSI addr range high  */
        case 0x3c: return  0xffff;                 /* MSI addr range low   */
        default: ;
    }
    return 0;
}

This driver is sufficient to run saftlib. However, the ECA does not work yet. I think this is due to the lack of support of uninterrupted cycles. The driver can be tested for example using the "saftlib/examples/SimpleFirmware" plugin. This programs the first user LM32 with a program that waits for an MSI from the host and responsd with an MSI to the host.

[saftlib]$ cd examples/SimpleFirmware/
[SimpleFirmware]$ make install
SimpleFirmware]$ simple-firmware-standalone dev/ebs_wbm_pcie_device
OpenDevice::OpenDevice("dev/ebs_wbm_pcie_device")
OpenDevice first,last,mask = 0,ffff,ffff
msi_target_adr for poll check: 00019868
needs polling? no
TimingReceiver: registered irq 0x1dc50
TimingReceiver: registered irq 0x15cfc
TimingReceiver: registered irq 0x19448
TimingReceiver: registered irq 0x158ec
LM32Cluster::LM32Cluster
found 1 lm32 cpus
triggerMSI: 1
got MSI 1
triggerMSI: 2
got MSI 2
triggerMSI: 3
got MSI 3
#...