[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH v13 17/19] multi-process: create IOHUB object to handle irq
From: |
Jagannathan Raman |
Subject: |
[PATCH v13 17/19] multi-process: create IOHUB object to handle irq |
Date: |
Mon, 14 Dec 2020 00:14:57 -0500 |
IOHUB object is added to manage PCI IRQs. It uses KVM_IRQFD
ioctl to create irqfd to injecting PCI interrupts to the guest.
IOHUB object forwards the irqfd to the remote process. Remote process
uses this fd to directly send interrupts to the guest, bypassing QEMU.
Signed-off-by: John G Johnson <john.g.johnson@oracle.com>
Signed-off-by: Jagannathan Raman <jag.raman@oracle.com>
Signed-off-by: Elena Ufimtseva <elena.ufimtseva@oracle.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
include/hw/pci/pci_ids.h | 3 +
include/hw/remote/iohub.h | 42 ++++++++++++++
include/hw/remote/machine.h | 3 +
include/hw/remote/mpqemu-link.h | 1 +
include/hw/remote/proxy.h | 4 ++
hw/remote/iohub.c | 119 ++++++++++++++++++++++++++++++++++++++++
hw/remote/machine.c | 10 ++++
hw/remote/message.c | 4 ++
hw/remote/mpqemu-link.c | 5 ++
hw/remote/proxy.c | 56 +++++++++++++++++++
MAINTAINERS | 2 +
hw/remote/meson.build | 1 +
12 files changed, 250 insertions(+)
create mode 100644 include/hw/remote/iohub.h
create mode 100644 hw/remote/iohub.c
diff --git a/include/hw/pci/pci_ids.h b/include/hw/pci/pci_ids.h
index 11f8ab7..bd0c17d 100644
--- a/include/hw/pci/pci_ids.h
+++ b/include/hw/pci/pci_ids.h
@@ -192,6 +192,9 @@
#define PCI_DEVICE_ID_SUN_SIMBA 0x5000
#define PCI_DEVICE_ID_SUN_SABRE 0xa000
+#define PCI_VENDOR_ID_ORACLE 0x108e
+#define PCI_DEVICE_ID_REMOTE_IOHUB 0xb000
+
#define PCI_VENDOR_ID_CMD 0x1095
#define PCI_DEVICE_ID_CMD_646 0x0646
diff --git a/include/hw/remote/iohub.h b/include/hw/remote/iohub.h
new file mode 100644
index 0000000..fc9f8b5
--- /dev/null
+++ b/include/hw/remote/iohub.h
@@ -0,0 +1,42 @@
+/*
+ * IO Hub for remote device
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef REMOTE_IOHUB_H
+#define REMOTE_IOHUB_H
+
+#include "hw/pci/pci.h"
+#include "qemu/event_notifier.h"
+#include "qemu/thread-posix.h"
+#include "hw/remote/mpqemu-link.h"
+
+#define REMOTE_IOHUB_NB_PIRQS PCI_DEVFN_MAX
+
+typedef struct ResampleToken {
+ void *iohub;
+ int pirq;
+} ResampleToken;
+
+typedef struct RemoteIOHubState {
+ PCIDevice d;
+ EventNotifier irqfds[REMOTE_IOHUB_NB_PIRQS];
+ EventNotifier resamplefds[REMOTE_IOHUB_NB_PIRQS];
+ unsigned int irq_level[REMOTE_IOHUB_NB_PIRQS];
+ ResampleToken token[REMOTE_IOHUB_NB_PIRQS];
+ QemuMutex irq_level_lock[REMOTE_IOHUB_NB_PIRQS];
+} RemoteIOHubState;
+
+int remote_iohub_map_irq(PCIDevice *pci_dev, int intx);
+void remote_iohub_set_irq(void *opaque, int pirq, int level);
+void process_set_irqfd_msg(PCIDevice *pci_dev, MPQemuMsg *msg);
+
+void remote_iohub_init(RemoteIOHubState *iohub);
+void remote_iohub_finalize(RemoteIOHubState *iohub);
+
+#endif
diff --git a/include/hw/remote/machine.h b/include/hw/remote/machine.h
index 3073db6..f8720d7 100644
--- a/include/hw/remote/machine.h
+++ b/include/hw/remote/machine.h
@@ -15,11 +15,14 @@
#include "hw/boards.h"
#include "hw/pci-host/remote.h"
#include "io/channel.h"
+#include "hw/remote/iohub.h"
typedef struct RemoteMachineState {
MachineState parent_obj;
RemotePCIHost *host;
+
+ RemoteIOHubState iohub;
} RemoteMachineState;
/* Used to pass to co-routine device and ioc. */
diff --git a/include/hw/remote/mpqemu-link.h b/include/hw/remote/mpqemu-link.h
index caa40da..0de995e 100644
--- a/include/hw/remote/mpqemu-link.h
+++ b/include/hw/remote/mpqemu-link.h
@@ -39,6 +39,7 @@ typedef enum {
MPQEMU_CMD_PCI_CFGREAD,
MPQEMU_CMD_BAR_WRITE,
MPQEMU_CMD_BAR_READ,
+ MPQEMU_CMD_SET_IRQFD,
MPQEMU_CMD_MAX,
} MPQemuCmd;
diff --git a/include/hw/remote/proxy.h b/include/hw/remote/proxy.h
index 5f35b3b..02f9c26 100644
--- a/include/hw/remote/proxy.h
+++ b/include/hw/remote/proxy.h
@@ -12,6 +12,7 @@
#include "hw/pci/pci.h"
#include "io/channel.h"
#include "hw/remote/proxy-memory-listener.h"
+#include "qemu/event_notifier.h"
#define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"
@@ -42,6 +43,9 @@ struct PCIProxyDev {
QIOChannel *ioc;
Error *migration_blocker;
ProxyMemoryListener proxy_listener;
+ int virq;
+ EventNotifier intr;
+ EventNotifier resample;
ProxyMemoryRegion region[PCI_NUM_REGIONS];
};
diff --git a/hw/remote/iohub.c b/hw/remote/iohub.c
new file mode 100644
index 0000000..4dbf43a
--- /dev/null
+++ b/hw/remote/iohub.c
@@ -0,0 +1,119 @@
+/*
+ * Remote IO Hub
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_ids.h"
+#include "hw/pci/pci_bus.h"
+#include "qemu/thread.h"
+#include "hw/boards.h"
+#include "hw/remote/machine.h"
+#include "hw/remote/iohub.h"
+#include "qemu/main-loop.h"
+
+void remote_iohub_init(RemoteIOHubState *iohub)
+{
+ int pirq;
+
+ memset(&iohub->irqfds, 0, sizeof(iohub->irqfds));
+ memset(&iohub->resamplefds, 0, sizeof(iohub->resamplefds));
+
+ for (pirq = 0; pirq < REMOTE_IOHUB_NB_PIRQS; pirq++) {
+ qemu_mutex_init(&iohub->irq_level_lock[pirq]);
+ iohub->irq_level[pirq] = 0;
+ event_notifier_init_fd(&iohub->irqfds[pirq], -1);
+ event_notifier_init_fd(&iohub->resamplefds[pirq], -1);
+ }
+}
+
+void remote_iohub_finalize(RemoteIOHubState *iohub)
+{
+ int pirq;
+
+ for (pirq = 0; pirq < REMOTE_IOHUB_NB_PIRQS; pirq++) {
+ qemu_set_fd_handler(event_notifier_get_fd(&iohub->resamplefds[pirq]),
+ NULL, NULL, NULL);
+ event_notifier_cleanup(&iohub->irqfds[pirq]);
+ event_notifier_cleanup(&iohub->resamplefds[pirq]);
+ qemu_mutex_destroy(&iohub->irq_level_lock[pirq]);
+ }
+}
+
+int remote_iohub_map_irq(PCIDevice *pci_dev, int intx)
+{
+ return pci_dev->devfn;
+}
+
+void remote_iohub_set_irq(void *opaque, int pirq, int level)
+{
+ RemoteIOHubState *iohub = opaque;
+
+ assert(pirq >= 0);
+ assert(pirq < PCI_DEVFN_MAX);
+
+ QEMU_LOCK_GUARD(&iohub->irq_level_lock[pirq]);
+
+ if (level) {
+ if (++iohub->irq_level[pirq] == 1) {
+ event_notifier_set(&iohub->irqfds[pirq]);
+ }
+ } else if (iohub->irq_level[pirq] > 0) {
+ iohub->irq_level[pirq]--;
+ }
+}
+
+static void intr_resample_handler(void *opaque)
+{
+ ResampleToken *token = opaque;
+ RemoteIOHubState *iohub = token->iohub;
+ int pirq, s;
+
+ pirq = token->pirq;
+
+ s = event_notifier_test_and_clear(&iohub->resamplefds[pirq]);
+
+ assert(s >= 0);
+
+ QEMU_LOCK_GUARD(&iohub->irq_level_lock[pirq]);
+
+ if (iohub->irq_level[pirq]) {
+ event_notifier_set(&iohub->irqfds[pirq]);
+ }
+}
+
+void process_set_irqfd_msg(PCIDevice *pci_dev, MPQemuMsg *msg)
+{
+ RemoteMachineState *machine = REMOTE_MACHINE(current_machine);
+ RemoteIOHubState *iohub = &machine->iohub;
+ int pirq, intx;
+
+ intx = pci_get_byte(pci_dev->config + PCI_INTERRUPT_PIN) - 1;
+
+ pirq = remote_iohub_map_irq(pci_dev, intx);
+
+ if (event_notifier_get_fd(&iohub->irqfds[pirq]) != -1) {
+ qemu_set_fd_handler(event_notifier_get_fd(&iohub->resamplefds[pirq]),
+ NULL, NULL, NULL);
+ event_notifier_cleanup(&iohub->irqfds[pirq]);
+ event_notifier_cleanup(&iohub->resamplefds[pirq]);
+ memset(&iohub->token[pirq], 0, sizeof(ResampleToken));
+ }
+
+ event_notifier_init_fd(&iohub->irqfds[pirq], msg->fds[0]);
+ event_notifier_init_fd(&iohub->resamplefds[pirq], msg->fds[1]);
+
+ iohub->token[pirq].iohub = iohub;
+ iohub->token[pirq].pirq = pirq;
+
+ qemu_set_fd_handler(msg->fds[1], intr_resample_handler, NULL,
+ &iohub->token[pirq]);
+}
diff --git a/hw/remote/machine.c b/hw/remote/machine.c
index 17d07ec..42d971e 100644
--- a/hw/remote/machine.c
+++ b/hw/remote/machine.c
@@ -20,12 +20,15 @@
#include "exec/address-spaces.h"
#include "exec/memory.h"
#include "qapi/error.h"
+#include "hw/pci/pci_host.h"
+#include "hw/remote/iohub.h"
static void remote_machine_init(MachineState *machine)
{
MemoryRegion *system_memory, *system_io, *pci_memory;
RemoteMachineState *s = REMOTE_MACHINE(machine);
RemotePCIHost *rem_host;
+ PCIHostState *pci_host;
system_memory = get_system_memory();
system_io = get_system_io();
@@ -45,6 +48,13 @@ static void remote_machine_init(MachineState *machine)
memory_region_add_subregion_overlap(system_memory, 0x0, pci_memory, -1);
qdev_realize(DEVICE(rem_host), sysbus_get_default(), &error_fatal);
+
+ pci_host = PCI_HOST_BRIDGE(rem_host);
+
+ remote_iohub_init(&s->iohub);
+
+ pci_bus_irqs(pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq,
+ &s->iohub, REMOTE_IOHUB_NB_PIRQS);
}
static void remote_machine_class_init(ObjectClass *oc, void *data)
diff --git a/hw/remote/message.c b/hw/remote/message.c
index 54003b5..96ed11d 100644
--- a/hw/remote/message.c
+++ b/hw/remote/message.c
@@ -18,6 +18,7 @@
#include "hw/pci/pci.h"
#include "exec/memattrs.h"
#include "hw/remote/memory.h"
+#include "hw/remote/iohub.h"
static void process_config_write(QIOChannel *ioc, PCIDevice *dev,
MPQemuMsg *msg, Error **errp);
@@ -65,6 +66,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)
case MPQEMU_CMD_SYNC_SYSMEM:
remote_sysmem_reconfig(&msg, &local_err);
break;
+ case MPQEMU_CMD_SET_IRQFD:
+ process_set_irqfd_msg(pci_dev, &msg);
+ break;
default:
error_setg(&local_err,
"Unknown command (%d) received for device %s"
diff --git a/hw/remote/mpqemu-link.c b/hw/remote/mpqemu-link.c
index 139a69f..07273a0 100644
--- a/hw/remote/mpqemu-link.c
+++ b/hw/remote/mpqemu-link.c
@@ -254,6 +254,11 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)
return false;
}
break;
+ case MPQEMU_CMD_SET_IRQFD:
+ if (msg->size || (msg->num_fds != 2)) {
+ return false;
+ }
+ break;
default:
break;
}
diff --git a/hw/remote/proxy.c b/hw/remote/proxy.c
index 95d11ab..8926a05 100644
--- a/hw/remote/proxy.c
+++ b/hw/remote/proxy.c
@@ -20,6 +20,57 @@
#include "qemu/error-report.h"
#include "hw/remote/proxy-memory-listener.h"
#include "qom/object.h"
+#include "qemu/event_notifier.h"
+#include "sysemu/kvm.h"
+#include "util/event_notifier-posix.c"
+
+static void proxy_intx_update(PCIDevice *pci_dev)
+{
+ PCIProxyDev *dev = PCI_PROXY_DEV(pci_dev);
+ PCIINTxRoute route;
+ int pin = pci_get_byte(pci_dev->config + PCI_INTERRUPT_PIN) - 1;
+
+ if (dev->virq != -1) {
+ kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &dev->intr,
dev->virq);
+ dev->virq = -1;
+ }
+
+ route = pci_device_route_intx_to_irq(pci_dev, pin);
+
+ dev->virq = route.irq;
+
+ if (dev->virq != -1) {
+ kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &dev->intr,
+ &dev->resample, dev->virq);
+ }
+}
+
+static void setup_irqfd(PCIProxyDev *dev)
+{
+ PCIDevice *pci_dev = PCI_DEVICE(dev);
+ MPQemuMsg msg;
+ Error *local_err = NULL;
+
+ event_notifier_init(&dev->intr, 0);
+ event_notifier_init(&dev->resample, 0);
+
+ memset(&msg, 0, sizeof(MPQemuMsg));
+ msg.cmd = MPQEMU_CMD_SET_IRQFD;
+ msg.num_fds = 2;
+ msg.fds[0] = event_notifier_get_fd(&dev->intr);
+ msg.fds[1] = event_notifier_get_fd(&dev->resample);
+ msg.size = 0;
+
+ if (!mpqemu_msg_send(&msg, dev->ioc, &local_err)) {
+ error_report_err(local_err);
+ }
+
+ dev->virq = -1;
+
+ proxy_intx_update(pci_dev);
+
+ pci_device_set_intx_routing_notifier(pci_dev, proxy_intx_update);
+}
static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)
{
@@ -48,6 +99,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error
**errp)
qio_channel_set_blocking(dev->ioc, true, NULL);
proxy_memory_listener_configure(&dev->proxy_listener, dev->ioc);
+
+ setup_irqfd(dev);
}
static void pci_proxy_dev_exit(PCIDevice *pdev)
@@ -63,6 +116,9 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)
error_free(dev->migration_blocker);
proxy_memory_listener_deconfigure(&dev->proxy_listener);
+
+ event_notifier_cleanup(&dev->intr);
+ event_notifier_cleanup(&dev->resample);
}
static void config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t *val,
diff --git a/MAINTAINERS b/MAINTAINERS
index b276053..d6de35b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3167,6 +3167,8 @@ F: hw/remote/proxy.c
F: include/hw/remote/proxy.h
F: hw/remote/proxy-memory-listener.c
F: include/hw/remote/proxy-memory-listener.h
+F: hw/remote/iohub.c
+F: include/hw/remote/iohub.h
Build and test automation
-------------------------
diff --git a/hw/remote/meson.build b/hw/remote/meson.build
index 7f11be4..e6a5574 100644
--- a/hw/remote/meson.build
+++ b/hw/remote/meson.build
@@ -5,6 +5,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true:
files('mpqemu-link.c'))
remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c'))
remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c'))
remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c'))
+remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c'))
specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c'))
specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true:
files('proxy-memory-listener.c'))
--
1.8.3.1
- [PATCH v13 00/19] Initial support for multi-process Qemu, Jagannathan Raman, 2020/12/14
- [PATCH v13 07/19] multi-process: add qio channel function to transmit data and fds, Jagannathan Raman, 2020/12/14
- [PATCH v13 03/19] memory: alloc RAM from file at offset, Jagannathan Raman, 2020/12/14
- [PATCH v13 04/19] multi-process: Add config option for multi-process QEMU, Jagannathan Raman, 2020/12/14
- [PATCH v13 01/19] multi-process: add the concept description to docs/devel/qemu-multiprocess, Jagannathan Raman, 2020/12/14
- [PATCH v13 12/19] multi-process: introduce proxy object, Jagannathan Raman, 2020/12/14
- [PATCH v13 09/19] multi-process: Initialize message handler in remote device, Jagannathan Raman, 2020/12/14
- [PATCH v13 10/19] multi-process: Associate fd of a PCIDevice with its object, Jagannathan Raman, 2020/12/14
- [PATCH v13 13/19] multi-process: add proxy communication functions, Jagannathan Raman, 2020/12/14
- [PATCH v13 17/19] multi-process: create IOHUB object to handle irq,
Jagannathan Raman <=
- [PATCH v13 14/19] multi-process: Forward PCI config space acceses to the remote process, Jagannathan Raman, 2020/12/14
- [PATCH v13 18/19] multi-process: Retrieve PCI info from remote process, Jagannathan Raman, 2020/12/14
- [PATCH v13 16/19] multi-process: Synchronize remote memory, Jagannathan Raman, 2020/12/14
- [PATCH v13 05/19] multi-process: setup PCI host bridge for remote device, Jagannathan Raman, 2020/12/14
- [PATCH v13 06/19] multi-process: setup a machine object for remote device process, Jagannathan Raman, 2020/12/14
- [PATCH v13 02/19] multi-process: add configure and usage information, Jagannathan Raman, 2020/12/14
- [PATCH v13 08/19] multi-process: define MPQemuMsg format and transmission functions, Jagannathan Raman, 2020/12/14
- [PATCH v13 11/19] multi-process: setup memory manager for remote device, Jagannathan Raman, 2020/12/14
- [PATCH v13 15/19] multi-process: PCI BAR read/write handling for proxy & remote endpoints, Jagannathan Raman, 2020/12/14
- [PATCH v13 19/19] multi-process: perform device reset in the remote process, Jagannathan Raman, 2020/12/14