From 5a7e126c435ffb70fa57e021cffafe1e35543117 Mon Sep 17 00:00:00 2001 From: Chenli Wei Date: Tue, 29 Jul 2025 14:08:30 +0800 Subject: [PATCH 01/32] acrn: initial libvirt/ACRN import Port the dev-acrn-v6.1.0 branch from the following commit to dev-acrn-v10.0.0: commit e14234462057aeb51675549074c223d3d4b9c0a0 Signed-off-by: Chenli Wei --- include/libvirt/virterror.h | 1 + src/acrn/Makefile.inc.am | 80 ++ src/acrn/acrn_common.h | 92 ++ src/acrn/acrn_device.c | 175 +++ src/acrn/acrn_device.h | 7 + src/acrn/acrn_domain.c | 349 +++++ src/acrn/acrn_domain.h | 20 + src/acrn/acrn_driver.c | 2641 +++++++++++++++++++++++++++++++++++ src/acrn/acrn_driver.h | 5 + src/libvirt.c | 3 + src/remote/remote_daemon.c | 4 + src/util/virerror.c | 1 + 12 files changed, 3378 insertions(+) create mode 100644 src/acrn/Makefile.inc.am create mode 100644 src/acrn/acrn_common.h create mode 100644 src/acrn/acrn_device.c create mode 100644 src/acrn/acrn_device.h create mode 100644 src/acrn/acrn_domain.c create mode 100644 src/acrn/acrn_domain.h create mode 100644 src/acrn/acrn_driver.c create mode 100644 src/acrn/acrn_driver.h diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h index 224eddc9e4..ad466f6a80 100644 --- a/include/libvirt/virterror.h +++ b/include/libvirt/virterror.h @@ -141,6 +141,7 @@ typedef enum { VIR_FROM_TPM = 70, /* Error from TPM (Since: 5.6.0) */ VIR_FROM_BPF = 71, /* Error from BPF code (Since: 5.10.0) */ VIR_FROM_CH = 72, /* Error from Cloud-Hypervisor driver (Since: 7.5.0) */ + VIR_FROM_ACRN = 73, /* Error from acrn driver */ # ifdef VIR_ENUM_SENTINELS VIR_ERR_DOMAIN_LAST /* (Since: 0.9.13) */ diff --git a/src/acrn/Makefile.inc.am b/src/acrn/Makefile.inc.am new file mode 100644 index 0000000000..22465c4ac3 --- /dev/null +++ b/src/acrn/Makefile.inc.am @@ -0,0 +1,80 @@ +# vim: filetype=automake + +ACRN_DRIVER_SOURCES = \ + acrn/acrn_common.h \ + acrn/acrn_driver.h \ + acrn/acrn_driver.c \ + acrn/acrn_domain.h \ + acrn/acrn_domain.c \ + acrn/acrn_device.h \ + acrn/acrn_device.c \ + $(NULL) + +DRIVER_SOURCE_FILES += $(addprefix $(srcdir)/,$(ACRN_DRIVER_SOURCES)) +STATEFUL_DRIVER_SOURCE_FILES += $(addprefix $(srcdir)/,$(ACRN_DRIVER_SOURCES)) + +EXTRA_DIST += $(ACRN_DRIVER_SOURCES) + + +if WITH_ACRN +noinst_LTLIBRARIES += libvirt_driver_acrn_impl.la +libvirt_driver_acrn_la_SOURCES = +libvirt_driver_acrn_la_LIBADD = \ + libvirt_driver_acrn_impl.la \ + libvirt.la \ + $(GLIB_LIBS) \ + $(NULL) +mod_LTLIBRARIES += libvirt_driver_acrn.la +libvirt_driver_acrn_la_LDFLAGS = $(AM_LDFLAGS_MOD_NOUNDEF) + +libvirt_driver_acrn_impl_la_CFLAGS = \ + -I$(srcdir)/access \ + -I$(builddir)/access \ + -I$(srcdir)/conf \ + -I$(srcdir)/hypervisor \ + $(AM_CFLAGS) \ + $(NULL) +libvirt_driver_acrn_impl_la_LDFLAGS = $(AM_LDFLAGS) +libvirt_driver_acrn_impl_la_LIBADD = -luuid +libvirt_driver_acrn_impl_la_SOURCES = $(ACRN_DRIVER_SOURCES) + +sbin_PROGRAMS += virtacrnd + +nodist_conf_DATA += acrn/virtacrnd.conf +augeas_DATA += acrn/virtacrnd.aug +augeastest_DATA += acrn/test_virtacrnd.aug +CLEANFILES += acrn/virtacrnd.aug + +virtacrnd_SOURCES = $(REMOTE_DAEMON_SOURCES) +nodist_virtacrnd_SOURCES = $(REMOTE_DAEMON_GENERATED) +virtacrnd_CFLAGS = \ + $(REMOTE_DAEMON_CFLAGS) \ + -DDAEMON_NAME="\"virtacrnd\"" \ + -DMODULE_NAME="\"acrn\"" \ + $(NULL) +virtacrnd_LDFLAGS = $(REMOTE_DAEMON_LD_FLAGS) +virtacrnd_LDADD = $(REMOTE_DAEMON_LD_ADD) + +acrn/virtacrnd.conf: remote/libvirtd.conf.in + $(AM_V_GEN)$(SED) \ + -e '/[@]CUT_ENABLE_IP[@]/,/[@]END[@]/d' \ + -e 's/[@]DAEMON_NAME[@]/virtacrnd/' \ + $< > $@ + +acrn/virtacrnd.aug: remote/libvirtd.aug.in + $(AM_V_GEN)$(SED) \ + -e '/[@]CUT_ENABLE_IP[@]/,/[@]END[@]/d' \ + -e 's/[@]DAEMON_NAME[@]/virtacrnd/' \ + -e 's/[@]DAEMON_NAME_UC[@]/Virtacrnd/' \ + $< > $@ + +acrn/test_virtacrnd.aug: remote/test_libvirtd.aug.in \ + acrn/virtacrnd.conf $(AUG_GENTEST_SCRIPT) + $(AM_V_GEN)$(AUG_GENTEST) acrn/virtacrnd.conf \ + $(srcdir)/remote/test_libvirtd.aug.in | \ + $(SED) \ + -e '/[@]CUT_ENABLE_IP[@]/,/[@]END[@]/d' \ + -e 's/[@]DAEMON_NAME[@]/virtacrnd/' \ + -e 's/[@]DAEMON_NAME_UC[@]/Virtacrnd/' \ + > $@ || rm -f $@ +endif WITH_ACRN diff --git a/src/acrn/acrn_common.h b/src/acrn/acrn_common.h new file mode 100644 index 0000000000..90e66623a7 --- /dev/null +++ b/src/acrn/acrn_common.h @@ -0,0 +1,92 @@ +/* + * Commmon IOCTL ID defination for VHM/DM + */ +#define _IC_ID(x, y) (((x)<<24)|(y)) +#define IC_ID 0x43UL + +/* General */ +#define IC_ID_GEN_BASE 0x0UL +#define IC_GET_PLATFORM_INFO _IC_ID(IC_ID, IC_ID_GEN_BASE + 0x03) + +/* ACRN guest severity */ +enum acrn_vm_severity { + SEVERITY_SAFETY_VM = 0x40U, + SEVERITY_RTVM = 0x30U, + SEVERITY_SOS = 0x20U, + SEVERITY_STANDARD_VM = 0x10U, +}; + +enum acrn_vm_load_order { + PRE_LAUNCHED_VM = 0, + SOS_VM, + POST_LAUNCHED_VM, + MAX_LOAD_ORDER +}; + +#define MAX_VM_OS_NAME_LEN 32U + +typedef struct acrn_vm_config acrnVmCfg; +typedef acrnVmCfg *acrnVmCfgPtr; +struct acrn_vm_config { + enum acrn_vm_load_order load_order; /* specify the load order of VM */ + char name[MAX_VM_OS_NAME_LEN]; /* VM name identifier, useful for debug. */ + const uint8_t uuid[16]; /* UUID of the VM */ + uint8_t reserved[2]; /* Temporarily reserve it so that don't need to update + * the users of get_platform_info frequently. + */ + uint8_t severity; /* severity of the VM */ + uint64_t cpu_affinity; /* The set bits represent the pCPUs the vCPUs of + * the VM may run on. + */ + uint64_t guest_flags; /* VM flags that we want to configure for guest */ + /* + * The following are hv-specific members and are thus opaque. + * vm_config_entry_size determines the real size of this structure. + */ +} __attribute__((aligned(8))); + +typedef struct platform_info acrnPlatformInfo; +typedef acrnPlatformInfo *acrnPlatformInfoPtr; +struct platform_info { + /** Hardware Information */ + /** Physical CPU number */ + uint16_t cpu_num; + + /** version of this structure */ + uint16_t version; + + /** Align the size of version & hardware info to 128Bytes. */ + uint8_t reserved0[124]; + + /** Configuration Information */ + /** Maximum vCPU number for one VM. */ + uint16_t max_vcpus_per_vm; + + /** Maximum Kata container number in SOS VM */ + uint8_t max_kata_containers; + + uint8_t reserved1[7]; + + /** Number of configured VMs */ + uint16_t max_vms; + + /** + * The size of acrn_vm_config is various on different platforms. + * This is the size of this struct which is used for the caller + * to parse the vm_configs array. + */ + uint32_t vm_config_entry_size; + + /** + * Address to an array of struct acrn_vm_config, containing all + * the configurations of all VMs. VHM treats it as an opague data + * structure. + * * + * The size of one array element is vm_config_entry_size while + * the number of elements is max_vms. + */ + uint64_t vm_configs_addr; + + /** Align the size of Configuration info to 128Bytes. */ + uint8_t reserved2[104]; +} __attribute__((aligned(8))); diff --git a/src/acrn/acrn_device.c b/src/acrn/acrn_device.c new file mode 100644 index 0000000000..8f242a4ec4 --- /dev/null +++ b/src/acrn/acrn_device.c @@ -0,0 +1,175 @@ +#include + +#include "acrn_device.h" +#include "domain_addr.h" +#include "virlog.h" + +#define VIR_FROM_THIS VIR_FROM_ACRN + +VIR_LOG_INIT("acrn.acrn_device"); + +static int +acrnCollectPCIAddress(virDomainDefPtr def G_GNUC_UNUSED, + virDomainDeviceDefPtr dev G_GNUC_UNUSED, + virDomainDeviceInfoPtr info, + void *opaque) +{ + virDomainPCIAddressSetPtr addrs; + virPCIDeviceAddressPtr addr; + + if (info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) + return 0; + + addrs = opaque; + addr = &info->addr.pci; + + if (!virPCIDeviceAddressIsEmpty(addr) && + virDomainPCIAddressReserveAddr( + addrs, addr, + VIR_PCI_CONNECT_TYPE_PCI_DEVICE, 0) < 0) { + virBuffer buf = VIR_BUFFER_INITIALIZER; + + virPCIDeviceAddressFormat(&buf, *addr, false); + virReportError(VIR_ERR_INTERNAL_ERROR, + _("failed to reserve PCI addr: %s"), + virBufferCurrentContent(&buf)); + virBufferFreeAndReset(&buf); + return -1; + } + + return 0; +} + +static virDomainPCIAddressSetPtr +acrnDomainPCIAddressSetCreate(virDomainDefPtr def, unsigned int nbuses) +{ + virDomainPCIAddressSetPtr addrs; + virPCIDeviceAddress lpc_addr; + + if (!(addrs = virDomainPCIAddressSetAlloc( + nbuses, VIR_PCI_ADDRESS_EXTENSION_NONE))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + return NULL; + } + + if (virDomainPCIAddressBusSetModel( + &addrs->buses[0], + VIR_DOMAIN_CONTROLLER_MODEL_PCI_ROOT) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("failed to set PCI bus model")); + goto error; + } + + memset(&lpc_addr, 0, sizeof(lpc_addr)); + lpc_addr.slot = 0x1; + + /* explicitly reserve 0:1:0 for LPC-ISA bridge */ + if (virDomainPCIAddressReserveAddr( + addrs, &lpc_addr, + VIR_PCI_CONNECT_TYPE_PCI_DEVICE, 0) < 0) { + virBuffer buf = VIR_BUFFER_INITIALIZER; + + virPCIDeviceAddressFormat(&buf, lpc_addr, false); + virReportError(VIR_ERR_INTERNAL_ERROR, + _("failed to reserve PCI addr for LPC-ISA bridge: %s"), + virBufferCurrentContent(&buf)); + virBufferFreeAndReset(&buf); + goto error; + } + + if (virDomainDeviceInfoIterate(def, acrnCollectPCIAddress, addrs) < 0) + goto error; + + return addrs; + +error: + virDomainPCIAddressSetFree(addrs); + return NULL; +} + +static int +acrnAssignPCIAddress(virDomainDefPtr def G_GNUC_UNUSED, + virDomainDeviceDefPtr dev, + virDomainDeviceInfoPtr info, + void *opaque) +{ + virDomainPCIAddressSetPtr addrs = opaque; + + switch (dev->type) { + case VIR_DOMAIN_DEVICE_DISK: + case VIR_DOMAIN_DEVICE_NET: + case VIR_DOMAIN_DEVICE_HOSTDEV: + if (virDeviceInfoPCIAddressIsWanted(info) && + virDomainPCIAddressReserveNextAddr(addrs, info, + VIR_PCI_CONNECT_TYPE_PCI_DEVICE, + -1) < 0) + goto fail; + break; + case VIR_DOMAIN_DEVICE_CONTROLLER: { + virDomainControllerDefPtr ctrl = dev->data.controller; + + /* PCI hostbridge is always 0:0:0 */ + if (ctrl->type != VIR_DOMAIN_CONTROLLER_TYPE_PCI && + virDeviceInfoPCIAddressIsWanted(info) && + virDomainPCIAddressReserveNextAddr(addrs, info, + VIR_PCI_CONNECT_TYPE_PCI_DEVICE, + -1) < 0) + goto fail; + break; + } + case VIR_DOMAIN_DEVICE_CHR: { + virDomainChrDefPtr chr = dev->data.chr; + + if (chr->deviceType == VIR_DOMAIN_CHR_DEVICE_TYPE_CONSOLE && + chr->targetType == VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_VIRTIO && + virDeviceInfoPCIAddressIsWanted(info) && + virDomainPCIAddressReserveNextAddr(addrs, info, + VIR_PCI_CONNECT_TYPE_PCI_DEVICE, + -1) < 0) + goto fail; + break; + } + case VIR_DOMAIN_DEVICE_INPUT: + case VIR_DOMAIN_DEVICE_WATCHDOG: + case VIR_DOMAIN_DEVICE_GRAPHICS: + case VIR_DOMAIN_DEVICE_RNG: + default: + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("device type %s"), + virDomainDeviceTypeToString(dev->type)); + return -1; + } + + return 0; + +fail: + virReportError(VIR_ERR_INTERNAL_ERROR, + _("PCI addr assignment failed for device type %s"), + virDomainDeviceTypeToString(dev->type)); + return -1; +} + +static int +acrnDomainAssignPCIAddresses(virDomainDefPtr def) +{ + virDomainPCIAddressSetPtr addrs; + int ret = -1; + + if (!(addrs = acrnDomainPCIAddressSetCreate(def, 1))) + goto cleanup; + + if (virDomainDeviceInfoIterate(def, acrnAssignPCIAddress, addrs) < 0) + goto cleanup; + + ret = 0; + +cleanup: + virDomainPCIAddressSetFree(addrs); + return ret; +} + +int +acrnDomainAssignAddresses(virDomainDefPtr def) +{ + return acrnDomainAssignPCIAddresses(def); +} diff --git a/src/acrn/acrn_device.h b/src/acrn/acrn_device.h new file mode 100644 index 0000000000..50b75b2a3a --- /dev/null +++ b/src/acrn/acrn_device.h @@ -0,0 +1,7 @@ +#ifndef __ACRN_DEVICE_H__ +#define __ACRN_DEVICE_H__ + +#include "domain_conf.h" + +int acrnDomainAssignAddresses(virDomainDefPtr def); +#endif /* __ACRN_DEVICE_H__ */ diff --git a/src/acrn/acrn_domain.c b/src/acrn/acrn_domain.c new file mode 100644 index 0000000000..e1f3dce6fb --- /dev/null +++ b/src/acrn/acrn_domain.c @@ -0,0 +1,349 @@ +#include + +#include "acrn_domain.h" +#include "acrn_device.h" +#include "viralloc.h" +#include "virfile.h" +#include "virlog.h" + +#define VIR_FROM_THIS VIR_FROM_ACRN + +VIR_LOG_INIT("acrn.acrn_domain"); + +static int +acrnDomainDefPostParse(virDomainDefPtr def, + unsigned int parseFlags G_GNUC_UNUSED, + void *opaque G_GNUC_UNUSED, + void *parseOpaque G_GNUC_UNUSED) +{ + /* Add an implicit PCI root controller */ + if (virDomainDefMaybeAddController(def, VIR_DOMAIN_CONTROLLER_TYPE_PCI, 0, + VIR_DOMAIN_CONTROLLER_MODEL_PCI_ROOT) < 0) + return -1; + + return 0; +} + +static int +acrnDomainDeviceDefPostParse(virDomainDeviceDefPtr dev, + const virDomainDef *def G_GNUC_UNUSED, + unsigned int parseFlags G_GNUC_UNUSED, + void *opaque G_GNUC_UNUSED, + void *parseOpaque G_GNUC_UNUSED) +{ + virDomainDeviceInfoPtr info; + + switch (dev->type) { + case VIR_DOMAIN_DEVICE_DISK: { + virDomainDiskDefPtr disk = dev->data.disk; + info = virDomainDeviceGetInfo(dev); + + if (virDomainDiskGetType(disk) != VIR_STORAGE_TYPE_FILE && + virDomainDiskGetType(disk) != VIR_STORAGE_TYPE_BLOCK) { + virReportError(VIR_ERR_XML_ERROR, + _("disk source %s not supported"), + virStorageTypeToString(disk->src->type)); + return -1; + } + + if (disk->bus == VIR_DOMAIN_DISK_BUS_VIRTIO) { + if (disk->device != VIR_DOMAIN_DISK_DEVICE_DISK) { + virReportError(VIR_ERR_XML_ERROR, + _("disk device %s not supported"), + virDomainDiskDeviceTypeToString(disk->device)); + return -1; + } + + if (info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE && + info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + virReportError(VIR_ERR_XML_ERROR, + _("disk address type %s not supported"), + virDomainDeviceAddressTypeToString(info->type)); + return -1; + } + } else if (disk->bus == VIR_DOMAIN_DISK_BUS_SATA) { + /* SATA disks must use VIR_DOMAIN_DEVICE_ADDRESS_TYPE_DRIVE */ + if (disk->device != VIR_DOMAIN_DISK_DEVICE_DISK && + disk->device != VIR_DOMAIN_DISK_DEVICE_CDROM) { + virReportError(VIR_ERR_XML_ERROR, + _("disk device %s not supported"), + virDomainDiskDeviceTypeToString(disk->device)); + return -1; + } + } else { + virReportError(VIR_ERR_XML_ERROR, + _("disk bus %s not supported"), + virDomainDiskBusTypeToString(disk->bus)); + return -1; + } + break; + } + case VIR_DOMAIN_DEVICE_NET: { + virDomainNetDefPtr net = dev->data.net; + info = virDomainDeviceGetInfo(dev); + + if (net->type == VIR_DOMAIN_NET_TYPE_ETHERNET) { + if (!net->ifname) { + virReportError(VIR_ERR_XML_ERROR, + _("net dev undefined")); + return -1; + } + } else if (net->type == VIR_DOMAIN_NET_TYPE_BRIDGE) { + if (!virDomainNetGetActualBridgeName(net)) { + virReportError(VIR_ERR_XML_ERROR, + _("bridge name undefined")); + return -1; + } + } else { + virReportError(VIR_ERR_XML_ERROR, + _("net type %s not supported"), + virDomainNetTypeToString(net->type)); + return -1; + } + + if (net->model != VIR_DOMAIN_NET_MODEL_VIRTIO) { + virReportError(VIR_ERR_XML_ERROR, + _("only virtio-net is supported")); + return -1; + } + + if (info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE && + info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + virReportError(VIR_ERR_XML_ERROR, + _("net address type %s not supported"), + virDomainDeviceAddressTypeToString(info->type)); + return -1; + } + break; + } + case VIR_DOMAIN_DEVICE_HOSTDEV: { + virDomainHostdevDefPtr hostdev = dev->data.hostdev; + info = virDomainDeviceGetInfo(dev); + + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) { + virReportError(VIR_ERR_XML_ERROR, + _("hostdev mode %s not supported"), + virDomainHostdevModeTypeToString(hostdev->mode)); + return -1; + } + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB && + hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) { + virReportError(VIR_ERR_XML_ERROR, + _("hostdev type %s not supported"), + virDomainHostdevSubsysTypeToString( + hostdev->source.subsys.type)); + return -1; + } + + if (hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI && + virPCIDeviceAddressIsEmpty(&hostdev->source.subsys.u.pci.addr)) { + virReportError(VIR_ERR_XML_ERROR, + _("PCI hostdev has no valid address")); + return -1; + } + + if (info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE && + info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + virReportError(VIR_ERR_XML_ERROR, + _("hostdev address type %s not supported"), + virDomainDeviceAddressTypeToString(info->type)); + return -1; + } + break; + } + case VIR_DOMAIN_DEVICE_CONTROLLER: { + virDomainControllerDefPtr ctrl = dev->data.controller; + + if (ctrl->type != VIR_DOMAIN_CONTROLLER_TYPE_SATA && + ctrl->type != VIR_DOMAIN_CONTROLLER_TYPE_VIRTIO_SERIAL && + ctrl->type != VIR_DOMAIN_CONTROLLER_TYPE_PCI) { + virReportError(VIR_ERR_XML_ERROR, + _("controller type %s not supported"), + virDomainControllerTypeToString(ctrl->type)); + return -1; + } + + if (ctrl->type == VIR_DOMAIN_CONTROLLER_TYPE_PCI) { + if ((virDomainControllerModelPCI)ctrl->model != + VIR_DOMAIN_CONTROLLER_MODEL_PCI_ROOT) { + virReportError(VIR_ERR_XML_ERROR, + _("PCI controller model %s not supported"), + virDomainControllerModelPCITypeToString(ctrl->model)); + return -1; + } + } else { + info = virDomainDeviceGetInfo(dev); + + if (info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE && + info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + virReportError(VIR_ERR_XML_ERROR, + _("controller address type %s not supported"), + virDomainDeviceAddressTypeToString(info->type)); + return -1; + } + } + break; + } + case VIR_DOMAIN_DEVICE_CHR: { + virDomainChrDefPtr chr = dev->data.chr; + + if (chr->deviceType == VIR_DOMAIN_CHR_DEVICE_TYPE_SERIAL) { + if (chr->source->type == VIR_DOMAIN_CHR_TYPE_TCP) { + if (!chr->source->data.tcp.listen) { + virReportError(VIR_ERR_XML_ERROR, + _("serial over tcp must be in listen mode")); + return -1; + } + } else if (chr->source->type != VIR_DOMAIN_CHR_TYPE_PTY && + chr->source->type != VIR_DOMAIN_CHR_TYPE_DEV && + chr->source->type != VIR_DOMAIN_CHR_TYPE_STDIO) { + virReportError(VIR_ERR_XML_ERROR, + _("serial type %s not supported"), + virDomainChrTypeToString(chr->source->type)); + return -1; + } + + if (chr->targetType != VIR_DOMAIN_CHR_SERIAL_TARGET_TYPE_NONE && + chr->targetType != VIR_DOMAIN_CHR_SERIAL_TARGET_TYPE_ISA) { + virReportError(VIR_ERR_XML_ERROR, + _("serial target type %s not supported"), + virDomainChrConsoleTargetTypeToString( + chr->targetType)); + return -1; + } + + if (chr->target.port > 1) { + virReportError(VIR_ERR_XML_ERROR, + _("serial port %d not supported"), + chr->target.port); + return -1; + } + } else if (chr->deviceType == VIR_DOMAIN_CHR_DEVICE_TYPE_CONSOLE) { + info = virDomainDeviceGetInfo(dev); + + if (chr->source->type != VIR_DOMAIN_CHR_TYPE_PTY && + chr->source->type != VIR_DOMAIN_CHR_TYPE_DEV && + chr->source->type != VIR_DOMAIN_CHR_TYPE_FILE && + chr->source->type != VIR_DOMAIN_CHR_TYPE_STDIO && + chr->source->type != VIR_DOMAIN_CHR_TYPE_UNIX) { + virReportError(VIR_ERR_XML_ERROR, + _("console type %s not supported"), + virDomainChrTypeToString(chr->source->type)); + return -1; + } + + /* + * VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_NONE will later be + * converted to VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_SERIAL. + * + * These types are considered as an implicit device and + * will be ignored. + * + * Only def->consoles[0] is allowed to be a serial port. + */ + if (chr->targetType != VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_NONE && + chr->targetType != VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_SERIAL && + chr->targetType != VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_VIRTIO) { + virReportError(VIR_ERR_XML_ERROR, + _("console target type %s not supported"), + virDomainChrConsoleTargetTypeToString( + chr->targetType)); + return -1; + } + + if (chr->targetType == VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_VIRTIO && + info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE && + info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI && + info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_SERIAL) { + virReportError(VIR_ERR_XML_ERROR, + _("virtio-console address type %s not supported"), + virDomainDeviceAddressTypeToString( + info->type)); + return -1; + } + } else { + virReportError(VIR_ERR_XML_ERROR, + _("chr device type %s not supported"), + virDomainChrDeviceTypeToString(chr->deviceType)); + return -1; + } + break; + } + case VIR_DOMAIN_DEVICE_INPUT: + case VIR_DOMAIN_DEVICE_WATCHDOG: + case VIR_DOMAIN_DEVICE_GRAPHICS: + case VIR_DOMAIN_DEVICE_RNG: + default: + virReportError(VIR_ERR_XML_ERROR, + _("device type %s not supported"), + virDomainDeviceTypeToString(dev->type)); + return -1; + } + + return 0; +} + +static int +acrnDomainDefAssignAddresses(virDomainDef *def, + unsigned int parseFlags G_GNUC_UNUSED, + void *opaque G_GNUC_UNUSED, + void *parseOpaque G_GNUC_UNUSED) +{ + return acrnDomainAssignAddresses(def); +} + +static virDomainDefParserConfig virAcrnDriverDomainDefParserConfig = { + .devicesPostParseCallback = acrnDomainDeviceDefPostParse, + .domainPostParseCallback = acrnDomainDefPostParse, + .assignAddressesCallback = acrnDomainDefAssignAddresses, +}; + +static void * +acrnDomainObjPrivateAlloc(void *opaque G_GNUC_UNUSED) +{ + acrnDomainObjPrivatePtr priv; + + if (VIR_ALLOC(priv) < 0) + return NULL; + + return priv; +} + +void +acrnDomainTtyCleanup(acrnDomainObjPrivatePtr priv) +{ + size_t i = priv->nttys; + + while (i--) { + VIR_FREE(priv->ttys[i].slave); + VIR_FORCE_CLOSE(priv->ttys[i].fd); + priv->ttys[i].fd = 0; + } + + priv->nttys = 0; +} + +static void +acrnDomainObjPrivateFree(void *data) +{ + /* priv is guaranteed non-NULL */ + acrnDomainObjPrivatePtr priv = data; + + acrnDomainTtyCleanup(priv); + virBitmapFree(priv->cpuAffinitySet); + VIR_FREE(priv); +} + +static virDomainXMLPrivateDataCallbacks virAcrnDriverPrivateDataCallbacks = { + .alloc = acrnDomainObjPrivateAlloc, + .free = acrnDomainObjPrivateFree, +}; + +virDomainXMLOptionPtr +virAcrnDriverCreateXMLConf(void) +{ + return virDomainXMLOptionNew(&virAcrnDriverDomainDefParserConfig, + &virAcrnDriverPrivateDataCallbacks, + NULL, NULL, NULL); +} diff --git a/src/acrn/acrn_domain.h b/src/acrn/acrn_domain.h new file mode 100644 index 0000000000..17641febcc --- /dev/null +++ b/src/acrn/acrn_domain.h @@ -0,0 +1,20 @@ +#ifndef __ACRN_DOMAIN_H__ +#define __ACRN_DOMAIN_H__ + +#include "domain_conf.h" + +typedef struct _acrnDomainObjPrivate acrnDomainObjPrivate; +typedef acrnDomainObjPrivate *acrnDomainObjPrivatePtr; +struct _acrnDomainObjPrivate { + unsigned char hvUUID[VIR_UUID_BUFLEN]; + virBitmapPtr cpuAffinitySet; + struct { + int fd; + char *slave; + } ttys[4]; + size_t nttys; +}; + +void acrnDomainTtyCleanup(acrnDomainObjPrivatePtr priv); +virDomainXMLOptionPtr virAcrnDriverCreateXMLConf(void); +#endif /* __ACRN_DOMAIN_H__ */ diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c new file mode 100644 index 0000000000..e519732c2f --- /dev/null +++ b/src/acrn/acrn_driver.c @@ -0,0 +1,2641 @@ +#include +#include +#include +#include +#include +#include +#include "configmake.h" +#include "datatypes.h" +#include "node_device_conf.h" +#include "virdomainobjlist.h" +#include "virerror.h" +#include "viralloc.h" +#include "virutil.h" +#include "cpu/cpu.h" +#include "virhostcpu.h" +#include "vircommand.h" +#include "virthread.h" +#include "virstring.h" +#include "virfile.h" +#include "virhostdev.h" +#include "virnodesuspend.h" +#include "virnetdevbridge.h" +#include "virnetdevtap.h" +#include "virfdstream.h" +#include "virlog.h" +#include "domain_event.h" +#include "acrn_common.h" +#include "acrn_driver.h" +#include "acrn_domain.h" + +#define VIR_FROM_THIS VIR_FROM_ACRN +#define ACRN_DM_PATH "/usr/bin/acrn-dm" +#define ACRN_CTL_PATH "/usr/bin/acrnctl" +#define ACRN_NAMESPACE_HREF "http://libvirt.org/schemas/domain/acrn/0.0" +#define ACRN_OFFLINE_PATH "/sys/class/vhm/acrn_vhm/offline_cpu" +#define SYSFS_CPU_PATH "/sys/devices/system/cpu" +#define ACRN_AUTOSTART_DIR SYSCONFDIR "/libvirt/acrn/autostart" +#define ACRN_CONFIG_DIR SYSCONFDIR "/libvirt/acrn" +#define ACRN_NET_GENERATED_TAP_PREFIX "tap" +#define ACRN_PI_VERSION (0x100) + +VIR_LOG_INIT("acrn.acrn_driver"); + +typedef struct _acrnConnect acrnConnect; +typedef struct _acrnConnect *acrnConnectPtr; +struct _acrnConnect { + virMutex lock; + virNodeInfo nodeInfo; + virDomainObjListPtr domains; + virCapsPtr caps; + virDomainXMLOptionPtr xmlopt; + virObjectEventStatePtr domainEventState; + virHostdevManagerPtr hostdevMgr; + acrnPlatformInfo pi; + size_t *vcpuAllocMap; +}; + +typedef struct _acrnDomainNamespaceDef acrnDomainNamespaceDef; +typedef acrnDomainNamespaceDef *acrnDomainNamespaceDefPtr; +struct _acrnDomainNamespaceDef { + size_t num_args; + char **args; +}; + +#define MAX_NUM_VMS (64) + +struct acrnVmList { + struct acrnVmEntry { + acrnVmCfg cfg; + int vcpu_num; + virBitmapPtr pcpus; + } vm[MAX_NUM_VMS]; + size_t size; +}; + +static acrnConnectPtr acrn_driver = NULL; + +static void +acrnDriverLock(acrnConnectPtr driver) +{ + virMutexLock(&driver->lock); +} + +static void +acrnDriverUnlock(acrnConnectPtr driver) +{ + virMutexUnlock(&driver->lock); +} + +/** + * Get a reference to the virCapsPtr instance for the + * driver. + * + * The caller must release the reference with virObjetUnref + * + * Returns: a reference to a virCapsPtr instance or NULL + */ +static virCapsPtr ATTRIBUTE_NONNULL(1) +acrnDriverGetCapabilities(acrnConnectPtr driver) +{ + return virObjectRef(driver->caps); +} + +static virDomainObjPtr +acrnDomObjFromDomain(virDomainPtr domain) +{ + virDomainObjPtr vm; + acrnConnectPtr privconn = domain->conn->privateData; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + vm = virDomainObjListFindByUUID(privconn->domains, domain->uuid); + + if (!vm) { + virUUIDFormat(domain->uuid, uuidstr); + virReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching uuid '%s' (%s)"), + uuidstr, domain->name); + return NULL; + } + + return vm; +} + +static struct acrnVmList * +acrnVmListNew(void) +{ + struct acrnVmList *list; + + if (VIR_ALLOC(list) < 0) + return NULL; + + return list; +} + +static void +acrnVmListFree(struct acrnVmList *list) +{ + size_t i; + + if (!list) + return; + + for (i = 0; i < list->size; i++) + virBitmapFree(list->vm[i].pcpus); + VIR_FREE(list); +} + +static int +acrnGetVhmFd(void) +{ + struct stat st; + int fd = -1; + + if (!stat("/dev/acrn_vhm", &st)) + fd = open("/dev/acrn_vhm", O_RDWR|O_CLOEXEC); + else if (!stat("/dev/acrn_hsm", &st)) + fd = open("/dev/acrn_hsm", O_RDWR|O_CLOEXEC); + + return fd; +} + +static int +acrnGetPlatformInfo(int fd, acrnPlatformInfoPtr pi) +{ + return ioctl(fd, IC_GET_PLATFORM_INFO, pi); +} + +static int +acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) +{ + acrnVmCfg vmcfg; + int fd, vcpu_num, pos, ret = -1; + uint8_t *p; + uint16_t i, j; + uint64_t pcpus; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + if (!pi || !vmList) + return 0; + + vmList->size = 0; + + if ((fd = acrnGetVhmFd()) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("acrnGetVhmFd failed")); + goto cleanup; + } + + /* get basic platform info first */ + if (!pi->vm_configs_addr) { + if (acrnGetPlatformInfo(fd, pi) < 0 || + !pi->cpu_num || !pi->max_vms || !pi->vm_config_entry_size) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("acrnGetPlatformInfo failed")); + goto cleanup; + } + + if (pi->version != ACRN_PI_VERSION) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("ACRN platform version mismatch: " + "got 0x%x, expecting 0x%x"), + pi->version, ACRN_PI_VERSION); + goto cleanup; + } + + if (!(pi->vm_configs_addr = (uint64_t)calloc( + pi->max_vms, + pi->vm_config_entry_size))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + } + + /* now get vm config */ + if (acrnGetPlatformInfo(fd, pi) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("acrnGetPlatformInfo failed")); + goto cleanup; + } + + for (i = 0, p = (uint8_t *)pi->vm_configs_addr; + i < pi->max_vms; + i++, p += pi->vm_config_entry_size) { + /* drop the hv-specific part of vmcfg */ + memcpy(&vmcfg, p, sizeof(vmcfg)); + + if (virUUIDIsValid(vmcfg.uuid)) { + if (vmList->size == G_N_ELEMENTS(vmList->vm)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("only %lu VMs are supported"), + G_N_ELEMENTS(vmList->vm)); + goto cleanup; + } + + if (!(pcpus = vmcfg.cpu_affinity)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("no pCPU in vm[%u]"), i); + goto cleanup; + } + + vcpu_num = __builtin_popcountl(pcpus); + + /* insertion sort based on vcpu_num */ + for (j = 0; j < vmList->size; j++) { + if (vcpu_num < vmList->vm[j].vcpu_num) + break; + } + + if (j < vmList->size) + memmove(&vmList->vm[j+1], &vmList->vm[j], + sizeof(vmList->vm[j]) * (vmList->size - j)); + + memcpy(&vmList->vm[j].cfg, &vmcfg, sizeof(vmcfg)); + + if (!(vmList->vm[j].pcpus = + virBitmapNew( + sizeof(vmcfg.cpu_affinity) * CHAR_BIT))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + + /* convert cpu_affinity to virBitmap */ + while ((pos = __builtin_ffsl(pcpus)) > 0) { + pos--; + + if (virBitmapSetBit(vmList->vm[j].pcpus, pos) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("virBitmapSetBit failed")); + goto cleanup; + } + pcpus &= ~(1ULL << pos); + } + + vmList->vm[j].vcpu_num = vcpu_num; + vmList->size++; + } + } + + for (i = 0; i < vmList->size; i++) + VIR_DEBUG("vm[%u] (%s): order: %d, uuid: %s, severity: 0x%x, " + "pCPU map: 0x%lx (%d vCPUs)", + i, vmList->vm[i].cfg.name, + vmList->vm[i].cfg.load_order, + virUUIDFormat(vmList->vm[i].cfg.uuid, uuidstr), + vmList->vm[i].cfg.severity, + vmList->vm[i].cfg.cpu_affinity, + vmList->vm[i].vcpu_num); + + ret = 0; + +cleanup: + if (fd >= 0) + close(fd); + return ret; +} + +struct acrnFindUUIDData { + const unsigned char *uuid; +}; + +static int +acrnFindHvUUID(virDomainObjPtr vm, void *opaque) +{ + struct acrnFindUUIDData *data = opaque; + acrnDomainObjPrivatePtr priv; + int ret = 0; + + virObjectLock(vm); + + priv = vm->privateData; + + if (!uuid_compare(priv->hvUUID, data->uuid)) + ret = -1; + + virObjectUnlock(vm); + return ret; +} + +/* + * This function must not be called with any virDomainObjPtr + * lock held, as it can attempt to hold any such lock in doms. + */ +static ssize_t +acrnAllocateVm(virDomainObjListPtr doms, virDomainDefPtr def, + acrnPlatformInfoPtr pi, struct acrnVmList *vmList, bool rtvm, + unsigned char *uuid) +{ + enum acrn_vm_severity severity; + struct acrnFindUUIDData data; + virBitmapPtr cpumask = NULL, testmask = NULL; + ssize_t i, start, candidate = -1; + size_t nvcpus, maxVcpusFit = 0; + char *maskstr = NULL; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + severity = (rtvm) ? SEVERITY_RTVM : SEVERITY_STANDARD_VM; + + if (def->cpumask) { + /* prepare a sanitized cpumask */ + if (!(cpumask = virBitmapNewCopy(def->cpumask))) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("virBitmapNewCopy failed")); + goto notfound; + } + + /* clamp cpumask to cpu_num */ + virBitmapShrink(cpumask, pi->cpu_num); + + if (!(testmask = virBitmapNew(virBitmapSize(cpumask)))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + goto notfound; + } + } + + /* determine where to begin the search, based on vcpu_num */ + for (i = 0; i < vmList->size; i++) { + if (def->maxvcpus <= vmList->vm[i].vcpu_num) + break; + } + + start = i; + + /* these VMs can fit maxvcpus */ + for (; i < vmList->size; i++) { + if (vmList->vm[i].cfg.load_order == POST_LAUNCHED_VM && + vmList->vm[i].cfg.severity == severity) { + data.uuid = vmList->vm[i].cfg.uuid; + + if (!virDomainObjListForEach(doms, false, acrnFindHvUUID, &data)) { + if (!cpumask) + goto done; + + if (virBitmapCopy(testmask, cpumask) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("virBitmapCopy failed")); + goto notfound; + } + + virBitmapIntersect(testmask, vmList->vm[i].pcpus); + nvcpus = virBitmapCountBits(testmask); + + if (nvcpus >= def->maxvcpus) + goto done; + + /* search for max fit */ + if (nvcpus > maxVcpusFit) { + maxVcpusFit = nvcpus; + candidate = i; + } + } + } + } + + i = start; + + /* just try to find the best VM available */ + while (i--) { + if (vmList->vm[i].cfg.load_order == POST_LAUNCHED_VM && + vmList->vm[i].cfg.severity == severity) { + data.uuid = vmList->vm[i].cfg.uuid; + + if (!virDomainObjListForEach(doms, false, acrnFindHvUUID, &data)) { + if (!cpumask) + goto done; + + if (virBitmapCopy(testmask, cpumask) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("virBitmapCopy failed")); + goto notfound; + } + + virBitmapIntersect(testmask, vmList->vm[i].pcpus); + nvcpus = virBitmapCountBits(testmask); + + /* search for max fit */ + if (nvcpus >= maxVcpusFit) { + maxVcpusFit = nvcpus; + candidate = i; + } + } + } + } + + if (maxVcpusFit > 0) { + /* max fit found */ + i = candidate; + if (virBitmapCopy(testmask, cpumask) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("virBitmapCopy failed")); + goto notfound; + } + virBitmapIntersect(testmask, vmList->vm[i].pcpus); + goto done; + } + +notfound: + i = -1; + + if (def->cpumask) + maskstr = virBitmapFormat(def->cpumask); + + virReportError(VIR_ERR_INTERNAL_ERROR, + _("no suitable vm found (%lu max vcpus, cpumask = %s)"), + def->maxvcpus, + maskstr ? maskstr : "auto"); + +done: + if (i >= 0) { + if (testmask) + maskstr = virBitmapFormat(testmask); + else + maskstr = virBitmapFormat(vmList->vm[i].pcpus); + + VIR_DEBUG("vm(%s) allocated: uuid = %s, " + "%lu max vcpus, %s cpumask = %s", + vmList->vm[i].cfg.name, + virUUIDFormat(vmList->vm[i].cfg.uuid, uuidstr), + def->maxvcpus, + testmask ? "allowed" : "auto", + maskstr ? maskstr : "n/a"); + uuid_copy(uuid, vmList->vm[i].cfg.uuid); + } + virBitmapFree(cpumask); + virBitmapFree(testmask); + if (maskstr) + VIR_FREE(maskstr); + return i; +} + +static ssize_t +acrnFindVm(struct acrnVmList *vmList, unsigned char *uuid) +{ + size_t i; + + for (i = 0; i < vmList->size; i++) + if (!uuid_compare(vmList->vm[i].cfg.uuid, uuid)) + return (ssize_t)i; + + return -1; +} + +static int +acrnAllocateVcpus(acrnPlatformInfoPtr pi, virBitmapPtr pcpus, bool rtvm, + size_t maxvcpus, size_t *allocMap, virBitmapPtr vcpus) +{ + ssize_t pos; + uint16_t totalCpus = pi->cpu_num; + + while (maxvcpus--) { + uint16_t minAllocated = USHRT_MAX; + uint16_t candidate = totalCpus; + + pos = -1; + + /* find a pCPU that is least occupied */ + while ((pos = virBitmapNextSetBit(pcpus, pos)) >= 0 && + pos < totalCpus) { + if (!virBitmapIsBitSet(vcpus, pos) && + allocMap[pos] < minAllocated) { + minAllocated = allocMap[pos]; + candidate = pos; + } + } + + /* all of the pCPUs in the VM have been allocated */ + if (candidate == totalCpus) + break; + + VIR_DEBUG("minAllocated = %u, candidate = %u", + minAllocated, candidate); + + if (virBitmapSetBit(vcpus, candidate) < 0 || + (rtvm && minAllocated > 0)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("vCPU placement failure")); + return -1; + } + } + + pos = -1; + + /* successful - update allocation map */ + while ((pos = virBitmapNextSetBit(vcpus, pos)) >= 0) { + allocMap[pos] += 1; + + VIR_DEBUG("pCPU[%ld]: %lu vCPU%s allocated", + pos, allocMap[pos], + (allocMap[pos] > 1) ? "s" : ""); + } + + return 0; +} + +static int +acrnFreeVcpus(virBitmapPtr vcpus, size_t *allocMap) +{ + ssize_t pos = -1; + int ret = 0; + + if (!vcpus || !allocMap) + return -1; + + /* update allocation map */ + while ((pos = virBitmapNextSetBit(vcpus, pos)) >= 0) { + if (allocMap[pos]) { + allocMap[pos] -= 1; + + VIR_DEBUG("pCPU[%ld]: %lu vCPU%s allocated", + pos, allocMap[pos], + (allocMap[pos] > 1) ? "s" : ""); + } else { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("vCPU allocation map error (bit %ld)"), + pos); + ret = -1; + } + } + + return ret; +} + +static int +acrnSetOnlineVcpus(virDomainDefPtr def, virBitmapPtr vcpus) +{ + return virDomainDefSetVcpus(def, virBitmapCountBits(vcpus)); +} + +static int +acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, + struct acrnVmEntry *entry, size_t *allocMap) +{ + virDomainDefPtr def; + virBitmapPtr allowedmask = NULL; + acrnDomainObjPrivatePtr priv; + int ret = -1; + + if (!vm || !(def = vm->def)) + return -1; + + priv = vm->privateData; + + if (def->cpumask) { + /* clamp cpumask to cpu_num */ + virBitmapShrink(def->cpumask, pi->cpu_num); + + if (!(allowedmask = virBitmapNewCopy(def->cpumask))) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("virBitmapNewCopy failed")); + goto cleanup; + } + + virBitmapIntersect(allowedmask, entry->pcpus); + + if (virBitmapIsAllClear(allowedmask)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("vm(%s) does not allow the given cpumask"), + entry->cfg.name); + goto cleanup; + } + } + + if (priv->cpuAffinitySet) + virBitmapFree(priv->cpuAffinitySet); + if (!(priv->cpuAffinitySet = virBitmapNew(pi->cpu_num))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + + /* vCPU placement */ + if (acrnAllocateVcpus(pi, + allowedmask ? allowedmask : entry->pcpus, + false, def->maxvcpus, allocMap, + priv->cpuAffinitySet) < 0) + goto cleanup; + + if (acrnSetOnlineVcpus(def, priv->cpuAffinitySet) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("acrnSetOnlineVcpus failed")); + acrnFreeVcpus(priv->cpuAffinitySet, allocMap); + goto cleanup; + } + + ret = 0; + +cleanup: + if (ret < 0 && priv->cpuAffinitySet) { + virBitmapFree(priv->cpuAffinitySet); + priv->cpuAffinitySet = NULL; + } + virBitmapFree(allowedmask); + return ret; +} + +static int +acrnCreateTapDev(virDomainNetDefPtr net, const unsigned char *uuid) +{ + int tapfd = -1, ret = -1; + + if (!net->ifname || + !STRPREFIX(net->ifname, ACRN_NET_GENERATED_TAP_PREFIX)) { + if (net->ifname) { + VIR_WARN("Tap name '%s' not supported", net->ifname); + VIR_FREE(net->ifname); + } + net->ifname = g_strdup(ACRN_NET_GENERATED_TAP_PREFIX "%d"); + } + + if (virNetDevTapCreateInBridgePort( + virDomainNetGetActualBridgeName(net), + &net->ifname, &net->mac, + uuid, NULL, &tapfd, 1, + virDomainNetGetActualVirtPortProfile(net), + virDomainNetGetActualVlan(net), + virDomainNetGetActualPortOptionsIsolated(net), + NULL, 0, NULL, + VIR_NETDEV_TAP_CREATE_IFUP | + VIR_NETDEV_TAP_CREATE_PERSIST) < 0) { + virReportError(VIR_WAR_NO_NETWORK, "%s", net->ifname); + goto cleanup; + } + + ret = 0; + +cleanup: + if (tapfd >= 0) + VIR_FORCE_CLOSE(tapfd); + return ret; +} + +static void +acrnNetCleanup(virDomainObjPtr vm) +{ + size_t i; + + for (i = 0; i < vm->def->nnets; i++) { + virDomainNetDefPtr net = vm->def->nets[i]; + virDomainNetType actualType = virDomainNetGetActualType(net); + + if (actualType == VIR_DOMAIN_NET_TYPE_BRIDGE) { + if (net->ifname) { + int retries = 5; + + ignore_value(virNetDevBridgeRemovePort( + virDomainNetGetActualBridgeName(net), + net->ifname)); + + /* + * FIXME + * There is currently no way to reliably know when the + * shutdown process is complete. + */ + while (virNetDevTapDelete(net->ifname, NULL) < 0 && retries--) + sleep(1); + } + } + } +} + +static int +acrnCreateTty(virDomainObjPtr vm, virDomainChrDefPtr chr) +{ + acrnDomainObjPrivatePtr priv = vm->privateData; + int ttyfd; + char *ttypath; + + if (priv->nttys == G_N_ELEMENTS(priv->ttys)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("too many ttys (max = %lu)"), + G_N_ELEMENTS(priv->ttys)); + return -1; + } + + if (virFileOpenTty(&ttyfd, &ttypath, 0) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("virFileOpenTty failed")); + return -1; + } + + priv->ttys[priv->nttys].slave = g_strdup(ttypath); + priv->ttys[priv->nttys].fd = ttyfd; + priv->nttys++; + + if (chr->source->data.file.path) + VIR_FREE(chr->source->data.file.path); + chr->source->data.file.path = ttypath; + + return 0; +} + +static void +acrnTtyCleanup(virDomainObjPtr vm) +{ + acrnDomainObjPrivatePtr priv = vm->privateData; + + acrnDomainTtyCleanup(priv); +} + +static void +acrnAddVirtioConsoleCmd(virBufferPtr buf, virDomainChrDefPtr chr) +{ + if (chr->deviceType != VIR_DOMAIN_CHR_DEVICE_TYPE_CONSOLE || + chr->targetType != VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_VIRTIO) + return; + + switch (chr->source->type) { + case VIR_DOMAIN_CHR_TYPE_PTY: + virBufferAddLit(buf, ",@pty:pty_port"); + break; + case VIR_DOMAIN_CHR_TYPE_DEV: + virBufferAsprintf(buf, ",@tty:tty_port=%s", + chr->source->data.file.path); + break; + case VIR_DOMAIN_CHR_TYPE_FILE: + virBufferAsprintf(buf, ",@file:file_port=%s", + chr->source->data.file.path); + break; + case VIR_DOMAIN_CHR_TYPE_STDIO: + virBufferAddLit(buf, ",@stdio:stdio_port"); + break; + case VIR_DOMAIN_CHR_TYPE_UNIX: + virBufferAsprintf(buf, ",socket:socket_file_name=%s:%s", + chr->source->data.nix.path, + chr->source->data.nix.listen ? + "server" : "client"); + break; + default: + return; + } +} + +struct acrnCmdDeviceData { + virDomainObjPtr vm; + virCommandPtr cmd; + bool lpc; +}; + +static int +acrnCommandAddDeviceArg(virDomainDefPtr def, + virDomainDeviceDefPtr dev, + virDomainDeviceInfoPtr info, + void *opaque) +{ + struct acrnCmdDeviceData *data = opaque; + virCommandPtr cmd = data->cmd; + + switch (dev->type) { + case VIR_DOMAIN_DEVICE_DISK: { + virDomainDiskDefPtr disk = dev->data.disk; + + if (disk->bus == VIR_DOMAIN_DISK_BUS_VIRTIO) { + /* + * VIR_DOMAIN_DISK_DEVICE_DISK && + * VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI + */ + virCommandAddArg(cmd, "-s"); + virCommandAddArgFormat(cmd, "%u:%u:%u,virtio-blk,%s", + info->addr.pci.bus, + info->addr.pci.slot, + info->addr.pci.function, + virDomainDiskGetSource(disk)); + } else { /* VIR_DOMAIN_DISK_BUS_SATA */ + size_t i; + + for (i = 0; i < def->ncontrollers; i++) { + virDomainControllerDefPtr ctrl = def->controllers[i]; + + if (ctrl->type == VIR_DOMAIN_CONTROLLER_TYPE_SATA && + ctrl->idx == disk->info.addr.drive.controller) { + virCommandAddArg(cmd, "-s"); + virCommandAddArgFormat(cmd, "%u:%u:%u,ahci-%s,%s", + ctrl->info.addr.pci.bus, + ctrl->info.addr.pci.slot, + ctrl->info.addr.pci.function, + (disk->device == + VIR_DOMAIN_DISK_DEVICE_DISK) ? + "hd" : "cd", + virDomainDiskGetSource(disk)); + /* a SATA controller can only have one disk attached */ + break; + } + } + } + break; + } + case VIR_DOMAIN_DEVICE_NET: { + virDomainNetDefPtr net = dev->data.net; + char macstr[VIR_MAC_STRING_BUFLEN]; + + if (net->type == VIR_DOMAIN_NET_TYPE_BRIDGE && + acrnCreateTapDev(net, def->uuid) < 0) + return -1; + + virCommandAddArg(cmd, "-s"); + virCommandAddArgFormat(cmd, "%u:%u:%u,virtio-net,%s,mac=%s", + info->addr.pci.bus, + info->addr.pci.slot, + info->addr.pci.function, + net->ifname, + virMacAddrFormat(&net->mac, macstr)); + break; + } + case VIR_DOMAIN_DEVICE_HOSTDEV: { + virDomainHostdevDefPtr hostdev = dev->data.hostdev; + virDomainHostdevSubsysPtr subsys = &hostdev->source.subsys; + + virCommandAddArg(cmd, "-s"); + + if (subsys->type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) { + virDomainHostdevSubsysUSBPtr usbsrc = &subsys->u.usb; + + if (!usbsrc->autoAddress) { + virReportError(VIR_ERR_NO_SOURCE, _("usb hostdev")); + return -1; + } + + virCommandAddArgFormat(cmd, "%u:%u:%u,passthru,%x/%x/0", + info->addr.pci.bus, + info->addr.pci.slot, + info->addr.pci.function, + usbsrc->bus, usbsrc->device); + } else { /* VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI */ + virDomainHostdevSubsysPCIPtr pcisrc = &subsys->u.pci; + + virCommandAddArgFormat(cmd, "%u:%u:%u,passthru,%x/%x/%x", + info->addr.pci.bus, + info->addr.pci.slot, + info->addr.pci.function, + pcisrc->addr.bus, + pcisrc->addr.slot, + pcisrc->addr.function); + } + break; + } + case VIR_DOMAIN_DEVICE_CONTROLLER: { + virDomainControllerDefPtr ctrl = dev->data.controller; + size_t i; + bool found = false; + virBuffer buf = VIR_BUFFER_INITIALIZER; + + /* PCI hostbridge is always included */ + if (ctrl->type == VIR_DOMAIN_CONTROLLER_TYPE_VIRTIO_SERIAL) { + for (i = 0; i < def->nconsoles; i++) { + virDomainChrDefPtr chr = def->consoles[i]; + + if (chr->deviceType == VIR_DOMAIN_CHR_DEVICE_TYPE_CONSOLE && + chr->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_SERIAL && + chr->info.addr.vioserial.controller == ctrl->idx) { + if (!found) { + virBufferAsprintf(&buf, "%u:%u:%u,virtio-console", + info->addr.pci.bus, + info->addr.pci.slot, + info->addr.pci.function); + found = true; + } + + acrnAddVirtioConsoleCmd(&buf, chr); + } + } + } + + if (found) { + virCommandAddArg(cmd, "-s"); + virCommandAddArgBuffer(cmd, &buf); + virBufferFreeAndReset(&buf); + } + break; + } + case VIR_DOMAIN_DEVICE_CHR: { + virDomainChrDefPtr chr = dev->data.chr; + + if (chr->deviceType == VIR_DOMAIN_CHR_DEVICE_TYPE_SERIAL) { + virBuffer buf = VIR_BUFFER_INITIALIZER; + + if (!data->lpc) { + virCommandAddArgList(cmd, "-s", "1:0,lpc", NULL); + data->lpc = true; + } + + virBufferAsprintf(&buf, "com%d,", chr->target.port + 1); + + switch (chr->source->type) { + case VIR_DOMAIN_CHR_TYPE_PTY: + if (acrnCreateTty(data->vm, chr) < 0) + return -1; + virBufferAsprintf(&buf, "%s", chr->source->data.file.path); + break; + case VIR_DOMAIN_CHR_TYPE_DEV: + virBufferAsprintf(&buf, "%s", chr->source->data.file.path); + break; + case VIR_DOMAIN_CHR_TYPE_STDIO: + virBufferAddLit(&buf, "stdio"); + break; + case VIR_DOMAIN_CHR_TYPE_TCP: { + unsigned int tcpport; + + if (virStrToLong_ui(chr->source->data.tcp.service, + NULL, 10, &tcpport) < 0) { + virBufferFreeAndReset(&buf); + virReportError(VIR_ERR_NO_SOURCE, + _("serial over tcp")); + return -1; + } + + virBufferAsprintf(&buf, "tcp:%u", tcpport); + break; + } + default: + virBufferFreeAndReset(&buf); + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("serial type %s"), + virDomainChrTypeToString(chr->source->type)); + return -1; + } + + virCommandAddArg(cmd, "-l"); + virCommandAddArgBuffer(cmd, &buf); + virBufferFreeAndReset(&buf); + } else { /* VIR_DOMAIN_CHR_DEVICE_TYPE_CONSOLE */ + /* may be an implicit serial device - ignore */ + if (chr->targetType == VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_NONE || + chr->targetType == VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_SERIAL) { + VIR_DEBUG("ignore implicit serial device"); + break; + } + + /* VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_VIRTIO */ + if (info->type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + virBuffer buf = VIR_BUFFER_INITIALIZER; + + virBufferAsprintf(&buf, "%u:%u:%u,virtio-console", + info->addr.pci.bus, + info->addr.pci.slot, + info->addr.pci.function); + acrnAddVirtioConsoleCmd(&buf, chr); + + virCommandAddArg(cmd, "-s"); + virCommandAddArgBuffer(cmd, &buf); + virBufferFreeAndReset(&buf); + } + /* + * VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_SERIAL was + * already dealt with when its controller was reached. + */ + } + break; + } + case VIR_DOMAIN_DEVICE_INPUT: + case VIR_DOMAIN_DEVICE_WATCHDOG: + case VIR_DOMAIN_DEVICE_GRAPHICS: + case VIR_DOMAIN_DEVICE_RNG: + default: + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("device type %s"), + virDomainDeviceTypeToString(dev->type)); + return -1; + } + + return 0; +} + +static virCommandPtr +acrnBuildStartCmd(virDomainObjPtr vm) +{ + virDomainDefPtr def; + virCommandPtr cmd; + acrnDomainObjPrivatePtr priv; + struct acrnCmdDeviceData data = { 0 }; + char *pcpus; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + if (!vm || !(def = vm->def)) + return NULL; + + if (!(cmd = virCommandNew(ACRN_DM_PATH))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + return NULL; + } + + priv = vm->privateData; + + /* ACPI */ + if (def->features[VIR_DOMAIN_FEATURE_ACPI] == VIR_TRISTATE_SWITCH_ON) + virCommandAddArg(cmd, "-A"); + + /* CPU */ + pcpus = virBitmapFormat(priv->cpuAffinitySet); + virCommandAddArgList(cmd, "--cpu_affinity", pcpus, NULL); + VIR_FREE(pcpus); + + /* Memory */ + virCommandAddArg(cmd, "-m"); + virCommandAddArgFormat(cmd, "%lluM", + VIR_DIV_UP(virDomainDefGetMemoryInitial(def), 1024)); + + /* UUID */ + if (virUUIDIsValid(priv->hvUUID)) { + virCommandAddArg(cmd, "-U"); + virCommandAddArg(cmd, virUUIDFormat(priv->hvUUID, uuidstr)); + } + + /* PCI hostbridge */ + virCommandAddArgList(cmd, "-s", "0:0,hostbridge", NULL); + + data.vm = vm; + data.cmd = cmd; + + /* Devices */ + if (virDomainDeviceInfoIterate(def, acrnCommandAddDeviceArg, &data)) { + virCommandFree(cmd); + return NULL; + } + + /* Bootloader */ + if (def->os.loader && def->os.loader->path) { + virBuffer buf = VIR_BUFFER_INITIALIZER; + + if (def->os.loader->readonly == VIR_TRISTATE_BOOL_NO) + virBufferAddLit(&buf, "w,"); + virBufferAdd(&buf, def->os.loader->path, -1); + + virCommandAddArg(cmd, "--ovmf"); + virCommandAddArgBuffer(cmd, &buf); + virBufferFreeAndReset(&buf); + } else if (def->os.kernel && def->os.cmdline) { + virCommandAddArg(cmd, "-k"); + virCommandAddArg(cmd, def->os.kernel); + virCommandAddArg(cmd, "-B"); + virCommandAddArg(cmd, def->os.cmdline); + + if (def->os.initrd) { + virCommandAddArg(cmd, "-r"); + virCommandAddArg(cmd, def->os.initrd); + } + } else { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("boot policy")); + } + + /* VM name */ + virCommandAddArg(cmd, def->name); + + return cmd; +} + +static int +acrnProcessStart(virDomainObjPtr vm) +{ + virCommandPtr cmd; + int ret = -1; + + if (!(cmd = acrnBuildStartCmd(vm))) + goto cleanup; + + virCommandDaemonize(cmd); + + VIR_DEBUG("Starting domain '%s'", vm->def->name); + + if (virCommandRun(cmd, NULL) < 0) + goto cleanup; + + /* XXX */ + if (sscanf(vm->def->name, "vm%d", &vm->def->id) != 1 && + sscanf(vm->def->name, "instance-%d", &vm->def->id) != 1) + vm->def->id = 0; + + virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_BOOTED); + ret = 0; + +cleanup: + virCommandFree(cmd); + if (ret < 0) { + acrnNetCleanup(vm); + acrnTtyCleanup(vm); + } + return ret; +} + +static virCommandPtr +acrnBuildStopCmd(virDomainDefPtr def) +{ + virCommandPtr cmd; + + if (!def) + return NULL; + + if (!(cmd = virCommandNewArgList(ACRN_CTL_PATH, "stop", "-f", + def->name, NULL))) + virReportError(VIR_ERR_NO_MEMORY, NULL); + + return cmd; +} + +static int +acrnProcessStop(virDomainObjPtr vm, int reason, size_t *allocMap) +{ + virDomainDefPtr def = vm->def; + virCommandPtr cmd; + acrnDomainObjPrivatePtr priv = vm->privateData; + int ret = -1; + + if (!(cmd = acrnBuildStopCmd(def))) + goto cleanup; + + VIR_DEBUG("Stopping domain '%s'", def->name); + + if (virCommandRun(cmd, NULL) < 0) + goto cleanup; + + /* clean up network interfaces */ + acrnNetCleanup(vm); + + /* clean up ttys */ + acrnTtyCleanup(vm); + + virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, reason); + + def->id = -1; + ret = 0; + +cleanup: + virCommandFree(cmd); + acrnFreeVcpus(priv->cpuAffinitySet, allocMap); + return ret; +} + +static virDomainPtr +acrnDomainLookupByUUID(virConnectPtr conn, + const unsigned char *uuid) +{ + acrnConnectPtr privconn = conn->privateData; + virDomainObjPtr vm; + virDomainPtr dom = NULL; + + vm = virDomainObjListFindByUUID(privconn->domains, uuid); + + if (!vm) { + char uuidstr[VIR_UUID_STRING_BUFLEN]; + virUUIDFormat(uuid, uuidstr); + virReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching uuid '%s'"), uuidstr); + goto cleanup; + } + + dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id); + +cleanup: + if (vm) + virObjectUnlock(vm); + return dom; +} + +static virDomainPtr +acrnDomainLookupByName(virConnectPtr conn, const char *name) +{ + acrnConnectPtr privconn = conn->privateData; + virDomainObjPtr vm; + virDomainPtr dom = NULL; + + vm = virDomainObjListFindByName(privconn->domains, name); + + if (!vm) { + virReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching name '%s'"), name); + goto cleanup; + } + + dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id); + +cleanup: + virDomainObjEndAPI(&vm); + return dom; +} + +static int +acrnDomainShutdown(virDomainPtr dom) +{ + acrnConnectPtr privconn = dom->conn->privateData; + virDomainObjPtr vm; + virObjectEventPtr event = NULL; + int ret = -1; + + acrnDriverLock(privconn); + + if (!(vm = acrnDomObjFromDomain(dom))) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("domain is not running")); + goto cleanup; + } + + if (acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, + privconn->vcpuAllocMap) < 0) + goto cleanup; + + if (!(event = virDomainEventLifecycleNewFromObj( + vm, + VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_SHUTDOWN))) + goto cleanup; + + ret = 0; + +cleanup: + if (vm) + virObjectUnlock(vm); + acrnDriverUnlock(privconn); + if (event) + virObjectEventStateQueue(privconn->domainEventState, event); + return ret; +} + +static int +acrnDomainDestroy(virDomainPtr dom) +{ + acrnConnectPtr privconn = dom->conn->privateData; + virDomainObjPtr vm; + virDomainState state; + virObjectEventPtr event = NULL; + int reason, ret = -1; + + acrnDriverLock(privconn); + + if (!(vm = acrnDomObjFromDomain(dom))) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("domain is not running")); + goto cleanup; + } + + state = virDomainObjGetState(vm, &reason); + + if (state == VIR_DOMAIN_SHUTOFF) { + if (reason != VIR_DOMAIN_SHUTOFF_DESTROYED) + virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, + VIR_DOMAIN_SHUTOFF_DESTROYED); + } else { + if (acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_DESTROYED, + privconn->vcpuAllocMap) < 0) + goto cleanup; + } + + event = virDomainEventLifecycleNewFromObj( + vm, + VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_DESTROYED); + + if (!vm->persistent && + (state != VIR_DOMAIN_SHUTOFF || + reason != VIR_DOMAIN_SHUTOFF_DESTROYED)) { + virDomainObjListRemove(privconn->domains, vm); + vm = NULL; + } + + if (!event) + goto cleanup; + + ret = 0; + +cleanup: + if (vm) + virObjectUnlock(vm); + acrnDriverUnlock(privconn); + if (event) + virObjectEventStateQueue(privconn->domainEventState, event); + return ret; +} + +static int +acrnDomainGetInfo(virDomainPtr dom, virDomainInfoPtr info) +{ + virDomainObjPtr vm; + int ret = -1; + + if (!(vm = acrnDomObjFromDomain(dom))) + goto cleanup; + + info->state = virDomainObjGetState(vm, NULL); + info->maxMem = virDomainDefGetMemoryTotal(vm->def); + info->nrVirtCpu = virDomainDefGetVcpus(vm->def); + ret = 0; + +cleanup: + if (vm) + virObjectUnlock(vm); + return ret; +} + +static int +acrnDomainGetState(virDomainPtr domain, + int *state, + int *reason, + unsigned int flags) +{ + virDomainObjPtr vm; + int ret = -1; + + virCheckFlags(0, -1); + + if (!(vm = acrnDomObjFromDomain(domain))) + goto cleanup; + + *state = virDomainObjGetState(vm, reason); + ret = 0; + +cleanup: + if (vm) + virObjectUnlock(vm); + return ret; +} + +static int +acrnDomainGetVcpus(virDomainPtr domain, + virVcpuInfoPtr info, + int maxinfo, + unsigned char *cpumaps, + int maplen) +{ + virDomainObjPtr vm; + virDomainDefPtr def; + acrnDomainObjPrivatePtr priv; + struct timeval tv; + virBitmapPtr cpumap = NULL; + unsigned long long statbase; + int i, ret = -1; + ssize_t pos; + + if (!(vm = acrnDomObjFromDomain(domain))) + return -1; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("cannot list vcpus for an inactive domain")); + goto cleanup; + } + + if (!(def = vm->def) || !(priv = vm->privateData)) + goto cleanup; + + if (gettimeofday(&tv, NULL) < 0) { + virReportSystemError(errno, + "%s", _("getting time of day")); + goto cleanup; + } + + statbase = (tv.tv_sec * 1000UL * 1000UL) + tv.tv_usec; + + /* clamp to actual number of vcpus */ + if (maxinfo > virDomainDefGetVcpus(vm->def)) + maxinfo = virDomainDefGetVcpus(vm->def); + + memset(info, 0, sizeof(*info) * maxinfo); + + if (cpumaps) { + memset(cpumaps, 0, maxinfo * maplen); + + if (!(cpumap = virBitmapNew(maplen * CHAR_BIT))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + } + + if (!priv->cpuAffinitySet) { + virReportError(VIR_ERR_INTERNAL_ERROR, _("cpumask missing")); + goto cleanup; + } + + for (i = 0, pos = -1; i < maxinfo; i++) { + virDomainVcpuDefPtr vcpu = virDomainDefGetVcpu(def, i); + + if (!vcpu->online) + continue; + + if ((pos = virBitmapNextSetBit(priv->cpuAffinitySet, pos)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("cpu missing in cpumask")); + goto cleanup; + } + + if (cpumaps) { + virBitmapClearAll(cpumap); + + if (virBitmapSetBit(cpumap, pos) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("failed to set bit %ld in cpumap"), pos); + goto cleanup; + } + + virBitmapToDataBuf(cpumap, VIR_GET_CPUMAP(cpumaps, maplen, i), + maplen); + } + + info[i].number = i; + info[i].state = VIR_VCPU_RUNNING; + info[i].cpu = pos; + + /* FIXME fake an increasing cpu time value */ + info[i].cpuTime = statbase / 10; + } + + ret = maxinfo; + +cleanup: + if (vm) + virObjectUnlock(vm); + virBitmapFree(cpumap); + return ret; +} + +static char * +acrnDomainGetXMLDesc(virDomainPtr domain, unsigned int flags) +{ + acrnConnectPtr privconn = domain->conn->privateData; + virDomainObjPtr vm; + char *ret = NULL; + + if (!(vm = acrnDomObjFromDomain(domain))) + goto cleanup; + + ret = virDomainDefFormat(vm->def, privconn->xmlopt, + virDomainDefFormatConvertXMLFlags(flags)); + +cleanup: + if (vm) + virObjectUnlock(vm); + return ret; +} + +static virDomainPtr +acrnDomainCreateXML(virConnectPtr conn, + const char *xml, + unsigned int flags) +{ + acrnConnectPtr privconn = conn->privateData; + struct acrnVmList *vmList = NULL; + acrnDomainObjPrivatePtr priv; + virDomainDefPtr def; + virDomainObjPtr vm = NULL; + virObjectEventPtr event = NULL; + virDomainPtr dom = NULL; + ssize_t idx; + unsigned int parse_flags = VIR_DOMAIN_DEF_PARSE_INACTIVE; + unsigned char hvUUID[VIR_UUID_BUFLEN]; + + /* VIR_DOMAIN_START_AUTODESTROY is not supported yet */ + virCheckFlags(VIR_DOMAIN_START_VALIDATE, NULL); + + if (flags & VIR_DOMAIN_START_VALIDATE) + parse_flags |= VIR_DOMAIN_DEF_PARSE_VALIDATE_SCHEMA; + + if (!(def = virDomainDefParseString(xml, privconn->xmlopt, + NULL, parse_flags))) + goto cleanup_nolock; + + if (!(vmList = acrnVmListNew())) + goto cleanup_nolock; + + acrnDriverLock(privconn); + + /* retrieve current platform info */ + if (acrnGetPlatform(&privconn->pi, vmList) < 0) + goto cleanup; + + /* get hv UUID for the allocated VM */ + if ((idx = acrnAllocateVm(privconn->domains, def, &privconn->pi, vmList, + false, hvUUID)) < 0) + goto cleanup; + + if (!(vm = virDomainObjListAdd(privconn->domains, def, + privconn->xmlopt, + VIR_DOMAIN_OBJ_LIST_ADD_LIVE | + VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE, NULL))) + goto cleanup; + + priv = vm->privateData; + uuid_copy(priv->hvUUID, hvUUID); + + def = NULL; + + if (acrnProcessPrepareDomain(vm, &privconn->pi, &vmList->vm[idx], + privconn->vcpuAllocMap) < 0) + goto cleanup; + + if (acrnProcessStart(vm) < 0) { + acrnFreeVcpus(priv->cpuAffinitySet, privconn->vcpuAllocMap); + goto cleanup; + } + + if (!(event = virDomainEventLifecycleNewFromObj( + vm, + VIR_DOMAIN_EVENT_STARTED, + VIR_DOMAIN_EVENT_STARTED_BOOTED))) { + acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_DESTROYED, + privconn->vcpuAllocMap); + goto cleanup; + } + + dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id); + +cleanup: + if (vm) { + if (!dom && !vm->persistent) + /* if domain is not persistent, remove its data */ + virDomainObjListRemove(privconn->domains, vm); + else + virObjectUnlock(vm); + } + acrnDriverUnlock(privconn); +cleanup_nolock: + virDomainDefFree(def); + if (event) + virObjectEventStateQueue(privconn->domainEventState, event); + if (vmList) + acrnVmListFree(vmList); + return dom; +} + +static int +acrnDomainCreateWithFlags(virDomainPtr domain, unsigned int flags) +{ + acrnConnectPtr privconn = domain->conn->privateData; + struct acrnVmList *vmList; + acrnDomainObjPrivatePtr priv; + virDomainObjPtr vm = NULL; + virObjectEventPtr event = NULL; + ssize_t idx; + int ret = -1; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + /* VIR_DOMAIN_START_AUTODESTROY is not supported yet */ + virCheckFlags(0, -1); + + if (!(vmList = acrnVmListNew())) + return -1; + + acrnDriverLock(privconn); + + /* retrieve current platform info */ + if (acrnGetPlatform(&privconn->pi, vmList) < 0) + goto cleanup; + + if (!(vm = acrnDomObjFromDomain(domain))) + goto cleanup; + + if (virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("domain is already running")); + goto cleanup; + } + + priv = vm->privateData; + + /* find the allocated VM */ + if ((idx = acrnFindVm(vmList, priv->hvUUID)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("vm(%s) not found"), + virUUIDFormat(priv->hvUUID, uuidstr)); + goto cleanup; + } + + if (acrnProcessPrepareDomain(vm, &privconn->pi, &vmList->vm[idx], + privconn->vcpuAllocMap) < 0) + goto cleanup; + + if (acrnProcessStart(vm) < 0) { + /* domain must be persistent */ + acrnFreeVcpus(priv->cpuAffinitySet, privconn->vcpuAllocMap); + goto cleanup; + } + + if (!(event = virDomainEventLifecycleNewFromObj( + vm, + VIR_DOMAIN_EVENT_STARTED, + VIR_DOMAIN_EVENT_STARTED_BOOTED))) { + /* domain must be persistent */ + acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_DESTROYED, + privconn->vcpuAllocMap); + goto cleanup; + } + + ret = 0; + +cleanup: + if (vm) + virObjectUnlock(vm); + acrnDriverUnlock(privconn); + if (event) + virObjectEventStateQueue(privconn->domainEventState, event); + acrnVmListFree(vmList); + return ret; +} + +static int +acrnDomainCreate(virDomainPtr domain) +{ + return acrnDomainCreateWithFlags(domain, 0); +} + +static virDomainPtr +acrnDomainDefineXMLFlags(virConnectPtr conn, const char *xml, + unsigned int flags) +{ + acrnConnectPtr privconn = conn->privateData; + struct acrnVmList *vmList = NULL; + acrnDomainObjPrivatePtr priv; + virDomainDefPtr def = NULL, oldDef = NULL; + virDomainObjPtr vm = NULL; + virObjectEventPtr event = NULL; + virDomainPtr dom = NULL; + unsigned int parse_flags = VIR_DOMAIN_DEF_PARSE_INACTIVE; + unsigned char hvUUID[VIR_UUID_BUFLEN]; + + virCheckFlags(VIR_DOMAIN_DEFINE_VALIDATE, NULL); + + if (flags & VIR_DOMAIN_DEFINE_VALIDATE) + parse_flags |= VIR_DOMAIN_DEF_PARSE_VALIDATE_SCHEMA; + + if (!(def = virDomainDefParseString(xml, privconn->xmlopt, + NULL, parse_flags))) + goto cleanup_nolock; + + if (virXMLCheckIllegalChars("name", def->name, "\n") < 0) + goto cleanup_nolock; + + if (!(vmList = acrnVmListNew())) + goto cleanup_nolock; + + acrnDriverLock(privconn); + + /* retrieve current platform info */ + if (acrnGetPlatform(&privconn->pi, vmList) < 0) + goto cleanup; + + /* get hv UUID for the allocated VM */ + if (acrnAllocateVm(privconn->domains, def, &privconn->pi, vmList, + false, hvUUID) < 0) + goto cleanup; + + if (!(vm = virDomainObjListAdd(privconn->domains, def, + privconn->xmlopt, + 0, &oldDef))) + goto cleanup; + + vm->persistent = 1; + priv = vm->privateData; + uuid_copy(priv->hvUUID, hvUUID); + + def = NULL; + + if (virDomainDefSave(vm->newDef ? vm->newDef : vm->def, + privconn->xmlopt, ACRN_CONFIG_DIR) < 0) + goto cleanup; + + if (!(event = virDomainEventLifecycleNewFromObj( + vm, + VIR_DOMAIN_EVENT_DEFINED, + !oldDef ? + VIR_DOMAIN_EVENT_DEFINED_ADDED : + VIR_DOMAIN_EVENT_DEFINED_UPDATED))) { + virDomainDeleteConfig(ACRN_CONFIG_DIR, + ACRN_AUTOSTART_DIR, + vm); + goto cleanup; + } + + dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id); + +cleanup: + if (vm) { + if (!dom) + virDomainObjListRemove(privconn->domains, vm); + else + virObjectUnlock(vm); + } + acrnDriverUnlock(privconn); + virDomainDefFree(oldDef); +cleanup_nolock: + virDomainDefFree(def); + if (event) + virObjectEventStateQueue(privconn->domainEventState, event); + if (vmList) + acrnVmListFree(vmList); + return dom; +} + +static virDomainPtr +acrnDomainDefineXML(virConnectPtr conn, const char *xml) +{ + return acrnDomainDefineXMLFlags(conn, xml, 0); +} + +static int +acrnDomainUndefineFlags(virDomainPtr domain, unsigned int flags) +{ + acrnConnectPtr privconn = domain->conn->privateData; + virObjectEventPtr event = NULL; + virDomainObjPtr vm; + int ret = -1; + + virCheckFlags(0, -1); + + if (!(vm = acrnDomObjFromDomain(domain))) + goto cleanup; + + if (!vm->persistent) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("cannot undefine transient domain")); + goto cleanup; + } + + if (virDomainDeleteConfig(ACRN_CONFIG_DIR, + ACRN_AUTOSTART_DIR, + vm) < 0) + goto cleanup; + + event = virDomainEventLifecycleNewFromObj( + vm, + VIR_DOMAIN_EVENT_UNDEFINED, + VIR_DOMAIN_EVENT_UNDEFINED_REMOVED); + + if (virDomainObjIsActive(vm)) { + vm->persistent = 0; + } else { + virDomainObjListRemove(privconn->domains, vm); + vm = NULL; + } + + if (!event) + goto cleanup; + + ret = 0; + +cleanup: + if (vm) + virObjectUnlock(vm); + if (event) + virObjectEventStateQueue(privconn->domainEventState, event); + return ret; +} + +static int +acrnDomainUndefine(virDomainPtr domain) +{ + return acrnDomainUndefineFlags(domain, 0); +} + +static int +acrnDomainMemoryStats(virDomainPtr dom, + virDomainMemoryStatPtr stats, + unsigned int nr_stats, + unsigned int flags) +{ + virDomainObjPtr vm; + int ret = -1; + + virCheckFlags(0, -1); + + if (!(vm = acrnDomObjFromDomain(dom))) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("domain is not active")); + goto cleanup; + } + + ret = 0; + + if (ret < nr_stats) { + stats[ret].tag = VIR_DOMAIN_MEMORY_STAT_RSS; + stats[ret].val = virDomainDefGetMemoryInitial(vm->def); + ret++; + } + +cleanup: + if (vm) + virObjectUnlock(vm); + return ret; +} + +static int +acrnDomainIsActive(virDomainPtr domain) +{ + virDomainObjPtr obj; + int ret = -1; + + if (!(obj = acrnDomObjFromDomain(domain))) + goto cleanup; + + ret = virDomainObjIsActive(obj); + +cleanup: + if (obj) + virObjectUnlock(obj); + return ret; +} + +static int +acrnDomainOpenConsole(virDomainPtr dom, + const char *dev_name, + virStreamPtr st, + unsigned int flags) +{ + virDomainObjPtr vm; + virDomainChrDefPtr chr = NULL; + acrnDomainObjPrivatePtr priv; + size_t i; + int dupfd, ret = -1; + + virCheckFlags(0, -1); + + if (!(vm = acrnDomObjFromDomain(dom))) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("domain is not running")); + goto cleanup; + } + + if (vm->def->nserials && + vm->def->serials[0]->source->type == VIR_DOMAIN_CHR_TYPE_PTY) + chr = vm->def->serials[0]; + else if (vm->def->nconsoles) + chr = vm->def->consoles[0]; + + if (!chr) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("cannot find character device %s"), + NULLSTR(dev_name)); + goto cleanup; + } + + if (chr->source->type != VIR_DOMAIN_CHR_TYPE_PTY) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("character device %s is not using a PTY"), + dev_name ? dev_name : NULLSTR(chr->info.alias)); + goto cleanup; + } + + priv = vm->privateData; + + for (i = 0; i < priv->nttys; i++) { + if (priv->ttys[i].slave && + chr->source->data.file.path && + STREQ(priv->ttys[i].slave, chr->source->data.file.path)) + break; + } + + if (i == priv->nttys) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("cannot find character device %s"), + NULLSTR(dev_name)); + goto cleanup; + } + + /* dup the master's fd so it can be closed by the caller */ + if ((dupfd = dup(priv->ttys[i].fd)) < 0) { + virReportSystemError(errno, "%s", _("dup")); + goto cleanup; + } + + if (virFDStreamOpen(st, dupfd) < 0) { + VIR_FORCE_CLOSE(dupfd); + goto cleanup; + } + + ret = 0; + +cleanup: + if (vm) + virObjectUnlock(vm); + return ret; +} + +static virDrvOpenStatus +acrnConnectOpen(virConnectPtr conn, + virConnectAuthPtr auth G_GNUC_UNUSED, + virConfPtr conf G_GNUC_UNUSED, + unsigned int flags) +{ + virCheckFlags(VIR_CONNECT_RO, VIR_DRV_OPEN_ERROR); + + if (conn->uri == NULL) { + if (acrn_driver == NULL) + return VIR_DRV_OPEN_DECLINED; + + if (!(conn->uri = virURIParse("acrn:///system"))) + return VIR_DRV_OPEN_ERROR; + } else { + if (STRNEQ_NULLABLE(conn->uri->scheme, "acrn")) + return VIR_DRV_OPEN_DECLINED; + + if (conn->uri->server) + return VIR_DRV_OPEN_DECLINED; + + if (STRNEQ_NULLABLE(conn->uri->path, "/system")) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unexpected ACRN URI path '%s', try acrn:///system"), + conn->uri->path ? conn->uri->path : "NULL"); + return VIR_DRV_OPEN_ERROR; + } + + if (acrn_driver == NULL) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("ACRN state driver is not active")); + return VIR_DRV_OPEN_ERROR; + } + } + + conn->privateData = acrn_driver; + + return VIR_DRV_OPEN_SUCCESS; +} + +static int +acrnConnectClose(virConnectPtr conn) +{ + /* autodestroy is not supported yet */ + conn->privateData = NULL; + return 0; +} + +static const char * +acrnConnectGetType(virConnectPtr conn G_GNUC_UNUSED) +{ + return "ACRN"; +} + +static int +acrnConnectGetVersion(virConnectPtr conn G_GNUC_UNUSED, + unsigned long *version) +{ + virCommandPtr cmd; + char *verstr = NULL; + const char *dmstr = "DM version is: "; + int ret = -1; + + if (!(cmd = virCommandNewArgList(ACRN_DM_PATH, "-v", NULL))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + + virCommandSetOutputBuffer(cmd, &verstr); + + if (virCommandRun(cmd, NULL) < 0) + goto cleanup; + + if (!(dmstr = STRSKIP(verstr, dmstr))) + goto cleanup; + + if (virParseVersionString(dmstr, version, true) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("unknown release: %s"), dmstr); + goto cleanup; + } + + ret = 0; + +cleanup: + if (verstr) + VIR_FREE(verstr); + virCommandFree(cmd); + return ret; +} + +static char * +acrnConnectGetHostname(virConnectPtr conn G_GNUC_UNUSED) +{ + return virGetHostname(); +} + +static char * +acrnConnectGetCapabilities(virConnectPtr conn) +{ + acrnConnectPtr privconn = conn->privateData; + virCapsPtr caps; + char *xml; + + if (!(caps = acrnDriverGetCapabilities(privconn))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("unable to get capabilities")); + return NULL; + } + + xml = virCapabilitiesFormatXML(caps); + virObjectUnref(caps); + return xml; +} + +static int +acrnConnectListAllDomains(virConnectPtr conn, + virDomainPtr **domains, + unsigned int flags) +{ + acrnConnectPtr privconn = conn->privateData; + + virCheckFlags(VIR_CONNECT_LIST_DOMAINS_FILTERS_ALL, -1); + + return virDomainObjListExport(privconn->domains, conn, domains, NULL, + flags); +} + +static char * +acrnConnectBaselineCPU(virConnectPtr conn G_GNUC_UNUSED, + const char **xmlCPUs, + unsigned int ncpus, + unsigned int flags) +{ + virCPUDefPtr *cpus; + virCPUDefPtr cpu = NULL; + char *cpustr = NULL; + + virCheckFlags(VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES | + VIR_CONNECT_BASELINE_CPU_MIGRATABLE, NULL); + + if (!(cpus = virCPUDefListParse(xmlCPUs, ncpus, VIR_CPU_TYPE_HOST))) + goto cleanup; + + if (!(cpu = + virCPUBaseline(VIR_ARCH_NONE, cpus, ncpus, NULL, NULL, + !!(flags & VIR_CONNECT_BASELINE_CPU_MIGRATABLE)))) + goto cleanup; + + if ((flags & VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES) && + virCPUExpandFeatures(cpus[0]->arch, cpu) < 0) + goto cleanup; + + cpustr = virCPUDefFormat(cpu, NULL); + +cleanup: + virCPUDefFree(cpu); + virCPUDefListFree(cpus); + return cpustr; +} + +static int +acrnConnectDomainEventRegisterAny(virConnectPtr conn, + virDomainPtr dom, + int eventID, + virConnectDomainEventGenericCallback callback, + void *opaque, + virFreeCallback freecb) +{ + acrnConnectPtr privconn = conn->privateData; + int ret; + + if (virDomainEventStateRegisterID(conn, + privconn->domainEventState, + dom, eventID, + callback, opaque, freecb, &ret) < 0) + ret = -1; + + return ret; +} + +static int +acrnConnectDomainEventDeregisterAny(virConnectPtr conn, + int callbackID) +{ + acrnConnectPtr privconn = conn->privateData; + + if (virObjectEventStateDeregisterID(conn, + privconn->domainEventState, + callbackID, true) < 0) + return -1; + + return 0; +} + +static int +acrnNodeGetInfo(virConnectPtr conn, + virNodeInfoPtr info) +{ + acrnConnectPtr privconn = conn->privateData; + + memcpy(info, &privconn->nodeInfo, sizeof(*info)); + return 0; +} + +static int +acrnNodeDeviceGetPCIInfo(virNodeDeviceDefPtr def, + unsigned *domain, + unsigned *bus, + unsigned *slot, + unsigned *function) +{ + virNodeDevCapsDefPtr cap = def->caps; + + while (cap) { + if (cap->data.type == VIR_NODE_DEV_CAP_PCI_DEV) { + *domain = cap->data.pci_dev.domain; + *bus = cap->data.pci_dev.bus; + *slot = cap->data.pci_dev.slot; + *function = cap->data.pci_dev.function; + break; + } + + cap = cap->next; + } + + if (!cap) { + virReportError(VIR_ERR_INVALID_ARG, + _("device %s is not a PCI device"), def->name); + return -1; + } + + return 0; +} + +static int +acrnNodeDeviceDetachFlags(virNodeDevicePtr dev, + const char *driverName, + unsigned int flags) +{ + char *xml; + virNodeDeviceDefPtr def = NULL; + virPCIDevicePtr pci = NULL; + acrnConnectPtr privconn = dev->conn->privateData; + virHostdevManagerPtr hostdev_mgr = privconn->hostdevMgr; + unsigned domain = 0, bus = 0, slot = 0, function = 0; + int ret = -1; + + virCheckFlags(0, -1); + + if (!(xml = virNodeDeviceGetXMLDesc(dev, 0))) + goto cleanup; + + if (!(def = virNodeDeviceDefParseString(xml, EXISTING_DEVICE, NULL))) + goto cleanup; + + if (acrnNodeDeviceGetPCIInfo(def, &domain, &bus, &slot, &function) < 0) + goto cleanup; + + if (!(pci = virPCIDeviceNew(domain, bus, slot, function))) + goto cleanup; + + /* use the pci-stub driver */ + if (!driverName || STREQ(driverName, "kvm")) { + virPCIDeviceSetStubDriver(pci, VIR_PCI_STUB_DRIVER_KVM); + } else { + virReportError(VIR_ERR_INVALID_ARG, + _("unsupported driver name '%s'"), driverName); + goto cleanup; + } + + if (virHostdevPCINodeDeviceDetach(hostdev_mgr, pci) < 0) + goto cleanup; + + ret = 0; + +cleanup: + virPCIDeviceFree(pci); + virNodeDeviceDefFree(def); + if (xml) + VIR_FREE(xml); + return ret; +} + +static int +acrnNodeDeviceDettach(virNodeDevicePtr dev) +{ + return acrnNodeDeviceDetachFlags(dev, NULL, 0); +} + +static int +acrnNodeDeviceReAttach(virNodeDevicePtr dev) +{ + char *xml; + virNodeDeviceDefPtr def = NULL; + virPCIDevicePtr pci = NULL; + acrnConnectPtr privconn = dev->conn->privateData; + virHostdevManagerPtr hostdev_mgr = privconn->hostdevMgr; + unsigned domain = 0, bus = 0, slot = 0, function = 0; + int ret = -1; + + if (!(xml = virNodeDeviceGetXMLDesc(dev, 0))) + goto cleanup; + + if (!(def = virNodeDeviceDefParseString(xml, EXISTING_DEVICE, NULL))) + goto cleanup; + + if (acrnNodeDeviceGetPCIInfo(def, &domain, &bus, &slot, &function) < 0) + goto cleanup; + + if (!(pci = virPCIDeviceNew(domain, bus, slot, function))) + goto cleanup; + + if (virHostdevPCINodeDeviceReAttach(hostdev_mgr, pci) < 0) + goto cleanup; + + ret = 0; + +cleanup: + virPCIDeviceFree(pci); + virNodeDeviceDefFree(def); + if (xml) + VIR_FREE(xml); + return ret; +} + +static int +acrnNodeGetCPUMap(virConnectPtr conn, + unsigned char **cpumap, + unsigned int *online, + unsigned int flags) +{ + acrnConnectPtr privconn = conn->privateData; + acrnPlatformInfoPtr pi = &privconn->pi; + struct acrnVmList *list; + virBitmapPtr cpus = NULL; + size_t i; + int dummy, ret = -1; + + virCheckFlags(0, -1); + + if (!(list = acrnVmListNew())) + return -1; + + if (acrnGetPlatform(pi, list) < 0) + goto cleanup; + + /* + * Mark pCPUs available to the SOS (online) or UOS + * (present in the pcpus bitmap). + */ + if (!(cpus = virHostCPUGetOnlineBitmap())) + goto cleanup; + + for (i = 0; i < list->size; i++) { + if (list->vm[i].cfg.load_order == POST_LAUNCHED_VM) { + ssize_t pos = -1; + + while ((pos = virBitmapNextSetBit(list->vm[i].pcpus, pos)) >= 0) { + if (virBitmapSetBitExpand(cpus, pos) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("virBitmapSetBitExpand failed")); + goto cleanup; + } + } + } + } + + if (cpumap && virBitmapToData(cpus, cpumap, &dummy) < 0) + goto cleanup; + + if (online) + *online = virBitmapCountBits(cpus); + + ret = pi->cpu_num; + +cleanup: + if (ret < 0 && cpumap && *cpumap) + VIR_FREE(*cpumap); + virBitmapFree(cpus); + acrnVmListFree(list); + return ret; +} + +static int +acrnStateCleanup(void) +{ + VIR_DEBUG("ACRN state cleanup"); + + if (!acrn_driver) + return -1; + + virObjectUnref(acrn_driver->hostdevMgr); + virObjectUnref(acrn_driver->domainEventState); + virObjectUnref(acrn_driver->xmlopt); + virObjectUnref(acrn_driver->caps); + virObjectUnref(acrn_driver->domains); + if (acrn_driver->pi.vm_configs_addr) { + void *p = (void *)acrn_driver->pi.vm_configs_addr; + VIR_FREE(p); + } + if (acrn_driver->vcpuAllocMap) + VIR_FREE(acrn_driver->vcpuAllocMap); + virMutexDestroy(&acrn_driver->lock); + VIR_FREE(acrn_driver); + + return 0; +} + +static virCapsPtr +virAcrnCapsBuild(void) +{ + virCapsPtr caps; + virCapsGuestPtr guest; + + if (!(caps = virCapabilitiesNew(virArchFromHost(), false, false))) + return NULL; + + if (!(guest = virCapabilitiesAddGuest(caps, + VIR_DOMAIN_OSTYPE_HVM, + VIR_ARCH_X86_64, "acrn-dm", + NULL, 0, NULL))) + goto error; + + if (!virCapabilitiesAddGuestDomain(guest, + VIR_DOMAIN_VIRT_ACRN, + NULL, NULL, 0, NULL)) + goto error; + +#if 0 + if (virCapabilitiesSetNetPrefix(caps, ACRN_NET_GENERATED_TAP_PREFIX) < 0) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + goto error; + } +#endif + + if (!(caps->host.numa = virCapabilitiesHostNUMANewHost())) + goto error; + + if (virCapabilitiesInitCaches(caps) < 0) + VIR_WARN("Failed to get host CPU cache info"); + + if (!(caps->host.cpu = virCPUProbeHost(caps->host.arch))) + VIR_WARN("Failed to get host CPU"); + + /* add the power management features of the host */ + if (virNodeSuspendGetTargetMask(&caps->host.powerMgmt) < 0) + VIR_WARN("Failed to get host power management capabilities"); + + /* add huge pages info */ + if (virCapabilitiesInitPages(caps) < 0) + VIR_WARN("Failed to get pages info"); + + return caps; + +error: + virObjectUnref(caps); + return NULL; +} + +/* + * Vacate SOS CPUs for UOS vCPU allocation. + */ +static int +acrnOfflineCpus(int nprocs, virBitmapPtr pcpus, size_t *allocMap) +{ + ssize_t i = -1; + int fd; + char path[128], chr, online; + ssize_t rc; + + while ((i = virBitmapNextSetBit(pcpus, i)) >= 0 && i < nprocs) { + /* cpu0 can't be offlined */ + if (i == 0) + continue; + + snprintf(path, sizeof(path), "%s/cpu%ld/online", SYSFS_CPU_PATH, i); + + if ((fd = open(path, O_RDWR)) < 0) { + virReportError(VIR_ERR_OPEN_FAILED, _("%s"), path); + return -1; + } + + chr = '0'; + + do { + if (pwrite(fd, &chr, sizeof(chr), 0) < sizeof(chr)) { + close(fd); + virReportError(VIR_ERR_WRITE_FAILED, _("%s"), path); + return -1; + } + } while ((rc = pread(fd, &online, sizeof(online), 0)) > 0 && + online != '0'); + + close(fd); + + if (rc <= 0) { + virReportError(VIR_ERR_READ_FAILED, _("%s"), path); + return -1; + } + + if ((fd = open(ACRN_OFFLINE_PATH, O_WRONLY)) < 0) { + virReportError(VIR_ERR_OPEN_FAILED, _(ACRN_OFFLINE_PATH)); + return -1; + } + + chr += i; + + if (write(fd, &chr, sizeof(chr)) < sizeof(chr)) { + close(fd); + virReportError(VIR_ERR_WRITE_FAILED, _(ACRN_OFFLINE_PATH)); + return -1; + } + + close(fd); + + if (!allocMap[i]) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("vCPU allocation map error (bit %ld)"), i); + return -1; + } + + allocMap[i] -= 1; + } + + return 0; +} + +static int +acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, + size_t **allocMap) +{ + struct acrnVmList *list; + virBitmapPtr postLaunchedPcpus = NULL; + uint16_t totalCpus; + size_t i, *map = NULL; + int ret = -1; + + if (!(list = acrnVmListNew())) + return -1; + + if (acrnGetPlatform(pi, list) < 0) + goto cleanup; + + totalCpus = pi->cpu_num; + + if (!(postLaunchedPcpus = virBitmapNew(totalCpus))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + + if (VIR_ALLOC_N(map, totalCpus) < 0) + goto cleanup; + + nodeInfo->cpus = totalCpus; + + /* + * Assume pre-launched VMs are always running. + * + * There is no way to figure out the current vCPU + * allocation map via platform_info. It needs to be + * tracked in this driver. + */ + for (i = 0; i < list->size; i++) { + ssize_t pos = -1; + + if (list->vm[i].cfg.load_order == POST_LAUNCHED_VM) { + /* collect all pCPUs that can be used by a UOS */ + while ((pos = virBitmapNextSetBit(list->vm[i].pcpus, pos)) >= 0) { + if (virBitmapSetBit(postLaunchedPcpus, pos) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("virBitmapSetBit failed")); + goto cleanup; + } + } + } else { + if (list->vm[i].cfg.load_order == SOS_VM) { + if (!virBitmapIsBitSet(list->vm[i].pcpus, 0)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("SOS BSP is not pCPU0")); + goto cleanup; + } + } + + while ((pos = virBitmapNextSetBit(list->vm[i].pcpus, pos)) >= 0 && + pos < totalCpus) + map[pos] += 1; + } + } + + if (acrnOfflineCpus(get_nprocs_conf(), postLaunchedPcpus, map) < 0) + goto cleanup; + + *allocMap = map; + map = NULL; + ret = 0; + +cleanup: + if (map) + VIR_FREE(map); + virBitmapFree(postLaunchedPcpus); + acrnVmListFree(list); + return ret; +} + +static int +acrnStateInitialize(bool privileged, + const char *root, + virStateInhibitCallback callback G_GNUC_UNUSED, + void *opaque G_GNUC_UNUSED) +{ + if (root) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Driver does not support embedded mode")); + return VIR_DRV_STATE_INIT_ERROR; + } + + if (!privileged) { + VIR_INFO("Not running privileged, disabling driver"); + return VIR_DRV_STATE_INIT_SKIPPED; + } + + if (VIR_ALLOC(acrn_driver) < 0) + return VIR_DRV_STATE_INIT_ERROR; + + if (virMutexInit(&acrn_driver->lock) < 0) { + VIR_FREE(acrn_driver); + return VIR_DRV_STATE_INIT_ERROR; + } + + /* store a copy of node info before CPU offlining */ + if (virCapabilitiesGetNodeInfo(&acrn_driver->nodeInfo) < 0) + goto cleanup; + + if (acrnInitPlatform(&acrn_driver->pi, &acrn_driver->nodeInfo, + &acrn_driver->vcpuAllocMap) < 0) + goto cleanup; + + if (!(acrn_driver->domains = virDomainObjListNew())) + goto cleanup; + + if (!(acrn_driver->caps = virAcrnCapsBuild())) + goto cleanup; + + if (!(acrn_driver->xmlopt = virAcrnDriverCreateXMLConf())) + goto cleanup; + + if (!(acrn_driver->domainEventState = virObjectEventStateNew())) + goto cleanup; + + if (!(acrn_driver->hostdevMgr = virHostdevManagerGetDefault())) + goto cleanup; + + if (virDomainObjListLoadAllConfigs(acrn_driver->domains, + ACRN_CONFIG_DIR, + ACRN_AUTOSTART_DIR, false, + acrn_driver->xmlopt, + NULL, NULL) < 0) + goto cleanup; + + return VIR_DRV_STATE_INIT_COMPLETE; + +cleanup: + acrnStateCleanup(); + return VIR_DRV_STATE_INIT_ERROR; +} + +static virHypervisorDriver acrnHypervisorDriver = { + .name = "ACRN", + .connectOpen = acrnConnectOpen, /* 0.0.1 */ + .connectClose = acrnConnectClose, /* 0.0.1 */ + .connectGetType = acrnConnectGetType, /* 0.0.1 */ + .connectGetVersion = acrnConnectGetVersion, /* 0.0.1 */ + .connectGetHostname = acrnConnectGetHostname, /* 0.0.1 */ + .nodeGetInfo = acrnNodeGetInfo, /* 0.0.1 */ + .connectGetCapabilities = acrnConnectGetCapabilities, /* 0.0.1 */ + .connectListAllDomains = acrnConnectListAllDomains, /* 0.0.1 */ + .domainCreateXML = acrnDomainCreateXML, /* 0.0.1 */ + .domainLookupByUUID = acrnDomainLookupByUUID, /* 0.0.1 */ + .domainLookupByName = acrnDomainLookupByName, /* 0.0.1 */ + .domainShutdown = acrnDomainShutdown, /* 0.0.1 */ + .domainDestroy = acrnDomainDestroy, /* 0.0.1 */ + .domainGetInfo = acrnDomainGetInfo, /* 0.0.1 */ + .domainGetState = acrnDomainGetState, /* 0.0.1 */ + .domainGetVcpus = acrnDomainGetVcpus, /* 0.0.1 */ + .domainGetXMLDesc = acrnDomainGetXMLDesc, /* 0.0.1 */ + .domainCreate = acrnDomainCreate, /* 0.0.1 */ + .domainCreateWithFlags = acrnDomainCreateWithFlags, /* 0.0.1 */ + .domainDefineXML = acrnDomainDefineXML, /* 0.0.1 */ + .domainDefineXMLFlags = acrnDomainDefineXMLFlags, /* 0.0.1 */ + .domainUndefine = acrnDomainUndefine, /* 0.0.1 */ + .domainUndefineFlags = acrnDomainUndefineFlags, /* 0.0.1 */ + .domainMemoryStats = acrnDomainMemoryStats, /* 0.0.1 */ + .nodeDeviceDettach = acrnNodeDeviceDettach, /* 0.0.1 */ + .nodeDeviceDetachFlags = acrnNodeDeviceDetachFlags, /* 0.0.1 */ + .nodeDeviceReAttach = acrnNodeDeviceReAttach, /* 0.0.1 */ + .domainIsActive = acrnDomainIsActive, /* 0.0.1 */ + .connectBaselineCPU = acrnConnectBaselineCPU, /* 0.0.1 */ + .connectDomainEventRegisterAny = acrnConnectDomainEventRegisterAny, /* 0.0.1 */ + .connectDomainEventDeregisterAny = acrnConnectDomainEventDeregisterAny, /* 0.0.1 */ + .domainOpenConsole = acrnDomainOpenConsole, /* 0.0.1 */ + .nodeGetCPUMap = acrnNodeGetCPUMap, /* 0.0.1 */ +}; + +static virConnectDriver acrnConnectDriver = { + .hypervisorDriver = &acrnHypervisorDriver, +}; + +static virStateDriver acrnStateDriver = { + .name = "ACRN", + .stateInitialize = acrnStateInitialize, + .stateCleanup = acrnStateCleanup, +}; + +int +acrnRegister(void) +{ + if (virRegisterConnectDriver(&acrnConnectDriver, true) < 0) + return -1; + if (virRegisterStateDriver(&acrnStateDriver) < 0) + return -1; + return 0; +} diff --git a/src/acrn/acrn_driver.h b/src/acrn/acrn_driver.h new file mode 100644 index 0000000000..46373f88d6 --- /dev/null +++ b/src/acrn/acrn_driver.h @@ -0,0 +1,5 @@ +#ifndef __ACRN_DRIVER_H__ +#define __ACRN_DRIVER_H__ + +int acrnRegister(void); +#endif /* __ACRN_DRIVER_H__ */ diff --git a/src/libvirt.c b/src/libvirt.c index 26c3fe454f..de1013cb8f 100644 --- a/src/libvirt.c +++ b/src/libvirt.c @@ -1064,6 +1064,9 @@ virConnectOpenInternal(const char *name, #endif #ifndef WITH_VZ STRCASEEQ(ret->uri->scheme, "parallels") || +#endif +#ifndef WITH_ACRN + STRCASEEQ(ret->uri->scheme, "acrn") || #endif false)) { virReportErrorHelper(VIR_FROM_NONE, VIR_ERR_CONFIG_UNSUPPORTED, diff --git a/src/remote/remote_daemon.c b/src/remote/remote_daemon.c index 657c053f6f..b103219f31 100644 --- a/src/remote/remote_daemon.c +++ b/src/remote/remote_daemon.c @@ -187,6 +187,10 @@ static int daemonInitialize(void) if (virDriverLoadModule("vz", "vzRegister", false) < 0) return -1; # endif +# ifdef WITH_ACRN + if (virDriverLoadModule("acrn", "acrnRegister", false) < 0) + return -1; +# endif #endif return 0; } diff --git a/src/util/virerror.c b/src/util/virerror.c index 227a182417..d20f2c428d 100644 --- a/src/util/virerror.c +++ b/src/util/virerror.c @@ -145,6 +145,7 @@ VIR_ENUM_IMPL(virErrorDomain, "TPM", /* 70 */ "BPF", "Cloud-Hypervisor Driver", + "ACRN driver", ); From c8e45c3eef69bec3e0b72b21b226d3329d22ed10 Mon Sep 17 00:00:00 2001 From: Peter Fang Date: Sun, 14 Jun 2020 20:44:16 -0700 Subject: [PATCH 02/32] acrn: add implementation for domainGetCPUStats Currently, only cputime is required by the orchestrators. Report fake cputime since this is currently unsupported by the hypervisor. Signed-off-by: Peter Fang --- src/acrn/acrn_driver.c | 162 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 153 insertions(+), 9 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index e519732c2f..3ef55eab2d 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -1380,14 +1380,6 @@ acrnDomainGetVcpus(virDomainPtr domain, if (!(def = vm->def) || !(priv = vm->privateData)) goto cleanup; - if (gettimeofday(&tv, NULL) < 0) { - virReportSystemError(errno, - "%s", _("getting time of day")); - goto cleanup; - } - - statbase = (tv.tv_sec * 1000UL * 1000UL) + tv.tv_usec; - /* clamp to actual number of vcpus */ if (maxinfo > virDomainDefGetVcpus(vm->def)) maxinfo = virDomainDefGetVcpus(vm->def); @@ -1408,6 +1400,15 @@ acrnDomainGetVcpus(virDomainPtr domain, goto cleanup; } + if (gettimeofday(&tv, NULL) < 0) { + virReportSystemError(errno, + "%s", _("getting time of day")); + goto cleanup; + } + + statbase = (tv.tv_sec * 1000UL * 1000UL) + tv.tv_usec; + statbase /= virBitmapCountBits(priv->cpuAffinitySet); + for (i = 0, pos = -1; i < maxinfo; i++) { virDomainVcpuDefPtr vcpu = virDomainDefGetVcpu(def, i); @@ -1438,7 +1439,7 @@ acrnDomainGetVcpus(virDomainPtr domain, info[i].cpu = pos; /* FIXME fake an increasing cpu time value */ - info[i].cpuTime = statbase / 10; + info[i].cpuTime = statbase; } ret = maxinfo; @@ -1916,6 +1917,148 @@ acrnDomainOpenConsole(virDomainPtr dom, return ret; } +static int +acrnGetDomainTotalCpuStats(virTypedParameterPtr params, + int nparams) +{ + struct timeval tv; + unsigned long long cpu_time; + + if (nparams == 0) /* return supported number of params */ + return 1; + + if (gettimeofday(&tv, NULL) < 0) { + virReportSystemError(errno, + "%s", _("getting time of day")); + return -1; + } + + /* FIXME fake an increasing cpu time value */ + cpu_time = (tv.tv_sec * 1000UL * 1000UL) + tv.tv_usec; + + /* entry 0 is cputime */ + if (virTypedParameterAssign(¶ms[0], VIR_DOMAIN_CPU_STATS_CPUTIME, + VIR_TYPED_PARAM_ULLONG, cpu_time) < 0) + return -1; + + if (nparams > 1) + nparams = 1; + + return nparams; +} + +static int +acrnGetPercpuStats(virTypedParameterPtr params, + unsigned int nparams, + int start_cpu, + unsigned int ncpus, + virBitmapPtr vcpus) +{ + int ret = -1; + size_t i; + int total_cpus, param_idx, need_cpus; + struct timeval tv; + unsigned long long cpu_time; + virBitmapPtr cpumap; + virTypedParameterPtr ent; + + /* return the number of supported params */ + if (nparams == 0 && ncpus != 0) + return 1; + + if (!(cpumap = virHostCPUGetPresentBitmap())) + goto cleanup; + + total_cpus = virBitmapSize(cpumap); + + /* return total number of cpus */ + if (ncpus == 0) { + ret = total_cpus; + goto cleanup; + } + + if (start_cpu >= total_cpus) { + virReportError(VIR_ERR_INVALID_ARG, + _("start_cpu %d larger than maximum of %d"), + start_cpu, total_cpus - 1); + goto cleanup; + } + + if (!vcpus) { + virReportError(VIR_ERR_INTERNAL_ERROR, _("cpumask missing")); + goto cleanup; + } + + if (gettimeofday(&tv, NULL) < 0) { + virReportSystemError(errno, + "%s", _("getting time of day")); + goto cleanup; + } + + /* FIXME fake an increasing cpu time value */ + cpu_time = (tv.tv_sec * 1000UL * 1000UL) + tv.tv_usec; + cpu_time /= virBitmapCountBits(vcpus); + + /* return percpu cputime in index 0 */ + param_idx = 0; + + /* number of cpus to compute */ + need_cpus = MIN(total_cpus, start_cpu + ncpus); + + for (i = start_cpu; i < need_cpus; i++) { + ent = ¶ms[(i - start_cpu) * nparams + param_idx]; + if (virTypedParameterAssign(ent, VIR_DOMAIN_CPU_STATS_CPUTIME, + VIR_TYPED_PARAM_ULLONG, + virBitmapIsBitSet(vcpus, i) ? + cpu_time : 0) < 0) + goto cleanup; + } + + param_idx++; + ret = param_idx; + +cleanup: + virBitmapFree(cpumap); + return ret; +} + +static int +acrnDomainGetCPUStats(virDomainPtr dom, + virTypedParameterPtr params, + unsigned int nparams, + int start_cpu, + unsigned int ncpus, + unsigned int flags) +{ + virDomainObjPtr vm; + acrnDomainObjPrivatePtr priv; + int ret = -1; + + virCheckFlags(0, -1); + + if (!(vm = acrnDomObjFromDomain(dom))) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("domain is not running")); + goto cleanup; + } + + if (start_cpu == -1) { + ret = acrnGetDomainTotalCpuStats(params, nparams); + } else { + priv = vm->privateData; + ret = acrnGetPercpuStats(params, nparams, start_cpu, ncpus, + priv->cpuAffinitySet); + } + +cleanup: + if (vm) + virObjectUnlock(vm); + return ret; +} + static virDrvOpenStatus acrnConnectOpen(virConnectPtr conn, virConnectAuthPtr auth G_GNUC_UNUSED, @@ -2617,6 +2760,7 @@ static virHypervisorDriver acrnHypervisorDriver = { .connectDomainEventRegisterAny = acrnConnectDomainEventRegisterAny, /* 0.0.1 */ .connectDomainEventDeregisterAny = acrnConnectDomainEventDeregisterAny, /* 0.0.1 */ .domainOpenConsole = acrnDomainOpenConsole, /* 0.0.1 */ + .domainGetCPUStats = acrnDomainGetCPUStats, /* 0.0.1 */ .nodeGetCPUMap = acrnNodeGetCPUMap, /* 0.0.1 */ }; From cf29f45530ae9fa0feead73497aabadec088f816 Mon Sep 17 00:00:00 2001 From: Peter Fang Date: Fri, 19 Jun 2020 00:18:35 -0700 Subject: [PATCH 03/32] acrn: fix connection failure Update acrnConnectDriver for the remote driver to respond correctly to "acrn:///system" and not hijack it. Additionally, add implementation for connectURIProbe and update acrnConnectOpen to comply with the current behavior of virConnectOpenInternal. Signed-off-by: Peter Fang --- src/acrn/acrn_driver.c | 55 +++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 3ef55eab2d..10f80a62da 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -2059,44 +2059,40 @@ acrnDomainGetCPUStats(virDomainPtr dom, return ret; } +static int +acrnConnectURIProbe(char **uri) +{ + if (!acrn_driver) + return 0; + + *uri = g_strdup("acrn:///system"); + return 1; +} + static virDrvOpenStatus acrnConnectOpen(virConnectPtr conn, virConnectAuthPtr auth G_GNUC_UNUSED, virConfPtr conf G_GNUC_UNUSED, unsigned int flags) { - virCheckFlags(VIR_CONNECT_RO, VIR_DRV_OPEN_ERROR); + virCheckFlags(VIR_CONNECT_RO, VIR_DRV_OPEN_ERROR); - if (conn->uri == NULL) { - if (acrn_driver == NULL) - return VIR_DRV_OPEN_DECLINED; - - if (!(conn->uri = virURIParse("acrn:///system"))) - return VIR_DRV_OPEN_ERROR; - } else { - if (STRNEQ_NULLABLE(conn->uri->scheme, "acrn")) - return VIR_DRV_OPEN_DECLINED; - - if (conn->uri->server) - return VIR_DRV_OPEN_DECLINED; - - if (STRNEQ_NULLABLE(conn->uri->path, "/system")) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("Unexpected ACRN URI path '%s', try acrn:///system"), - conn->uri->path ? conn->uri->path : "NULL"); - return VIR_DRV_OPEN_ERROR; - } + if (STRNEQ(conn->uri->path, "/system")) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unexpected ACRN URI path '%s', try acrn:///system"), + conn->uri->path); + return VIR_DRV_OPEN_ERROR; + } - if (acrn_driver == NULL) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("ACRN state driver is not active")); - return VIR_DRV_OPEN_ERROR; - } - } + if (!acrn_driver) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("ACRN state driver is not active")); + return VIR_DRV_OPEN_ERROR; + } - conn->privateData = acrn_driver; + conn->privateData = acrn_driver; - return VIR_DRV_OPEN_SUCCESS; + return VIR_DRV_OPEN_SUCCESS; } static int @@ -2728,6 +2724,7 @@ acrnStateInitialize(bool privileged, static virHypervisorDriver acrnHypervisorDriver = { .name = "ACRN", + .connectURIProbe = acrnConnectURIProbe, /* 0.0.1 */ .connectOpen = acrnConnectOpen, /* 0.0.1 */ .connectClose = acrnConnectClose, /* 0.0.1 */ .connectGetType = acrnConnectGetType, /* 0.0.1 */ @@ -2765,6 +2762,8 @@ static virHypervisorDriver acrnHypervisorDriver = { }; static virConnectDriver acrnConnectDriver = { + .localOnly = true, + .uriSchemes = (const char *[]){ "acrn", NULL }, .hypervisorDriver = &acrnHypervisorDriver, }; From 4c8bc32b9f3a1f6f06f39a5e316a4cec5b75c3c3 Mon Sep 17 00:00:00 2001 From: Peter Fang Date: Thu, 18 Jun 2020 20:01:30 -0700 Subject: [PATCH 04/32] acrn: add virtacrnd systemd services Libvirtd will eventually be phased out, so add support for virtacrnd systemd services. Signed-off-by: Peter Fang --- src/acrn/Makefile.inc.am | 33 +++++++++++++++++++++++++ src/acrn/virtacrnd.init.in | 26 ++++++++++++++++++++ src/acrn/virtacrnd.service.in | 46 +++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 src/acrn/virtacrnd.init.in create mode 100644 src/acrn/virtacrnd.service.in diff --git a/src/acrn/Makefile.inc.am b/src/acrn/Makefile.inc.am index 22465c4ac3..0cfcc3d3da 100644 --- a/src/acrn/Makefile.inc.am +++ b/src/acrn/Makefile.inc.am @@ -55,6 +55,39 @@ virtacrnd_CFLAGS = \ virtacrnd_LDFLAGS = $(REMOTE_DAEMON_LD_FLAGS) virtacrnd_LDADD = $(REMOTE_DAEMON_LD_ADD) +SYSTEMD_UNIT_FILES += \ + virtacrnd.service \ + virtacrnd.socket \ + virtacrnd-ro.socket \ + virtacrnd-admin.socket \ + $(NULL) +SYSTEMD_UNIT_FILES_IN += \ + acrn/virtacrnd.service.in \ + $(NULL) + +OPENRC_INIT_FILES += \ + virtacrnd.init \ + $(NULL) +OPENRC_INIT_FILES_IN += \ + acrn/virtacrnd.init.in \ + $(NULL) + +VIRTACRND_UNIT_VARS = \ + $(VIRTD_UNIT_VARS) \ + -e 's|[@]name[@]|Libvirt acrn|g' \ + -e 's|[@]service[@]|virtacrnd|g' \ + -e 's|[@]sockprefix[@]|virtacrnd|g' \ + $(NULL) + +virtacrnd.init: acrn/virtacrnd.init.in $(top_builddir)/config.status + $(AM_V_GEN)$(SED) $(LIBVIRTD_INIT_VARS) $< > $@-t && mv $@-t $@ + +virtacrnd.service: acrn/virtacrnd.service.in $(top_builddir)/config.status + $(AM_V_GEN)$(SED) $(VIRTACRND_UNIT_VARS) $< > $@-t && mv $@-t $@ + +virtacrn%.socket: remote/libvirt%.socket.in $(top_builddir)/config.status + $(AM_V_GEN)$(SED) $(VIRTACRND_UNIT_VARS) $< > $@-t && mv $@-t $@ + acrn/virtacrnd.conf: remote/libvirtd.conf.in $(AM_V_GEN)$(SED) \ -e '/[@]CUT_ENABLE_IP[@]/,/[@]END[@]/d' \ diff --git a/src/acrn/virtacrnd.init.in b/src/acrn/virtacrnd.init.in new file mode 100644 index 0000000000..a34e84c944 --- /dev/null +++ b/src/acrn/virtacrnd.init.in @@ -0,0 +1,26 @@ +#!/sbin/openrc-run + +description="Virtualization acrn daemon" + +VIRTACRND_OPTS=${VIRTACRND_OPTS:-"${VIRTACRND_OPTS}"} +VIRTACRND_TIMEOUT=${VIRTACRND_TERMTIMEOUT:-"TERM/25/KILL/5"} + +command="@sbindir@/virtacrnd" +command_args="-d ${VIRTACRND_OPTS}" +pidfile="@runstatedir@/virtacrnd.pid" +retry="${VIRTACRND_TERMTIMEOUT}" + +extra_started_commands="reload" +description_reload="re-exec the daemon to enforce configuration reload" + +depend() { + use ceph dbus iscsid virtlockd + after nfs nfsmount +} + +reload() { + ebegin "re-exec() virtacrnd" + + start-stop-daemon --signal SIGHUP \ + --exec "${command}" --pidfile "${pidfile}" +} diff --git a/src/acrn/virtacrnd.service.in b/src/acrn/virtacrnd.service.in new file mode 100644 index 0000000000..fc5b0e9867 --- /dev/null +++ b/src/acrn/virtacrnd.service.in @@ -0,0 +1,46 @@ +[Unit] +Description=Virtualization acrn daemon +Conflicts=libvirtd.service +Requires=virtacrnd.socket +Requires=virtacrnd-ro.socket +Requires=virtacrnd-admin.socket +Wants=systemd-machined.service +Before=libvirt-guests.service +After=network.target +After=dbus.service +After=apparmor.service +After=local-fs.target +After=remote-fs.target +After=systemd-logind.service +After=systemd-machined.service +Documentation=man:libvirtd(8) +Documentation=https://libvirt.org + +[Service] +Type=notify +ExecStart=@sbindir@/virtacrnd --timeout 120 +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=on-failure +# At least 1 FD per guest, often 2 (eg qemu monitor + qemu agent). +# eg if we want to support 4096 guests, we'll typically need 8192 FDs +# If changing this, also consider virtlogd.service & virtlockd.service +# limits which are also related to number of guests +LimitNOFILE=8192 +# The cgroups pids controller can limit the number of tasks started by +# the daemon, which can limit the number of domains for some hypervisors. +# A conservative default of 8 tasks per guest results in a TasksMax of +# 32k to support 4096 guests. +TasksMax=32768 +# With cgroups v2 there is no devices controller anymore, we have to use +# eBPF to control access to devices. In order to do that we create a eBPF +# hash MAP which locks memory. The default map size for 64 devices together +# with program takes 12k per guest. After rounding up we will get 64M to +# support 4096 guests. +LimitMEMLOCK=64M + +[Install] +WantedBy=multi-user.target +Also=virtacrnd.socket +Also=virtacrnd-ro.socket +Also=virtacrnd-admin.socket From 97712aa8e96cb7f09539b520540dd4058fd10fc1 Mon Sep 17 00:00:00 2001 From: Helmut Buchsbaum Date: Mon, 8 Jun 2020 10:37:49 +0200 Subject: [PATCH 05/32] acrn_driver: use error codes in acrnGetPlatform Return approproiate error codes in acrnGetPlatform to be able to handle the errors differently on caller's side. Signed-off-by: Helmut Buchsbaum --- src/acrn/acrn_driver.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 10f80a62da..85598e2398 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -169,7 +169,7 @@ static int acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) { acrnVmCfg vmcfg; - int fd, vcpu_num, pos, ret = -1; + int fd, vcpu_num, pos, ret; uint8_t *p; uint16_t i, j; uint64_t pcpus; @@ -181,8 +181,7 @@ acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) vmList->size = 0; if ((fd = acrnGetVhmFd()) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("acrnGetVhmFd failed")); + ret = -ENODEV; goto cleanup; } @@ -192,6 +191,7 @@ acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) !pi->cpu_num || !pi->max_vms || !pi->vm_config_entry_size) { virReportError(VIR_ERR_INTERNAL_ERROR, _("acrnGetPlatformInfo failed")); + ret = -EINVAL; goto cleanup; } @@ -200,6 +200,7 @@ acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) _("ACRN platform version mismatch: " "got 0x%x, expecting 0x%x"), pi->version, ACRN_PI_VERSION); + ret = -EOPNOTSUPP; goto cleanup; } @@ -207,12 +208,14 @@ acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) pi->max_vms, pi->vm_config_entry_size))) { virReportError(VIR_ERR_NO_MEMORY, NULL); + ret = -ENOMEM; goto cleanup; } } /* now get vm config */ - if (acrnGetPlatformInfo(fd, pi) < 0) { + ret = acrnGetPlatformInfo(fd, pi); + if (ret < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("acrnGetPlatformInfo failed")); goto cleanup; @@ -229,12 +232,14 @@ acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) virReportError(VIR_ERR_INTERNAL_ERROR, _("only %lu VMs are supported"), G_N_ELEMENTS(vmList->vm)); + ret = -EINVAL; goto cleanup; } if (!(pcpus = vmcfg.cpu_affinity)) { virReportError(VIR_ERR_INTERNAL_ERROR, _("no pCPU in vm[%u]"), i); + ret = -EINVAL; goto cleanup; } @@ -256,6 +261,7 @@ acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) virBitmapNew( sizeof(vmcfg.cpu_affinity) * CHAR_BIT))) { virReportError(VIR_ERR_NO_MEMORY, NULL); + ret = -ENOMEM; goto cleanup; } @@ -266,6 +272,7 @@ acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) if (virBitmapSetBit(vmList->vm[j].pcpus, pos) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("virBitmapSetBit failed")); + ret = -EINVAL; goto cleanup; } pcpus &= ~(1ULL << pos); From c6ea996400f8cc84316c644071b65dcdd9b56a1f Mon Sep 17 00:00:00 2001 From: Helmut Buchsbaum Date: Mon, 8 Jun 2020 10:46:00 +0200 Subject: [PATCH 06/32] acrn_driver: use error codes in acrnInitPlatform Return appropriate error codes in acrnInitPlatform to be able to handle the errors differently on caller's side. In particular propagate error code when possible. Signed-off-by: Helmut Buchsbaum --- src/acrn/acrn_driver.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 85598e2398..6dae9cbb69 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -2598,23 +2598,27 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, virBitmapPtr postLaunchedPcpus = NULL; uint16_t totalCpus; size_t i, *map = NULL; - int ret = -1; + int ret; if (!(list = acrnVmListNew())) - return -1; + return -ENOMEM; - if (acrnGetPlatform(pi, list) < 0) + ret = acrnGetPlatform(pi, list); + if (ret < 0) goto cleanup; totalCpus = pi->cpu_num; if (!(postLaunchedPcpus = virBitmapNew(totalCpus))) { virReportError(VIR_ERR_NO_MEMORY, NULL); + ret = -ENOMEM; goto cleanup; } - if (VIR_ALLOC_N(map, totalCpus) < 0) + if (VIR_ALLOC_N(map, totalCpus) < 0) { + ret = -ENOMEM; goto cleanup; + } nodeInfo->cpus = totalCpus; @@ -2634,6 +2638,7 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, if (virBitmapSetBit(postLaunchedPcpus, pos) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("virBitmapSetBit failed")); + ret = -EINVAL; goto cleanup; } } @@ -2642,6 +2647,7 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, if (!virBitmapIsBitSet(list->vm[i].pcpus, 0)) { virReportError(VIR_ERR_INTERNAL_ERROR, _("SOS BSP is not pCPU0")); + ret = -EINVAL; goto cleanup; } } @@ -2652,8 +2658,10 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, } } - if (acrnOfflineCpus(get_nprocs_conf(), postLaunchedPcpus, map) < 0) + if (acrnOfflineCpus(get_nprocs_conf(), postLaunchedPcpus, map) < 0) { + ret = -EIO; goto cleanup; + } *allocMap = map; map = NULL; From 5f106a60ae10488dcc27a70557055f561fea4568 Mon Sep 17 00:00:00 2001 From: Helmut Buchsbaum Date: Mon, 8 Jun 2020 11:25:03 +0200 Subject: [PATCH 07/32] acrn: do not fail on non-ACRN systems To be able to run libvirt on any system we must not fail if there is no ARCN hypervisor available. Simply skip initialization with proper cleanup but without failure in this case. Signed-off-by: Helmut Buchsbaum --- src/acrn/acrn_driver.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 6dae9cbb69..c1db42de0f 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -2675,12 +2675,14 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, return ret; } -static int +static virDrvStateInitResult acrnStateInitialize(bool privileged, const char *root, virStateInhibitCallback callback G_GNUC_UNUSED, void *opaque G_GNUC_UNUSED) { + int ret; + if (root) { virReportError(VIR_ERR_INVALID_ARG, "%s", _("Driver does not support embedded mode")); @@ -2704,8 +2706,15 @@ acrnStateInitialize(bool privileged, if (virCapabilitiesGetNodeInfo(&acrn_driver->nodeInfo) < 0) goto cleanup; - if (acrnInitPlatform(&acrn_driver->pi, &acrn_driver->nodeInfo, - &acrn_driver->vcpuAllocMap) < 0) + ret = acrnInitPlatform(&acrn_driver->pi, &acrn_driver->nodeInfo, + &acrn_driver->vcpuAllocMap); + if (ret == -ENODEV) { + /* we are not running on an ACRN enabled system */ + VIR_INFO("ACRN hypervisor not available, disabling driver"); + ret = VIR_DRV_STATE_INIT_SKIPPED; + goto cleanup_nofail; + } + if (ret < 0) goto cleanup; if (!(acrn_driver->domains = virDomainObjListNew())) @@ -2733,8 +2742,10 @@ acrnStateInitialize(bool privileged, return VIR_DRV_STATE_INIT_COMPLETE; cleanup: + ret = VIR_DRV_STATE_INIT_ERROR; +cleanup_nofail: acrnStateCleanup(); - return VIR_DRV_STATE_INIT_ERROR; + return ret; } static virHypervisorDriver acrnHypervisorDriver = { From 221d92703b90c7dec92cde71dab325ffc38c8af3 Mon Sep 17 00:00:00 2001 From: Peter Fang Date: Fri, 19 Jun 2020 21:41:45 -0700 Subject: [PATCH 08/32] acrn: add RTVM support RTVM support is added using XML namespace "acrn": - : indicate RTVM via - : miscellaneous supporting command-line args via Signed-off-by: Peter Fang --- src/acrn/acrn_domain.c | 164 ++++++++++++++++++++++++++++++++++++++++- src/acrn/acrn_domain.h | 8 ++ src/acrn/acrn_driver.c | 34 +++++++-- 3 files changed, 200 insertions(+), 6 deletions(-) diff --git a/src/acrn/acrn_domain.c b/src/acrn/acrn_domain.c index e1f3dce6fb..fa32c088b8 100644 --- a/src/acrn/acrn_domain.c +++ b/src/acrn/acrn_domain.c @@ -2,6 +2,7 @@ #include "acrn_domain.h" #include "acrn_device.h" +#include "virstring.h" #include "viralloc.h" #include "virfile.h" #include "virlog.h" @@ -340,10 +341,171 @@ static virDomainXMLPrivateDataCallbacks virAcrnDriverPrivateDataCallbacks = { .free = acrnDomainObjPrivateFree, }; +static void +acrnDomainDefNamespaceFree(void *nsdata) +{ + acrnDomainXmlNsDefPtr nsdef = nsdata; + + if (!nsdef) + return; + + virStringListFreeCount(nsdef->args, nsdef->nargs); + VIR_FREE(nsdef); +} + +static int +acrnDomainDefNamespaceParseConfig(acrnDomainXmlNsDefPtr nsdef, + xmlXPathContextPtr ctxt) +{ + g_autofree xmlNodePtr *nodes = NULL; + xmlNodePtr node; + int nnodes; + + if ((nnodes = virXPathNodeSet("./acrn:config", + ctxt, &nodes)) < 0) { + virReportError(VIR_ERR_XML_ERROR, + _("Invalid acrn:config node")); + return -1; + } + + if (nnodes == 0) + return 0; + + if (nnodes > 1) { + virReportError(VIR_ERR_XML_ERROR, + _("More than 1 acrn:config nodes")); + return -1; + } + + for (node = nodes[0]->children; node; node = node->next) { + if (node->type == XML_ELEMENT_NODE) { + if (virXMLNodeNameEqual(node, "rtvm")) + nsdef->rtvm = true; + } + } + + return 0; +} + +static int +acrnDomainDefNamespaceParseCommandlineArgs(acrnDomainXmlNsDefPtr nsdef, + xmlXPathContextPtr ctxt) +{ + g_autofree xmlNodePtr *nodes = NULL; + int nnodes, i; + + if ((nnodes = virXPathNodeSet("./acrn:commandline/acrn:arg", + ctxt, &nodes)) < 0) { + virReportError(VIR_ERR_XML_ERROR, + _("Invalid acrn:arg node")); + return -1; + } + + if (nnodes == 0) + return 0; + + if (VIR_ALLOC_N(nsdef->args, nnodes) < 0) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + return -1; + } + + for (i = 0; i < nnodes; i++) { + if (!(nsdef->args[nsdef->nargs++] = + virXMLPropString(nodes[i], "value"))) { + virReportError(VIR_ERR_XML_ERROR, + _("No command-line argument specified")); + return -1; + } + } + + return 0; +} + +static int +acrnDomainDefNamespaceParse(xmlXPathContextPtr ctxt, + void **data) +{ + acrnDomainXmlNsDefPtr nsdata; + int ret = -1; + + if (VIR_ALLOC(nsdata) < 0) + return -1; + + if (acrnDomainDefNamespaceParseConfig(nsdata, ctxt) < 0 || + acrnDomainDefNamespaceParseCommandlineArgs(nsdata, ctxt) < 0) + goto cleanup; + + if (nsdata->rtvm || nsdata->nargs) + *data = g_steal_pointer(&nsdata); + + ret = 0; + +cleanup: + acrnDomainDefNamespaceFree(nsdata); + return ret; +} + +static void +acrnDomainDefNamespaceFormatXMLConfig(virBufferPtr buf, + acrnDomainXmlNsDefPtr xmlns) +{ + if (!xmlns->rtvm) + return; + + virBufferAddLit(buf, "\n"); + virBufferAdjustIndent(buf, 2); + + virBufferAddLit(buf, "\n"); + + virBufferAdjustIndent(buf, -2); + virBufferAddLit(buf, "\n"); +} + +static void +acrnDomainDefNamespaceFormatXMLCommandlineArgs(virBufferPtr buf, + acrnDomainXmlNsDefPtr cmd) +{ + size_t i; + + if (!cmd->nargs) + return; + + virBufferAddLit(buf, "\n"); + virBufferAdjustIndent(buf, 2); + + for (i = 0; i < cmd->nargs; i++) + virBufferEscapeString(buf, "\n", + cmd->args[i]); + + virBufferAdjustIndent(buf, -2); + virBufferAddLit(buf, "\n"); +} + +static int +acrnDomainDefNamespaceFormatXML(virBufferPtr buf, + void *nsdata) +{ + acrnDomainXmlNsDefPtr xmlns = nsdata; + + acrnDomainDefNamespaceFormatXMLConfig(buf, xmlns); + acrnDomainDefNamespaceFormatXMLCommandlineArgs(buf, xmlns); + + return 0; +} + +static virXMLNamespace virAcrnDriverDomainXMLNamespace = { + .parse = acrnDomainDefNamespaceParse, + .free = acrnDomainDefNamespaceFree, + .format = acrnDomainDefNamespaceFormatXML, + .prefix = "acrn", + .uri = "http://libvirt.org/schemas/domain/acrn/1.0", +}; + virDomainXMLOptionPtr virAcrnDriverCreateXMLConf(void) { return virDomainXMLOptionNew(&virAcrnDriverDomainDefParserConfig, &virAcrnDriverPrivateDataCallbacks, - NULL, NULL, NULL); + &virAcrnDriverDomainXMLNamespace, + NULL, NULL); } diff --git a/src/acrn/acrn_domain.h b/src/acrn/acrn_domain.h index 17641febcc..0e3c20b4cc 100644 --- a/src/acrn/acrn_domain.h +++ b/src/acrn/acrn_domain.h @@ -15,6 +15,14 @@ struct _acrnDomainObjPrivate { size_t nttys; }; +typedef struct _acrnDomainXmlNsDef acrnDomainXmlNsDef; +typedef acrnDomainXmlNsDef *acrnDomainXmlNsDefPtr; +struct _acrnDomainXmlNsDef { + bool rtvm; + size_t nargs; + char **args; +}; + void acrnDomainTtyCleanup(acrnDomainObjPrivatePtr priv); virDomainXMLOptionPtr virAcrnDriverCreateXMLConf(void); #endif /* __ACRN_DOMAIN_H__ */ diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index c1db42de0f..42028e4269 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -3,7 +3,6 @@ #include #include #include -#include #include "configmake.h" #include "datatypes.h" #include "node_device_conf.h" @@ -31,7 +30,6 @@ #define VIR_FROM_THIS VIR_FROM_ACRN #define ACRN_DM_PATH "/usr/bin/acrn-dm" #define ACRN_CTL_PATH "/usr/bin/acrnctl" -#define ACRN_NAMESPACE_HREF "http://libvirt.org/schemas/domain/acrn/0.0" #define ACRN_OFFLINE_PATH "/sys/class/vhm/acrn_vhm/offline_cpu" #define SYSFS_CPU_PATH "/sys/devices/system/cpu" #define ACRN_AUTOSTART_DIR SYSCONFDIR "/libvirt/acrn/autostart" @@ -579,6 +577,7 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, virDomainDefPtr def; virBitmapPtr allowedmask = NULL; acrnDomainObjPrivatePtr priv; + acrnDomainXmlNsDefPtr nsdef; int ret = -1; if (!vm || !(def = vm->def)) @@ -613,10 +612,12 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, goto cleanup; } + nsdef = def->namespaceData; + /* vCPU placement */ if (acrnAllocateVcpus(pi, allowedmask ? allowedmask : entry->pcpus, - false, def->maxvcpus, allocMap, + nsdef && nsdef->rtvm, def->maxvcpus, allocMap, priv->cpuAffinitySet) < 0) goto cleanup; @@ -1011,9 +1012,11 @@ acrnBuildStartCmd(virDomainObjPtr vm) virDomainDefPtr def; virCommandPtr cmd; acrnDomainObjPrivatePtr priv; + acrnDomainXmlNsDefPtr nsdef; struct acrnCmdDeviceData data = { 0 }; char *pcpus; char uuidstr[VIR_UUID_STRING_BUFLEN]; + size_t i; if (!vm || !(def = vm->def)) return NULL; @@ -1045,6 +1048,15 @@ acrnBuildStartCmd(virDomainObjPtr vm) virCommandAddArg(cmd, virUUIDFormat(priv->hvUUID, uuidstr)); } + nsdef = def->namespaceData; + + /* RTVM */ + if (nsdef && nsdef->rtvm) + virCommandAddArgList(cmd, + "--lapic_pt", + "--virtio_poll", "1000000", + NULL); + /* PCI hostbridge */ virCommandAddArgList(cmd, "-s", "0:0,hostbridge", NULL); @@ -1057,6 +1069,12 @@ acrnBuildStartCmd(virDomainObjPtr vm) return NULL; } + /* User-defined command-line args */ + if (nsdef) { + for (i = 0; i < nsdef->nargs; i++) + virCommandAddArg(cmd, nsdef->args[i]); + } + /* Bootloader */ if (def->os.loader && def->os.loader->path) { virBuffer buf = VIR_BUFFER_INITIALIZER; @@ -1485,6 +1503,7 @@ acrnDomainCreateXML(virConnectPtr conn, acrnConnectPtr privconn = conn->privateData; struct acrnVmList *vmList = NULL; acrnDomainObjPrivatePtr priv; + acrnDomainXmlNsDefPtr nsdef; virDomainDefPtr def; virDomainObjPtr vm = NULL; virObjectEventPtr event = NULL; @@ -1512,9 +1531,11 @@ acrnDomainCreateXML(virConnectPtr conn, if (acrnGetPlatform(&privconn->pi, vmList) < 0) goto cleanup; + nsdef = def->namespaceData; + /* get hv UUID for the allocated VM */ if ((idx = acrnAllocateVm(privconn->domains, def, &privconn->pi, vmList, - false, hvUUID)) < 0) + nsdef && nsdef->rtvm, hvUUID)) < 0) goto cleanup; if (!(vm = virDomainObjListAdd(privconn->domains, def, @@ -1654,6 +1675,7 @@ acrnDomainDefineXMLFlags(virConnectPtr conn, const char *xml, acrnConnectPtr privconn = conn->privateData; struct acrnVmList *vmList = NULL; acrnDomainObjPrivatePtr priv; + acrnDomainXmlNsDefPtr nsdef; virDomainDefPtr def = NULL, oldDef = NULL; virDomainObjPtr vm = NULL; virObjectEventPtr event = NULL; @@ -1682,9 +1704,11 @@ acrnDomainDefineXMLFlags(virConnectPtr conn, const char *xml, if (acrnGetPlatform(&privconn->pi, vmList) < 0) goto cleanup; + nsdef = def->namespaceData; + /* get hv UUID for the allocated VM */ if (acrnAllocateVm(privconn->domains, def, &privconn->pi, vmList, - false, hvUUID) < 0) + nsdef && nsdef->rtvm, hvUUID) < 0) goto cleanup; if (!(vm = virDomainObjListAdd(privconn->domains, def, From e47d09887405ee68af66203141c877975dc6ed6f Mon Sep 17 00:00:00 2001 From: Peter Fang Date: Mon, 20 Jul 2020 00:34:50 -0700 Subject: [PATCH 09/32] Create SECURITY.md Signed-off-by: Peter Fang --- SECURITY.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..65eef4898e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ +# libvirt’s read-only privilege issue +From https://access.redhat.com/libvirt-privesc-vulnerabilities: + +As the libvirt API has evolved over time, the line between “privileged” and “unprivileged” operations became less clear. API calls that were originally exposed read-only, gained more capabilities that made them more powerful, but introduced security risks that were not obvious at the time. + +Administrative changes made to `/etc/libvirt/libvirtd.conf` may affect your level of exposure: +- If you have enabled the setting `listen_tcp`, network users that can reach the libvirt port may be able to conduct an attack. +- If access to libvirt is restricted by changing `unix_sock_ro_perms` to something more restrictive than `0777`, only those users able to `connect()` to the socket will be able to attack libvirtd. For example, the following allows only members of the `libvirt` group: +``` +unix_sock_group = "libvirt" +unix_sock_ro_perms = "0770" +``` + +We recommend that users at least adopt coarse-grained access control and properly configure `unix_sock_group` and `unix_sock_ro_perms` in order to minimize libvirt's attack surface. From 1683499254c559bc06d24b4af86d5f5d00ae0d17 Mon Sep 17 00:00:00 2001 From: Helmut Buchsbaum Date: Tue, 23 Jun 2020 11:52:12 +0200 Subject: [PATCH 10/32] acrn_driver: introduce acrnIsRtvm Since rtvm property lives in ACRN specific namespace data which is stored as opaque namespaceData pointer in the domain definition, it always can be derived from the actual domain definition. Provide an accessor to the rtvm property via domain definition and use it wherever possible. Furthermore use this acrnAllocateVm() which already uses the domain definition and thus can access the rtvm property without the need to pass it as parameter. Signed-off-by: Helmut Buchsbaum --- src/acrn/acrn_driver.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 42028e4269..7d2074bcb2 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -321,13 +321,21 @@ acrnFindHvUUID(virDomainObjPtr vm, void *opaque) return ret; } +static bool +acrnIsRtvm(virDomainDefPtr def) +{ + acrnDomainXmlNsDefPtr nsdef = def->namespaceData; + + return (nsdef && nsdef->rtvm); +} + /* * This function must not be called with any virDomainObjPtr * lock held, as it can attempt to hold any such lock in doms. */ static ssize_t acrnAllocateVm(virDomainObjListPtr doms, virDomainDefPtr def, - acrnPlatformInfoPtr pi, struct acrnVmList *vmList, bool rtvm, + acrnPlatformInfoPtr pi, struct acrnVmList *vmList, unsigned char *uuid) { enum acrn_vm_severity severity; @@ -338,7 +346,7 @@ acrnAllocateVm(virDomainObjListPtr doms, virDomainDefPtr def, char *maskstr = NULL; char uuidstr[VIR_UUID_STRING_BUFLEN]; - severity = (rtvm) ? SEVERITY_RTVM : SEVERITY_STANDARD_VM; + severity = (acrnIsRtvm(def)) ? SEVERITY_RTVM : SEVERITY_STANDARD_VM; if (def->cpumask) { /* prepare a sanitized cpumask */ @@ -577,7 +585,6 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, virDomainDefPtr def; virBitmapPtr allowedmask = NULL; acrnDomainObjPrivatePtr priv; - acrnDomainXmlNsDefPtr nsdef; int ret = -1; if (!vm || !(def = vm->def)) @@ -612,12 +619,10 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, goto cleanup; } - nsdef = def->namespaceData; - /* vCPU placement */ if (acrnAllocateVcpus(pi, allowedmask ? allowedmask : entry->pcpus, - nsdef && nsdef->rtvm, def->maxvcpus, allocMap, + acrnIsRtvm(def), def->maxvcpus, allocMap, priv->cpuAffinitySet) < 0) goto cleanup; @@ -1048,10 +1053,8 @@ acrnBuildStartCmd(virDomainObjPtr vm) virCommandAddArg(cmd, virUUIDFormat(priv->hvUUID, uuidstr)); } - nsdef = def->namespaceData; - /* RTVM */ - if (nsdef && nsdef->rtvm) + if (acrnIsRtvm(def)) virCommandAddArgList(cmd, "--lapic_pt", "--virtio_poll", "1000000", @@ -1069,6 +1072,8 @@ acrnBuildStartCmd(virDomainObjPtr vm) return NULL; } + nsdef = def->namespaceData; + /* User-defined command-line args */ if (nsdef) { for (i = 0; i < nsdef->nargs; i++) @@ -1503,7 +1508,6 @@ acrnDomainCreateXML(virConnectPtr conn, acrnConnectPtr privconn = conn->privateData; struct acrnVmList *vmList = NULL; acrnDomainObjPrivatePtr priv; - acrnDomainXmlNsDefPtr nsdef; virDomainDefPtr def; virDomainObjPtr vm = NULL; virObjectEventPtr event = NULL; @@ -1531,11 +1535,9 @@ acrnDomainCreateXML(virConnectPtr conn, if (acrnGetPlatform(&privconn->pi, vmList) < 0) goto cleanup; - nsdef = def->namespaceData; - /* get hv UUID for the allocated VM */ if ((idx = acrnAllocateVm(privconn->domains, def, &privconn->pi, vmList, - nsdef && nsdef->rtvm, hvUUID)) < 0) + hvUUID)) < 0) goto cleanup; if (!(vm = virDomainObjListAdd(privconn->domains, def, @@ -1675,7 +1677,6 @@ acrnDomainDefineXMLFlags(virConnectPtr conn, const char *xml, acrnConnectPtr privconn = conn->privateData; struct acrnVmList *vmList = NULL; acrnDomainObjPrivatePtr priv; - acrnDomainXmlNsDefPtr nsdef; virDomainDefPtr def = NULL, oldDef = NULL; virDomainObjPtr vm = NULL; virObjectEventPtr event = NULL; @@ -1704,11 +1705,9 @@ acrnDomainDefineXMLFlags(virConnectPtr conn, const char *xml, if (acrnGetPlatform(&privconn->pi, vmList) < 0) goto cleanup; - nsdef = def->namespaceData; - /* get hv UUID for the allocated VM */ if (acrnAllocateVm(privconn->domains, def, &privconn->pi, vmList, - nsdef && nsdef->rtvm, hvUUID) < 0) + hvUUID) < 0) goto cleanup; if (!(vm = virDomainObjListAdd(privconn->domains, def, From 86c667faa9482e91b35c3dc7ce890f0b4774c15d Mon Sep 17 00:00:00 2001 From: Helmut Buchsbaum Date: Mon, 8 Jun 2020 14:58:05 +0200 Subject: [PATCH 11/32] acrn: cannot start predefined ACRN domain Starting a predefined ACRN VM fails: root@nuc7i7dnh:~# virsh list --all Id Name State -------------------------------- - DebianBuster0 shut off root@nuc7i7dnh:~# virsh start DebianBuster0 error: Failed to start domain DebianBuster0 error: internal error: vm(00000000-0000-0000-0000-000000000000) not found The libvirtd log shows libvirtd: 1623: error : acrnDomainCreateWithFlags:1619 : internal error: vm(00000000-0000-0000-0000-000000000000) not found Fix this by allocating a VM for each predefined ACRN domain in acrnStateInitialize(). Signed-off-by: Helmut Buchsbaum --- src/acrn/acrn_driver.c | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 7d2074bcb2..8c3970a913 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -2698,6 +2698,36 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, return ret; } +static int +acrnPersistentDomainInit(virDomainObjPtr dom, void *opaque) +{ + unsigned char hvUUID[VIR_UUID_BUFLEN]; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + struct acrnVmList *vmList = opaque; + acrnDomainObjPrivatePtr priv = dom->privateData; + virObjectEventPtr event = NULL; + + if (acrnAllocateVm(acrn_driver->domains, dom->def, &acrn_driver->pi, vmList, + hvUUID) < 0) + return -1; + + VIR_DEBUG("Adding ACRN %sdomain %s (%s)", + acrnIsRtvm(dom->def) ? "RT " : "", + virUUIDFormat(hvUUID, uuidstr), dom->def->name); + + uuid_copy(priv->hvUUID, hvUUID); + + event = virDomainEventLifecycleNewFromObj(dom, + VIR_DOMAIN_EVENT_DEFINED, + VIR_DOMAIN_EVENT_DEFINED_ADDED); + if (!event) + return -1; + + virObjectEventStateQueue(acrn_driver->domainEventState, event); + return 0; +} + static virDrvStateInitResult acrnStateInitialize(bool privileged, const char *root, @@ -2705,6 +2735,7 @@ acrnStateInitialize(bool privileged, void *opaque G_GNUC_UNUSED) { int ret; + struct acrnVmList *list = NULL; if (root) { virReportError(VIR_ERR_INVALID_ARG, "%s", @@ -2755,6 +2786,7 @@ acrnStateInitialize(bool privileged, if (!(acrn_driver->hostdevMgr = virHostdevManagerGetDefault())) goto cleanup; + /* load inactive persistent configs */ if (virDomainObjListLoadAllConfigs(acrn_driver->domains, ACRN_CONFIG_DIR, ACRN_AUTOSTART_DIR, false, @@ -2762,12 +2794,25 @@ acrnStateInitialize(bool privileged, NULL, NULL) < 0) goto cleanup; + list = acrnVmListNew(); + if (!list) + goto cleanup; + + if (acrnGetPlatform(&acrn_driver->pi, list) < 0) + goto cleanup; + + if (virDomainObjListForEach(acrn_driver->domains, false, + acrnPersistentDomainInit, list) < 0) + goto cleanup; + + acrnVmListFree(list); return VIR_DRV_STATE_INIT_COMPLETE; cleanup: ret = VIR_DRV_STATE_INIT_ERROR; cleanup_nofail: acrnStateCleanup(); + acrnVmListFree(list); return ret; } From dd039f1626b541302b33573e66171a2587993b28 Mon Sep 17 00:00:00 2001 From: Helmut Buchsbaum Date: Tue, 23 Jun 2020 11:27:32 +0200 Subject: [PATCH 12/32] acrn_driver: avoid duplicate read of ACRN platform data during init Since the ACRN vm list has to be obtained in acrnStateInitialize() for iterating through all domains, reuse this list for acrnInitPlatform() to avoid unnecessary duplicate read of ACRN platform data during initialization. Signed-off-by: Helmut Buchsbaum --- src/acrn/acrn_driver.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 8c3970a913..8dfc9e9803 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -2615,21 +2615,13 @@ acrnOfflineCpus(int nprocs, virBitmapPtr pcpus, size_t *allocMap) static int acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, - size_t **allocMap) + size_t **allocMap, const struct acrnVmList *list) { - struct acrnVmList *list; virBitmapPtr postLaunchedPcpus = NULL; uint16_t totalCpus; size_t i, *map = NULL; int ret; - if (!(list = acrnVmListNew())) - return -ENOMEM; - - ret = acrnGetPlatform(pi, list); - if (ret < 0) - goto cleanup; - totalCpus = pi->cpu_num; if (!(postLaunchedPcpus = virBitmapNew(totalCpus))) { @@ -2694,7 +2686,6 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, if (map) VIR_FREE(map); virBitmapFree(postLaunchedPcpus); - acrnVmListFree(list); return ret; } @@ -2760,8 +2751,11 @@ acrnStateInitialize(bool privileged, if (virCapabilitiesGetNodeInfo(&acrn_driver->nodeInfo) < 0) goto cleanup; - ret = acrnInitPlatform(&acrn_driver->pi, &acrn_driver->nodeInfo, - &acrn_driver->vcpuAllocMap); + list = acrnVmListNew(); + if (!list) + goto cleanup; + + ret = acrnGetPlatform(&acrn_driver->pi, list); if (ret == -ENODEV) { /* we are not running on an ACRN enabled system */ VIR_INFO("ACRN hypervisor not available, disabling driver"); @@ -2771,6 +2765,10 @@ acrnStateInitialize(bool privileged, if (ret < 0) goto cleanup; + if (acrnInitPlatform(&acrn_driver->pi, &acrn_driver->nodeInfo, + &acrn_driver->vcpuAllocMap, list) < 0) + goto cleanup; + if (!(acrn_driver->domains = virDomainObjListNew())) goto cleanup; @@ -2794,13 +2792,6 @@ acrnStateInitialize(bool privileged, NULL, NULL) < 0) goto cleanup; - list = acrnVmListNew(); - if (!list) - goto cleanup; - - if (acrnGetPlatform(&acrn_driver->pi, list) < 0) - goto cleanup; - if (virDomainObjListForEach(acrn_driver->domains, false, acrnPersistentDomainInit, list) < 0) goto cleanup; From 8b24c6b7a5643fe39e7cfc0de082341f83f6457f Mon Sep 17 00:00:00 2001 From: Peter Fang Date: Thu, 17 Dec 2020 01:55:55 -0800 Subject: [PATCH 13/32] acrn: support the tag when creating a tap device Allow an XML file to configure the MTU size of a tap device. This is required to properly support VXLAN in OpenStack. Signed-off-by: Peter Fang --- src/acrn/acrn_driver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 8dfc9e9803..0f8915bcf2 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -665,7 +665,7 @@ acrnCreateTapDev(virDomainNetDefPtr net, const unsigned char *uuid) virDomainNetGetActualVirtPortProfile(net), virDomainNetGetActualVlan(net), virDomainNetGetActualPortOptionsIsolated(net), - NULL, 0, NULL, + NULL, net->mtu, NULL, VIR_NETDEV_TAP_CREATE_IFUP | VIR_NETDEV_TAP_CREATE_PERSIST) < 0) { virReportError(VIR_WAR_NO_NETWORK, "%s", net->ifname); From 6e47904e410b43720b525e82bbb6f063b9dc398c Mon Sep 17 00:00:00 2001 From: Xiangyang Wu Date: Tue, 7 Sep 2021 11:23:07 +0800 Subject: [PATCH 14/32] acrn: update getting platform info and offline cpu Update platform information data sturcture since it is updated in the kernel v5.10. Update offline cpu path since it since it changes to /sys/devices/virtual/misc/acrn_hsm/remove_cpu for kernel v5.10. Tracked-On: projectacrn/acrn-hypervisor#6564 Signed-off-by: Xiangyang Wu --- src/acrn/acrn_common.h | 80 ++++++++++++++++++------------------------ src/acrn/acrn_driver.c | 43 ++++++++++++----------- 2 files changed, 57 insertions(+), 66 deletions(-) diff --git a/src/acrn/acrn_common.h b/src/acrn/acrn_common.h index 90e66623a7..d39d5421d8 100644 --- a/src/acrn/acrn_common.h +++ b/src/acrn/acrn_common.h @@ -4,10 +4,9 @@ #define _IC_ID(x, y) (((x)<<24)|(y)) #define IC_ID 0x43UL -/* General */ -#define IC_ID_GEN_BASE 0x0UL -#define IC_GET_PLATFORM_INFO _IC_ID(IC_ID, IC_ID_GEN_BASE + 0x03) +#define ACRN_IOCTL_TYPE 0xA2 +#define IC_GET_PLATFORM_INFO _IOR(ACRN_IOCTL_TYPE, 0x03, struct acrn_platform_info) /* ACRN guest severity */ enum acrn_vm_severity { SEVERITY_SAFETY_VM = 0x40U, @@ -45,48 +44,39 @@ struct acrn_vm_config { */ } __attribute__((aligned(8))); -typedef struct platform_info acrnPlatformInfo; +typedef struct acrn_platform_info acrnPlatformInfo; typedef acrnPlatformInfo *acrnPlatformInfoPtr; -struct platform_info { - /** Hardware Information */ - /** Physical CPU number */ - uint16_t cpu_num; +#define ACRN_PLATFORM_LAPIC_IDS_MAX 64 +struct acrn_platform_info { + struct { + /** Physical CPU number of the platform */ + __u16 cpu_num; + /** Version of this structure */ + __u16 version; + /** Order of the number of threads sharing L2 cache */ + __u32 l2_cat_shift; + /** Order of the number of threads sharing L3 cache */ + __u32 l3_cat_shift; + /** IDs of LAPICs of all threads */ + __u8 lapic_ids[ACRN_PLATFORM_LAPIC_IDS_MAX]; + /** Reserved for alignment and should be 0 */ + __u8 reserved[52]; + } hw; - /** version of this structure */ - uint16_t version; + struct { + /** Maximum number of vCPU of a VM */ + __u16 max_vcpus_per_vm; + /** Maximum number of VM */ + __u16 max_vms; + /** Size of configuration of a VM */ + __u32 vm_config_size; - /** Align the size of version & hardware info to 128Bytes. */ - uint8_t reserved0[124]; - - /** Configuration Information */ - /** Maximum vCPU number for one VM. */ - uint16_t max_vcpus_per_vm; - - /** Maximum Kata container number in SOS VM */ - uint8_t max_kata_containers; - - uint8_t reserved1[7]; - - /** Number of configured VMs */ - uint16_t max_vms; - - /** - * The size of acrn_vm_config is various on different platforms. - * This is the size of this struct which is used for the caller - * to parse the vm_configs array. - */ - uint32_t vm_config_entry_size; - - /** - * Address to an array of struct acrn_vm_config, containing all - * the configurations of all VMs. VHM treats it as an opague data - * structure. - * * - * The size of one array element is vm_config_entry_size while - * the number of elements is max_vms. - */ - uint64_t vm_configs_addr; - - /** Align the size of Configuration info to 128Bytes. */ - uint8_t reserved2[104]; -} __attribute__((aligned(8))); + /** Memory address which user space provided to + * store the VM configurations + */ + void *vm_configs_addr; + /** Maximum number of VM for Kata containers */ + __u64 max_kata_containers; + __u8 reserved[104]; + } sw; +}; diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 0f8915bcf2..5b1ae47bc1 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -30,7 +30,7 @@ #define VIR_FROM_THIS VIR_FROM_ACRN #define ACRN_DM_PATH "/usr/bin/acrn-dm" #define ACRN_CTL_PATH "/usr/bin/acrnctl" -#define ACRN_OFFLINE_PATH "/sys/class/vhm/acrn_vhm/offline_cpu" +#define ACRN_OFFLINE_PATH "/sys/devices/virtual/misc/acrn_hsm/remove_cpu" #define SYSFS_CPU_PATH "/sys/devices/system/cpu" #define ACRN_AUTOSTART_DIR SYSCONFDIR "/libvirt/acrn/autostart" #define ACRN_CONFIG_DIR SYSCONFDIR "/libvirt/acrn" @@ -184,27 +184,28 @@ acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) } /* get basic platform info first */ - if (!pi->vm_configs_addr) { + if (!pi->sw.vm_configs_addr) { if (acrnGetPlatformInfo(fd, pi) < 0 || - !pi->cpu_num || !pi->max_vms || !pi->vm_config_entry_size) { + !pi->sw.max_vms || !pi->sw.vm_config_size) { virReportError(VIR_ERR_INTERNAL_ERROR, - _("acrnGetPlatformInfo failed")); + _("acrnGetPlatformInfo first time failed")); + VIR_DEBUG("acrnGetPlatformInfo:max_vms=0x%x\n", pi->sw.max_vms); ret = -EINVAL; goto cleanup; } - if (pi->version != ACRN_PI_VERSION) { + if (pi->hw.version != ACRN_PI_VERSION) { virReportError(VIR_ERR_INTERNAL_ERROR, _("ACRN platform version mismatch: " "got 0x%x, expecting 0x%x"), - pi->version, ACRN_PI_VERSION); + pi->hw.version, ACRN_PI_VERSION); ret = -EOPNOTSUPP; goto cleanup; } - if (!(pi->vm_configs_addr = (uint64_t)calloc( - pi->max_vms, - pi->vm_config_entry_size))) { + if (!(pi->sw.vm_configs_addr = calloc( + pi->sw.max_vms, + pi->sw.vm_config_size))) { virReportError(VIR_ERR_NO_MEMORY, NULL); ret = -ENOMEM; goto cleanup; @@ -215,13 +216,13 @@ acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) ret = acrnGetPlatformInfo(fd, pi); if (ret < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, - _("acrnGetPlatformInfo failed")); + _("acrnGetPlatformInfo second time failed")); goto cleanup; } - for (i = 0, p = (uint8_t *)pi->vm_configs_addr; - i < pi->max_vms; - i++, p += pi->vm_config_entry_size) { + for (i = 0, p = (uint8_t *)pi->sw.vm_configs_addr; + i < pi->sw.max_vms; + i++, p += pi->sw.vm_config_size) { /* drop the hv-specific part of vmcfg */ memcpy(&vmcfg, p, sizeof(vmcfg)); @@ -357,7 +358,7 @@ acrnAllocateVm(virDomainObjListPtr doms, virDomainDefPtr def, } /* clamp cpumask to cpu_num */ - virBitmapShrink(cpumask, pi->cpu_num); + virBitmapShrink(cpumask, pi->hw.cpu_num); if (!(testmask = virBitmapNew(virBitmapSize(cpumask)))) { virReportError(VIR_ERR_NO_MEMORY, NULL); @@ -497,7 +498,7 @@ acrnAllocateVcpus(acrnPlatformInfoPtr pi, virBitmapPtr pcpus, bool rtvm, size_t maxvcpus, size_t *allocMap, virBitmapPtr vcpus) { ssize_t pos; - uint16_t totalCpus = pi->cpu_num; + uint16_t totalCpus = pi->hw.cpu_num; while (maxvcpus--) { uint16_t minAllocated = USHRT_MAX; @@ -594,7 +595,7 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, if (def->cpumask) { /* clamp cpumask to cpu_num */ - virBitmapShrink(def->cpumask, pi->cpu_num); + virBitmapShrink(def->cpumask, pi->hw.cpu_num); if (!(allowedmask = virBitmapNewCopy(def->cpumask))) { virReportError(VIR_ERR_INTERNAL_ERROR, @@ -614,7 +615,7 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, if (priv->cpuAffinitySet) virBitmapFree(priv->cpuAffinitySet); - if (!(priv->cpuAffinitySet = virBitmapNew(pi->cpu_num))) { + if (!(priv->cpuAffinitySet = virBitmapNew(pi->hw.cpu_num))) { virReportError(VIR_ERR_NO_MEMORY, NULL); goto cleanup; } @@ -2459,7 +2460,7 @@ acrnNodeGetCPUMap(virConnectPtr conn, if (online) *online = virBitmapCountBits(cpus); - ret = pi->cpu_num; + ret = pi->hw.cpu_num; cleanup: if (ret < 0 && cpumap && *cpumap) @@ -2482,8 +2483,8 @@ acrnStateCleanup(void) virObjectUnref(acrn_driver->xmlopt); virObjectUnref(acrn_driver->caps); virObjectUnref(acrn_driver->domains); - if (acrn_driver->pi.vm_configs_addr) { - void *p = (void *)acrn_driver->pi.vm_configs_addr; + if (acrn_driver->pi.sw.vm_configs_addr) { + void *p = (void *)acrn_driver->pi.sw.vm_configs_addr; VIR_FREE(p); } if (acrn_driver->vcpuAllocMap) @@ -2622,7 +2623,7 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, size_t i, *map = NULL; int ret; - totalCpus = pi->cpu_num; + totalCpus = pi->hw.cpu_num; if (!(postLaunchedPcpus = virBitmapNew(totalCpus))) { virReportError(VIR_ERR_NO_MEMORY, NULL); From 9ae2926a589e9b9752aefddcfd16c250b754fd28 Mon Sep 17 00:00:00 2001 From: Xiangyang Wu Date: Tue, 17 Aug 2021 13:41:04 +0800 Subject: [PATCH 15/32] acrn-driver: add call back to check guest persistent To support guest domain auto-start, the guest domain should be persistent. In the current ACRN driver, user can't get persistent status of a guest domain since related call back is not implemented in the ACRN driver. There is a bug about persistent domain initialization since acrnPersistentDomainInit is invoked for any guest domain. In this patch, add a call back to check guest persistent state, and make acrnPersistentDomainInit only to initialize persistent guest domain. Signed-off-by: Xiangyang Wu --- src/acrn/acrn_driver.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 5b1ae47bc1..21db223550 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -1340,6 +1340,25 @@ acrnDomainDestroy(virDomainPtr dom) return ret; } +static int +acrnDomainIsPersistent(virDomainPtr domain) +{ + virDomainObjPtr obj; + int ret = -1; + + if (!(obj = acrnDomObjFromDomain(domain))) + goto cleanup; + + if (virDomainIsPersistentEnsureACL(domain->conn, obj->def) < 0) + goto cleanup; + + ret = obj->persistent; + + cleanup: + virDomainObjEndAPI(&obj); + return ret; +} + static int acrnDomainGetInfo(virDomainPtr dom, virDomainInfoPtr info) { @@ -2700,6 +2719,9 @@ acrnPersistentDomainInit(virDomainObjPtr dom, void *opaque) acrnDomainObjPrivatePtr priv = dom->privateData; virObjectEventPtr event = NULL; + if (!dom->persistent) + return -1; + if (acrnAllocateVm(acrn_driver->domains, dom->def, &acrn_driver->pi, vmList, hvUUID) < 0) return -1; @@ -2824,6 +2846,7 @@ static virHypervisorDriver acrnHypervisorDriver = { .domainLookupByName = acrnDomainLookupByName, /* 0.0.1 */ .domainShutdown = acrnDomainShutdown, /* 0.0.1 */ .domainDestroy = acrnDomainDestroy, /* 0.0.1 */ + .domainIsPersistent = acrnDomainIsPersistent, /* 0.0.1 */ .domainGetInfo = acrnDomainGetInfo, /* 0.0.1 */ .domainGetState = acrnDomainGetState, /* 0.0.1 */ .domainGetVcpus = acrnDomainGetVcpus, /* 0.0.1 */ From 405ae15635cd4248c5556e3656a72c3aa28ccd21 Mon Sep 17 00:00:00 2001 From: Xiangyang Wu Date: Tue, 17 Aug 2021 13:54:16 +0800 Subject: [PATCH 16/32] acrn-driver: enable related auto start commands Enable and disable the automatic starting of a guest domain when the libvirt daemon starts. Signed-off-by: Xiangyang Wu --- src/acrn/acrn_driver.c | 164 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 21db223550..5ca4b1a644 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -34,6 +34,7 @@ #define SYSFS_CPU_PATH "/sys/devices/system/cpu" #define ACRN_AUTOSTART_DIR SYSCONFDIR "/libvirt/acrn/autostart" #define ACRN_CONFIG_DIR SYSCONFDIR "/libvirt/acrn" +#define ACRN_STATE_DIR RUNSTATEDIR "/libvirt/acrn" #define ACRN_NET_GENERATED_TAP_PREFIX "tap" #define ACRN_PI_VERSION (0x100) @@ -71,6 +72,11 @@ struct acrnVmList { size_t size; }; +struct acrnAutostartData { + acrnConnectPtr driver; + virConnectPtr conn; + struct acrnVmList *vmlist; +}; static acrnConnectPtr acrn_driver = NULL; static void @@ -1145,7 +1151,60 @@ acrnProcessStart(virDomainObjPtr vm) } return ret; } +static int +acrnAutostartDomain(virDomainObjPtr vm, void *opaque) +{ + const struct acrnAutostartData *data = opaque; + int ret = 0; + acrnConnectPtr privconn = data->driver; + acrnDomainObjPrivatePtr priv; + struct acrnVmList *vmList = data->vmlist; + ssize_t idx; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + virObjectLock(vm); + if (vm->autostart && !virDomainObjIsActive(vm)) { + virResetLastError(); + priv = vm->privateData; + /* find the allocated VM */ + if ((idx = acrnFindVm(vmList, priv->hvUUID)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("vm(%s) not found"), + virUUIDFormat(priv->hvUUID, uuidstr)); + goto cleanup; + } + + if (acrnProcessPrepareDomain(vm, &privconn->pi, &vmList->vm[idx], + privconn->vcpuAllocMap) < 0) + goto cleanup; + if (acrnProcessStart(vm) < 0) { + /* domain must be persistent */ + acrnFreeVcpus(priv->cpuAffinitySet, privconn->vcpuAllocMap); + goto cleanup; + } + if (ret < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Failed to autostart VM '%s': %s"), + vm->def->name, virGetLastErrorMessage()); + } + } +cleanup: + virObjectUnlock(vm); + return ret; +} + +static void +acrnAutostartDomains(acrnConnectPtr driver, struct acrnVmList *vmlist) +{ + virConnectPtr conn = virConnectOpen("acrn:///system"); + /* Ignoring NULL conn which is mostly harmless here */ + + struct acrnAutostartData data = { driver, conn, vmlist}; + virDomainObjListForEach(driver->domains, false, acrnAutostartDomain, &data); + + virObjectUnref(conn); +} static virCommandPtr acrnBuildStopCmd(virDomainDefPtr def) { @@ -1359,6 +1418,89 @@ acrnDomainIsPersistent(virDomainPtr domain) return ret; } +static int +acrnDomainGetAutostart(virDomainPtr domain, int *autostart) +{ + virDomainObjPtr vm; + int ret = -1; + + if (!(vm = acrnDomObjFromDomain(domain))) + goto cleanup; + + if (virDomainGetAutostartEnsureACL(domain->conn, vm->def) < 0) + goto cleanup; + + *autostart = vm->autostart; + ret = 0; + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + +static int +acrnDomainSetAutostart(virDomainPtr domain, int autostart) +{ + virDomainObjPtr vm; + char *configFile = NULL; + char *autostartLink = NULL; + int ret = -1; + + if (!(vm = acrnDomObjFromDomain(domain))) + goto cleanup; + + if (virDomainSetAutostartEnsureACL(domain->conn, vm->def) < 0) + goto cleanup; + + if (!vm->persistent) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cannot set autostart for transient domain")); + goto cleanup; + } + + autostart = (autostart != 0); + + if (vm->autostart != autostart) { + if ((configFile = virDomainConfigFile(ACRN_CONFIG_DIR, vm->def->name)) == NULL) + goto cleanup; + if ((autostartLink = virDomainConfigFile(ACRN_AUTOSTART_DIR, vm->def->name)) == NULL) + goto cleanup; + + if (autostart) { + if (virFileMakePath(ACRN_AUTOSTART_DIR) < 0) { + virReportSystemError(errno, + _("cannot create autostart directory %s"), + ACRN_AUTOSTART_DIR); + goto cleanup; + } + + if (symlink(configFile, autostartLink) < 0) { + virReportSystemError(errno, + _("Failed to create symlink '%s' to '%s'"), + autostartLink, configFile); + goto cleanup; + } + } else { + if (unlink(autostartLink) < 0 && errno != ENOENT && errno != ENOTDIR) { + virReportSystemError(errno, + _("Failed to delete symlink '%s'"), + autostartLink); + goto cleanup; + } + } + + vm->autostart = autostart; + } + + ret = 0; + + cleanup: + VIR_FREE(configFile); + VIR_FREE(autostartLink); + virDomainObjEndAPI(&vm); + return ret; +} + static int acrnDomainGetInfo(virDomainPtr dom, virDomainInfoPtr info) { @@ -2750,6 +2892,7 @@ acrnStateInitialize(bool privileged, { int ret; struct acrnVmList *list = NULL; + bool autostart = true; if (root) { virReportError(VIR_ERR_INVALID_ARG, "%s", @@ -2807,6 +2950,19 @@ acrnStateInitialize(bool privileged, if (!(acrn_driver->hostdevMgr = virHostdevManagerGetDefault())) goto cleanup; + if (virFileMakePath(ACRN_STATE_DIR) < 0) { + virReportSystemError(errno, + _("Failed to mkdir %s"), + ACRN_STATE_DIR); + goto cleanup; + } + + if (virDomainObjListLoadAllConfigs(acrn_driver->domains, + ACRN_STATE_DIR, + NULL, true, + acrn_driver->xmlopt, + NULL, NULL) < 0) + goto cleanup; /* load inactive persistent configs */ if (virDomainObjListLoadAllConfigs(acrn_driver->domains, ACRN_CONFIG_DIR, @@ -2819,6 +2975,12 @@ acrnStateInitialize(bool privileged, acrnPersistentDomainInit, list) < 0) goto cleanup; + if (virDriverShouldAutostart(ACRN_STATE_DIR, &autostart) < 0) + goto cleanup; + + if (autostart) + acrnAutostartDomains(acrn_driver, list); + acrnVmListFree(list); return VIR_DRV_STATE_INIT_COMPLETE; @@ -2847,6 +3009,8 @@ static virHypervisorDriver acrnHypervisorDriver = { .domainShutdown = acrnDomainShutdown, /* 0.0.1 */ .domainDestroy = acrnDomainDestroy, /* 0.0.1 */ .domainIsPersistent = acrnDomainIsPersistent, /* 0.0.1 */ + .domainGetAutostart = acrnDomainGetAutostart, /* 0.0.1 */ + .domainSetAutostart = acrnDomainSetAutostart, /* 0.0.1 */ .domainGetInfo = acrnDomainGetInfo, /* 0.0.1 */ .domainGetState = acrnDomainGetState, /* 0.0.1 */ .domainGetVcpus = acrnDomainGetVcpus, /* 0.0.1 */ From 6e615223992bfa3d220dbf6994b299b8a55a6288 Mon Sep 17 00:00:00 2001 From: Xiangyang Wu Date: Wed, 3 Nov 2021 23:33:51 +0800 Subject: [PATCH 17/32] acrn-driver: update calculation of cpu affinity and UUID Currently, acrn driver uses GET_PLATFORM_INFO ioctl in ACRN to get VM configuration, then calculate cpu affinity and UUID for user VM. GET_PLATFORM_INFO ioctl will be not supported in ACRN v2.7, this patch will refine calculation logic of cpu affinity and UUID. The following changes are made: - Get cpu affiniity from cpuset in XML file directly. - Get UUID from XML file directly. - Remove get platform information related code and vm list code. Todo: will refine vcpu allocation logic to support vcpu allocation when users only specify cpu number in XML file. Will add some checks for RTVM since this VM needs exclusive vcpu. Tracked-On: projectacrn/acrn-hypervisor#6724 Signed-off-by: Xiangyang Wu --- src/acrn/acrn_common.h | 82 ----- src/acrn/acrn_driver.c | 713 +++-------------------------------------- 2 files changed, 47 insertions(+), 748 deletions(-) delete mode 100644 src/acrn/acrn_common.h diff --git a/src/acrn/acrn_common.h b/src/acrn/acrn_common.h deleted file mode 100644 index d39d5421d8..0000000000 --- a/src/acrn/acrn_common.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Commmon IOCTL ID defination for VHM/DM - */ -#define _IC_ID(x, y) (((x)<<24)|(y)) -#define IC_ID 0x43UL - -#define ACRN_IOCTL_TYPE 0xA2 - -#define IC_GET_PLATFORM_INFO _IOR(ACRN_IOCTL_TYPE, 0x03, struct acrn_platform_info) -/* ACRN guest severity */ -enum acrn_vm_severity { - SEVERITY_SAFETY_VM = 0x40U, - SEVERITY_RTVM = 0x30U, - SEVERITY_SOS = 0x20U, - SEVERITY_STANDARD_VM = 0x10U, -}; - -enum acrn_vm_load_order { - PRE_LAUNCHED_VM = 0, - SOS_VM, - POST_LAUNCHED_VM, - MAX_LOAD_ORDER -}; - -#define MAX_VM_OS_NAME_LEN 32U - -typedef struct acrn_vm_config acrnVmCfg; -typedef acrnVmCfg *acrnVmCfgPtr; -struct acrn_vm_config { - enum acrn_vm_load_order load_order; /* specify the load order of VM */ - char name[MAX_VM_OS_NAME_LEN]; /* VM name identifier, useful for debug. */ - const uint8_t uuid[16]; /* UUID of the VM */ - uint8_t reserved[2]; /* Temporarily reserve it so that don't need to update - * the users of get_platform_info frequently. - */ - uint8_t severity; /* severity of the VM */ - uint64_t cpu_affinity; /* The set bits represent the pCPUs the vCPUs of - * the VM may run on. - */ - uint64_t guest_flags; /* VM flags that we want to configure for guest */ - /* - * The following are hv-specific members and are thus opaque. - * vm_config_entry_size determines the real size of this structure. - */ -} __attribute__((aligned(8))); - -typedef struct acrn_platform_info acrnPlatformInfo; -typedef acrnPlatformInfo *acrnPlatformInfoPtr; -#define ACRN_PLATFORM_LAPIC_IDS_MAX 64 -struct acrn_platform_info { - struct { - /** Physical CPU number of the platform */ - __u16 cpu_num; - /** Version of this structure */ - __u16 version; - /** Order of the number of threads sharing L2 cache */ - __u32 l2_cat_shift; - /** Order of the number of threads sharing L3 cache */ - __u32 l3_cat_shift; - /** IDs of LAPICs of all threads */ - __u8 lapic_ids[ACRN_PLATFORM_LAPIC_IDS_MAX]; - /** Reserved for alignment and should be 0 */ - __u8 reserved[52]; - } hw; - - struct { - /** Maximum number of vCPU of a VM */ - __u16 max_vcpus_per_vm; - /** Maximum number of VM */ - __u16 max_vms; - /** Size of configuration of a VM */ - __u32 vm_config_size; - - /** Memory address which user space provided to - * store the VM configurations - */ - void *vm_configs_addr; - /** Maximum number of VM for Kata containers */ - __u64 max_kata_containers; - __u8 reserved[104]; - } sw; -}; diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 5ca4b1a644..1bf5b5018b 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -23,7 +23,7 @@ #include "virfdstream.h" #include "virlog.h" #include "domain_event.h" -#include "acrn_common.h" +#include "viraccessapicheck.h" #include "acrn_driver.h" #include "acrn_domain.h" @@ -50,7 +50,6 @@ struct _acrnConnect { virDomainXMLOptionPtr xmlopt; virObjectEventStatePtr domainEventState; virHostdevManagerPtr hostdevMgr; - acrnPlatformInfo pi; size_t *vcpuAllocMap; }; @@ -63,19 +62,9 @@ struct _acrnDomainNamespaceDef { #define MAX_NUM_VMS (64) -struct acrnVmList { - struct acrnVmEntry { - acrnVmCfg cfg; - int vcpu_num; - virBitmapPtr pcpus; - } vm[MAX_NUM_VMS]; - size_t size; -}; - struct acrnAutostartData { acrnConnectPtr driver; virConnectPtr conn; - struct acrnVmList *vmlist; }; static acrnConnectPtr acrn_driver = NULL; @@ -125,209 +114,6 @@ acrnDomObjFromDomain(virDomainPtr domain) return vm; } -static struct acrnVmList * -acrnVmListNew(void) -{ - struct acrnVmList *list; - - if (VIR_ALLOC(list) < 0) - return NULL; - - return list; -} - -static void -acrnVmListFree(struct acrnVmList *list) -{ - size_t i; - - if (!list) - return; - - for (i = 0; i < list->size; i++) - virBitmapFree(list->vm[i].pcpus); - VIR_FREE(list); -} - -static int -acrnGetVhmFd(void) -{ - struct stat st; - int fd = -1; - - if (!stat("/dev/acrn_vhm", &st)) - fd = open("/dev/acrn_vhm", O_RDWR|O_CLOEXEC); - else if (!stat("/dev/acrn_hsm", &st)) - fd = open("/dev/acrn_hsm", O_RDWR|O_CLOEXEC); - - return fd; -} - -static int -acrnGetPlatformInfo(int fd, acrnPlatformInfoPtr pi) -{ - return ioctl(fd, IC_GET_PLATFORM_INFO, pi); -} - -static int -acrnGetPlatform(acrnPlatformInfoPtr pi, struct acrnVmList *vmList) -{ - acrnVmCfg vmcfg; - int fd, vcpu_num, pos, ret; - uint8_t *p; - uint16_t i, j; - uint64_t pcpus; - char uuidstr[VIR_UUID_STRING_BUFLEN]; - - if (!pi || !vmList) - return 0; - - vmList->size = 0; - - if ((fd = acrnGetVhmFd()) < 0) { - ret = -ENODEV; - goto cleanup; - } - - /* get basic platform info first */ - if (!pi->sw.vm_configs_addr) { - if (acrnGetPlatformInfo(fd, pi) < 0 || - !pi->sw.max_vms || !pi->sw.vm_config_size) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("acrnGetPlatformInfo first time failed")); - VIR_DEBUG("acrnGetPlatformInfo:max_vms=0x%x\n", pi->sw.max_vms); - ret = -EINVAL; - goto cleanup; - } - - if (pi->hw.version != ACRN_PI_VERSION) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("ACRN platform version mismatch: " - "got 0x%x, expecting 0x%x"), - pi->hw.version, ACRN_PI_VERSION); - ret = -EOPNOTSUPP; - goto cleanup; - } - - if (!(pi->sw.vm_configs_addr = calloc( - pi->sw.max_vms, - pi->sw.vm_config_size))) { - virReportError(VIR_ERR_NO_MEMORY, NULL); - ret = -ENOMEM; - goto cleanup; - } - } - - /* now get vm config */ - ret = acrnGetPlatformInfo(fd, pi); - if (ret < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("acrnGetPlatformInfo second time failed")); - goto cleanup; - } - - for (i = 0, p = (uint8_t *)pi->sw.vm_configs_addr; - i < pi->sw.max_vms; - i++, p += pi->sw.vm_config_size) { - /* drop the hv-specific part of vmcfg */ - memcpy(&vmcfg, p, sizeof(vmcfg)); - - if (virUUIDIsValid(vmcfg.uuid)) { - if (vmList->size == G_N_ELEMENTS(vmList->vm)) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("only %lu VMs are supported"), - G_N_ELEMENTS(vmList->vm)); - ret = -EINVAL; - goto cleanup; - } - - if (!(pcpus = vmcfg.cpu_affinity)) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("no pCPU in vm[%u]"), i); - ret = -EINVAL; - goto cleanup; - } - - vcpu_num = __builtin_popcountl(pcpus); - - /* insertion sort based on vcpu_num */ - for (j = 0; j < vmList->size; j++) { - if (vcpu_num < vmList->vm[j].vcpu_num) - break; - } - - if (j < vmList->size) - memmove(&vmList->vm[j+1], &vmList->vm[j], - sizeof(vmList->vm[j]) * (vmList->size - j)); - - memcpy(&vmList->vm[j].cfg, &vmcfg, sizeof(vmcfg)); - - if (!(vmList->vm[j].pcpus = - virBitmapNew( - sizeof(vmcfg.cpu_affinity) * CHAR_BIT))) { - virReportError(VIR_ERR_NO_MEMORY, NULL); - ret = -ENOMEM; - goto cleanup; - } - - /* convert cpu_affinity to virBitmap */ - while ((pos = __builtin_ffsl(pcpus)) > 0) { - pos--; - - if (virBitmapSetBit(vmList->vm[j].pcpus, pos) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("virBitmapSetBit failed")); - ret = -EINVAL; - goto cleanup; - } - pcpus &= ~(1ULL << pos); - } - - vmList->vm[j].vcpu_num = vcpu_num; - vmList->size++; - } - } - - for (i = 0; i < vmList->size; i++) - VIR_DEBUG("vm[%u] (%s): order: %d, uuid: %s, severity: 0x%x, " - "pCPU map: 0x%lx (%d vCPUs)", - i, vmList->vm[i].cfg.name, - vmList->vm[i].cfg.load_order, - virUUIDFormat(vmList->vm[i].cfg.uuid, uuidstr), - vmList->vm[i].cfg.severity, - vmList->vm[i].cfg.cpu_affinity, - vmList->vm[i].vcpu_num); - - ret = 0; - -cleanup: - if (fd >= 0) - close(fd); - return ret; -} - -struct acrnFindUUIDData { - const unsigned char *uuid; -}; - -static int -acrnFindHvUUID(virDomainObjPtr vm, void *opaque) -{ - struct acrnFindUUIDData *data = opaque; - acrnDomainObjPrivatePtr priv; - int ret = 0; - - virObjectLock(vm); - - priv = vm->privateData; - - if (!uuid_compare(priv->hvUUID, data->uuid)) - ret = -1; - - virObjectUnlock(vm); - return ret; -} - static bool acrnIsRtvm(virDomainDefPtr def) { @@ -336,213 +122,31 @@ acrnIsRtvm(virDomainDefPtr def) return (nsdef && nsdef->rtvm); } -/* - * This function must not be called with any virDomainObjPtr - * lock held, as it can attempt to hold any such lock in doms. - */ -static ssize_t -acrnAllocateVm(virDomainObjListPtr doms, virDomainDefPtr def, - acrnPlatformInfoPtr pi, struct acrnVmList *vmList, - unsigned char *uuid) -{ - enum acrn_vm_severity severity; - struct acrnFindUUIDData data; - virBitmapPtr cpumask = NULL, testmask = NULL; - ssize_t i, start, candidate = -1; - size_t nvcpus, maxVcpusFit = 0; - char *maskstr = NULL; - char uuidstr[VIR_UUID_STRING_BUFLEN]; - - severity = (acrnIsRtvm(def)) ? SEVERITY_RTVM : SEVERITY_STANDARD_VM; - - if (def->cpumask) { - /* prepare a sanitized cpumask */ - if (!(cpumask = virBitmapNewCopy(def->cpumask))) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("virBitmapNewCopy failed")); - goto notfound; - } - - /* clamp cpumask to cpu_num */ - virBitmapShrink(cpumask, pi->hw.cpu_num); - - if (!(testmask = virBitmapNew(virBitmapSize(cpumask)))) { - virReportError(VIR_ERR_NO_MEMORY, NULL); - goto notfound; - } - } - - /* determine where to begin the search, based on vcpu_num */ - for (i = 0; i < vmList->size; i++) { - if (def->maxvcpus <= vmList->vm[i].vcpu_num) - break; - } - - start = i; - - /* these VMs can fit maxvcpus */ - for (; i < vmList->size; i++) { - if (vmList->vm[i].cfg.load_order == POST_LAUNCHED_VM && - vmList->vm[i].cfg.severity == severity) { - data.uuid = vmList->vm[i].cfg.uuid; - - if (!virDomainObjListForEach(doms, false, acrnFindHvUUID, &data)) { - if (!cpumask) - goto done; - - if (virBitmapCopy(testmask, cpumask) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("virBitmapCopy failed")); - goto notfound; - } - - virBitmapIntersect(testmask, vmList->vm[i].pcpus); - nvcpus = virBitmapCountBits(testmask); - - if (nvcpus >= def->maxvcpus) - goto done; - - /* search for max fit */ - if (nvcpus > maxVcpusFit) { - maxVcpusFit = nvcpus; - candidate = i; - } - } - } - } - - i = start; - - /* just try to find the best VM available */ - while (i--) { - if (vmList->vm[i].cfg.load_order == POST_LAUNCHED_VM && - vmList->vm[i].cfg.severity == severity) { - data.uuid = vmList->vm[i].cfg.uuid; - - if (!virDomainObjListForEach(doms, false, acrnFindHvUUID, &data)) { - if (!cpumask) - goto done; - - if (virBitmapCopy(testmask, cpumask) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("virBitmapCopy failed")); - goto notfound; - } - - virBitmapIntersect(testmask, vmList->vm[i].pcpus); - nvcpus = virBitmapCountBits(testmask); - - /* search for max fit */ - if (nvcpus >= maxVcpusFit) { - maxVcpusFit = nvcpus; - candidate = i; - } - } - } - } - - if (maxVcpusFit > 0) { - /* max fit found */ - i = candidate; - if (virBitmapCopy(testmask, cpumask) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("virBitmapCopy failed")); - goto notfound; - } - virBitmapIntersect(testmask, vmList->vm[i].pcpus); - goto done; - } - -notfound: - i = -1; - - if (def->cpumask) - maskstr = virBitmapFormat(def->cpumask); - - virReportError(VIR_ERR_INTERNAL_ERROR, - _("no suitable vm found (%lu max vcpus, cpumask = %s)"), - def->maxvcpus, - maskstr ? maskstr : "auto"); - -done: - if (i >= 0) { - if (testmask) - maskstr = virBitmapFormat(testmask); - else - maskstr = virBitmapFormat(vmList->vm[i].pcpus); - - VIR_DEBUG("vm(%s) allocated: uuid = %s, " - "%lu max vcpus, %s cpumask = %s", - vmList->vm[i].cfg.name, - virUUIDFormat(vmList->vm[i].cfg.uuid, uuidstr), - def->maxvcpus, - testmask ? "allowed" : "auto", - maskstr ? maskstr : "n/a"); - uuid_copy(uuid, vmList->vm[i].cfg.uuid); - } - virBitmapFree(cpumask); - virBitmapFree(testmask); - if (maskstr) - VIR_FREE(maskstr); - return i; -} - -static ssize_t -acrnFindVm(struct acrnVmList *vmList, unsigned char *uuid) -{ - size_t i; - - for (i = 0; i < vmList->size; i++) - if (!uuid_compare(vmList->vm[i].cfg.uuid, uuid)) - return (ssize_t)i; - - return -1; -} - static int -acrnAllocateVcpus(acrnPlatformInfoPtr pi, virBitmapPtr pcpus, bool rtvm, - size_t maxvcpus, size_t *allocMap, virBitmapPtr vcpus) +acrnAllocateVcpus(virBitmapPtr pcpus, size_t maxvcpus, + size_t *allocMap, virBitmapPtr vcpus) { ssize_t pos; - uint16_t totalCpus = pi->hw.cpu_num; - while (maxvcpus--) { - uint16_t minAllocated = USHRT_MAX; - uint16_t candidate = totalCpus; + if (maxvcpus == 0) + return -1; + pos = -1; - pos = -1; + /* successful - update allocation map */ + while (((pos = virBitmapNextSetBit(pcpus, pos)) >= 0) && (maxvcpus > 0)) { - /* find a pCPU that is least occupied */ - while ((pos = virBitmapNextSetBit(pcpus, pos)) >= 0 && - pos < totalCpus) { - if (!virBitmapIsBitSet(vcpus, pos) && - allocMap[pos] < minAllocated) { - minAllocated = allocMap[pos]; - candidate = pos; - } + if (pos >= acrn_driver->nodeInfo.cpus) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("pCPU[%ld] doesn't exist"), pos); + return -1; } - - /* all of the pCPUs in the VM have been allocated */ - if (candidate == totalCpus) - break; - - VIR_DEBUG("minAllocated = %u, candidate = %u", - minAllocated, candidate); - - if (virBitmapSetBit(vcpus, candidate) < 0 || - (rtvm && minAllocated > 0)) { + if (virBitmapSetBit(vcpus, pos) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, - _("vCPU placement failure")); + _("failed to set bit %ld in cpu affinity"), pos); return -1; } - } - - pos = -1; - - /* successful - update allocation map */ - while ((pos = virBitmapNextSetBit(vcpus, pos)) >= 0) { allocMap[pos] += 1; - + maxvcpus --; VIR_DEBUG("pCPU[%ld]: %lu vCPU%s allocated", pos, allocMap[pos], (allocMap[pos] > 1) ? "s" : ""); @@ -586,11 +190,9 @@ acrnSetOnlineVcpus(virDomainDefPtr def, virBitmapPtr vcpus) } static int -acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, - struct acrnVmEntry *entry, size_t *allocMap) +acrnProcessPrepareDomain(virDomainObjPtr vm, size_t *allocMap) { virDomainDefPtr def; - virBitmapPtr allowedmask = NULL; acrnDomainObjPrivatePtr priv; int ret = -1; @@ -598,38 +200,21 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, return -1; priv = vm->privateData; - - if (def->cpumask) { - /* clamp cpumask to cpu_num */ - virBitmapShrink(def->cpumask, pi->hw.cpu_num); - - if (!(allowedmask = virBitmapNewCopy(def->cpumask))) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("virBitmapNewCopy failed")); - goto cleanup; - } - - virBitmapIntersect(allowedmask, entry->pcpus); - - if (virBitmapIsAllClear(allowedmask)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("vm(%s) does not allow the given cpumask"), - entry->cfg.name); - goto cleanup; - } + if (def->cpumask == NULL || virBitmapIsAllClear(def->cpumask)) { + virReportError(VIR_ERR_INTERNAL_ERROR, _("cpuset is empty")); + goto cleanup; } - + virBitmapShrink(def->cpumask, acrn_driver->nodeInfo.cpus); if (priv->cpuAffinitySet) virBitmapFree(priv->cpuAffinitySet); - if (!(priv->cpuAffinitySet = virBitmapNew(pi->hw.cpu_num))) { + if (!(priv->cpuAffinitySet = virBitmapNew(acrn_driver->nodeInfo.cpus))) { virReportError(VIR_ERR_NO_MEMORY, NULL); goto cleanup; } /* vCPU placement */ - if (acrnAllocateVcpus(pi, - allowedmask ? allowedmask : entry->pcpus, - acrnIsRtvm(def), def->maxvcpus, allocMap, + if (acrnAllocateVcpus(def->cpumask, + def->maxvcpus, allocMap, priv->cpuAffinitySet) < 0) goto cleanup; @@ -647,7 +232,6 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, acrnPlatformInfoPtr pi, virBitmapFree(priv->cpuAffinitySet); priv->cpuAffinitySet = NULL; } - virBitmapFree(allowedmask); return ret; } @@ -1055,9 +639,9 @@ acrnBuildStartCmd(virDomainObjPtr vm) VIR_DIV_UP(virDomainDefGetMemoryInitial(def), 1024)); /* UUID */ - if (virUUIDIsValid(priv->hvUUID)) { + if (virUUIDIsValid(def->uuid)) { virCommandAddArg(cmd, "-U"); - virCommandAddArg(cmd, virUUIDFormat(priv->hvUUID, uuidstr)); + virCommandAddArg(cmd, virUUIDFormat(def->uuid, uuidstr)); } /* RTVM */ @@ -1158,24 +742,13 @@ acrnAutostartDomain(virDomainObjPtr vm, void *opaque) int ret = 0; acrnConnectPtr privconn = data->driver; acrnDomainObjPrivatePtr priv; - struct acrnVmList *vmList = data->vmlist; - ssize_t idx; - char uuidstr[VIR_UUID_STRING_BUFLEN]; virObjectLock(vm); if (vm->autostart && !virDomainObjIsActive(vm)) { virResetLastError(); priv = vm->privateData; - /* find the allocated VM */ - if ((idx = acrnFindVm(vmList, priv->hvUUID)) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("vm(%s) not found"), - virUUIDFormat(priv->hvUUID, uuidstr)); - goto cleanup; - } - if (acrnProcessPrepareDomain(vm, &privconn->pi, &vmList->vm[idx], - privconn->vcpuAllocMap) < 0) + if (acrnProcessPrepareDomain(vm, privconn->vcpuAllocMap) < 0) goto cleanup; if (acrnProcessStart(vm) < 0) { /* domain must be persistent */ @@ -1194,12 +767,12 @@ acrnAutostartDomain(virDomainObjPtr vm, void *opaque) } static void -acrnAutostartDomains(acrnConnectPtr driver, struct acrnVmList *vmlist) +acrnAutostartDomains(acrnConnectPtr driver) { virConnectPtr conn = virConnectOpen("acrn:///system"); /* Ignoring NULL conn which is mostly harmless here */ - struct acrnAutostartData data = { driver, conn, vmlist}; + struct acrnAutostartData data = { driver, conn }; virDomainObjListForEach(driver->domains, false, acrnAutostartDomain, &data); @@ -1668,15 +1241,12 @@ acrnDomainCreateXML(virConnectPtr conn, unsigned int flags) { acrnConnectPtr privconn = conn->privateData; - struct acrnVmList *vmList = NULL; acrnDomainObjPrivatePtr priv; virDomainDefPtr def; virDomainObjPtr vm = NULL; virObjectEventPtr event = NULL; virDomainPtr dom = NULL; - ssize_t idx; unsigned int parse_flags = VIR_DOMAIN_DEF_PARSE_INACTIVE; - unsigned char hvUUID[VIR_UUID_BUFLEN]; /* VIR_DOMAIN_START_AUTODESTROY is not supported yet */ virCheckFlags(VIR_DOMAIN_START_VALIDATE, NULL); @@ -1688,20 +1258,6 @@ acrnDomainCreateXML(virConnectPtr conn, NULL, parse_flags))) goto cleanup_nolock; - if (!(vmList = acrnVmListNew())) - goto cleanup_nolock; - - acrnDriverLock(privconn); - - /* retrieve current platform info */ - if (acrnGetPlatform(&privconn->pi, vmList) < 0) - goto cleanup; - - /* get hv UUID for the allocated VM */ - if ((idx = acrnAllocateVm(privconn->domains, def, &privconn->pi, vmList, - hvUUID)) < 0) - goto cleanup; - if (!(vm = virDomainObjListAdd(privconn->domains, def, privconn->xmlopt, VIR_DOMAIN_OBJ_LIST_ADD_LIVE | @@ -1709,12 +1265,10 @@ acrnDomainCreateXML(virConnectPtr conn, goto cleanup; priv = vm->privateData; - uuid_copy(priv->hvUUID, hvUUID); def = NULL; - if (acrnProcessPrepareDomain(vm, &privconn->pi, &vmList->vm[idx], - privconn->vcpuAllocMap) < 0) + if (acrnProcessPrepareDomain(vm, privconn->vcpuAllocMap) < 0) goto cleanup; if (acrnProcessStart(vm) < 0) { @@ -1746,8 +1300,6 @@ acrnDomainCreateXML(virConnectPtr conn, virDomainDefFree(def); if (event) virObjectEventStateQueue(privconn->domainEventState, event); - if (vmList) - acrnVmListFree(vmList); return dom; } @@ -1755,26 +1307,16 @@ static int acrnDomainCreateWithFlags(virDomainPtr domain, unsigned int flags) { acrnConnectPtr privconn = domain->conn->privateData; - struct acrnVmList *vmList; acrnDomainObjPrivatePtr priv; virDomainObjPtr vm = NULL; virObjectEventPtr event = NULL; - ssize_t idx; int ret = -1; - char uuidstr[VIR_UUID_STRING_BUFLEN]; /* VIR_DOMAIN_START_AUTODESTROY is not supported yet */ virCheckFlags(0, -1); - if (!(vmList = acrnVmListNew())) - return -1; - acrnDriverLock(privconn); - /* retrieve current platform info */ - if (acrnGetPlatform(&privconn->pi, vmList) < 0) - goto cleanup; - if (!(vm = acrnDomObjFromDomain(domain))) goto cleanup; @@ -1786,16 +1328,7 @@ acrnDomainCreateWithFlags(virDomainPtr domain, unsigned int flags) priv = vm->privateData; - /* find the allocated VM */ - if ((idx = acrnFindVm(vmList, priv->hvUUID)) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("vm(%s) not found"), - virUUIDFormat(priv->hvUUID, uuidstr)); - goto cleanup; - } - - if (acrnProcessPrepareDomain(vm, &privconn->pi, &vmList->vm[idx], - privconn->vcpuAllocMap) < 0) + if (acrnProcessPrepareDomain(vm, privconn->vcpuAllocMap) < 0) goto cleanup; if (acrnProcessStart(vm) < 0) { @@ -1822,7 +1355,6 @@ acrnDomainCreateWithFlags(virDomainPtr domain, unsigned int flags) acrnDriverUnlock(privconn); if (event) virObjectEventStateQueue(privconn->domainEventState, event); - acrnVmListFree(vmList); return ret; } @@ -1837,14 +1369,11 @@ acrnDomainDefineXMLFlags(virConnectPtr conn, const char *xml, unsigned int flags) { acrnConnectPtr privconn = conn->privateData; - struct acrnVmList *vmList = NULL; - acrnDomainObjPrivatePtr priv; virDomainDefPtr def = NULL, oldDef = NULL; virDomainObjPtr vm = NULL; virObjectEventPtr event = NULL; virDomainPtr dom = NULL; unsigned int parse_flags = VIR_DOMAIN_DEF_PARSE_INACTIVE; - unsigned char hvUUID[VIR_UUID_BUFLEN]; virCheckFlags(VIR_DOMAIN_DEFINE_VALIDATE, NULL); @@ -1858,28 +1387,14 @@ acrnDomainDefineXMLFlags(virConnectPtr conn, const char *xml, if (virXMLCheckIllegalChars("name", def->name, "\n") < 0) goto cleanup_nolock; - if (!(vmList = acrnVmListNew())) - goto cleanup_nolock; - acrnDriverLock(privconn); - /* retrieve current platform info */ - if (acrnGetPlatform(&privconn->pi, vmList) < 0) - goto cleanup; - - /* get hv UUID for the allocated VM */ - if (acrnAllocateVm(privconn->domains, def, &privconn->pi, vmList, - hvUUID) < 0) - goto cleanup; - if (!(vm = virDomainObjListAdd(privconn->domains, def, privconn->xmlopt, 0, &oldDef))) goto cleanup; vm->persistent = 1; - priv = vm->privateData; - uuid_copy(priv->hvUUID, hvUUID); def = NULL; @@ -1914,8 +1429,6 @@ acrnDomainDefineXMLFlags(virConnectPtr conn, const char *xml, virDomainDefFree(def); if (event) virObjectEventStateQueue(privconn->domainEventState, event); - if (vmList) - acrnVmListFree(vmList); return dom; } @@ -2579,56 +2092,9 @@ acrnNodeGetCPUMap(virConnectPtr conn, unsigned int *online, unsigned int flags) { - acrnConnectPtr privconn = conn->privateData; - acrnPlatformInfoPtr pi = &privconn->pi; - struct acrnVmList *list; - virBitmapPtr cpus = NULL; - size_t i; - int dummy, ret = -1; - - virCheckFlags(0, -1); - - if (!(list = acrnVmListNew())) + if (virNodeGetCPUMapEnsureACL(conn) < 0) return -1; - - if (acrnGetPlatform(pi, list) < 0) - goto cleanup; - - /* - * Mark pCPUs available to the SOS (online) or UOS - * (present in the pcpus bitmap). - */ - if (!(cpus = virHostCPUGetOnlineBitmap())) - goto cleanup; - - for (i = 0; i < list->size; i++) { - if (list->vm[i].cfg.load_order == POST_LAUNCHED_VM) { - ssize_t pos = -1; - - while ((pos = virBitmapNextSetBit(list->vm[i].pcpus, pos)) >= 0) { - if (virBitmapSetBitExpand(cpus, pos) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("virBitmapSetBitExpand failed")); - goto cleanup; - } - } - } - } - - if (cpumap && virBitmapToData(cpus, cpumap, &dummy) < 0) - goto cleanup; - - if (online) - *online = virBitmapCountBits(cpus); - - ret = pi->hw.cpu_num; - -cleanup: - if (ret < 0 && cpumap && *cpumap) - VIR_FREE(*cpumap); - virBitmapFree(cpus); - acrnVmListFree(list); - return ret; + return virHostCPUGetMap(cpumap, online, flags); } static int @@ -2644,10 +2110,7 @@ acrnStateCleanup(void) virObjectUnref(acrn_driver->xmlopt); virObjectUnref(acrn_driver->caps); virObjectUnref(acrn_driver->domains); - if (acrn_driver->pi.sw.vm_configs_addr) { - void *p = (void *)acrn_driver->pi.sw.vm_configs_addr; - VIR_FREE(p); - } + if (acrn_driver->vcpuAllocMap) VIR_FREE(acrn_driver->vcpuAllocMap); virMutexDestroy(&acrn_driver->lock); @@ -2711,14 +2174,16 @@ virAcrnCapsBuild(void) * Vacate SOS CPUs for UOS vCPU allocation. */ static int -acrnOfflineCpus(int nprocs, virBitmapPtr pcpus, size_t *allocMap) +acrnOfflineCpus(int nprocs) { ssize_t i = -1; int fd; char path[128], chr, online; ssize_t rc; + virBitmapPtr cpus; - while ((i = virBitmapNextSetBit(pcpus, i)) >= 0 && i < nprocs) { + cpus = virHostCPUGetOnlineBitmap(); + while ((i = virBitmapNextSetBit(cpus, i)) >= 0 && i < nprocs) { /* cpu0 can't be offlined */ if (i == 0) continue; @@ -2762,35 +2227,19 @@ acrnOfflineCpus(int nprocs, virBitmapPtr pcpus, size_t *allocMap) } close(fd); - - if (!allocMap[i]) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("vCPU allocation map error (bit %ld)"), i); - return -1; - } - - allocMap[i] -= 1; } return 0; } static int -acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, - size_t **allocMap, const struct acrnVmList *list) +acrnInitPlatform(virNodeInfoPtr nodeInfo, size_t **allocMap) { - virBitmapPtr postLaunchedPcpus = NULL; uint16_t totalCpus; - size_t i, *map = NULL; + size_t *map = NULL; int ret; - totalCpus = pi->hw.cpu_num; - - if (!(postLaunchedPcpus = virBitmapNew(totalCpus))) { - virReportError(VIR_ERR_NO_MEMORY, NULL); - ret = -ENOMEM; - goto cleanup; - } + totalCpus = get_nprocs_conf(); if (VIR_ALLOC_N(map, totalCpus) < 0) { ret = -ENOMEM; @@ -2799,43 +2248,7 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, nodeInfo->cpus = totalCpus; - /* - * Assume pre-launched VMs are always running. - * - * There is no way to figure out the current vCPU - * allocation map via platform_info. It needs to be - * tracked in this driver. - */ - for (i = 0; i < list->size; i++) { - ssize_t pos = -1; - - if (list->vm[i].cfg.load_order == POST_LAUNCHED_VM) { - /* collect all pCPUs that can be used by a UOS */ - while ((pos = virBitmapNextSetBit(list->vm[i].pcpus, pos)) >= 0) { - if (virBitmapSetBit(postLaunchedPcpus, pos) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("virBitmapSetBit failed")); - ret = -EINVAL; - goto cleanup; - } - } - } else { - if (list->vm[i].cfg.load_order == SOS_VM) { - if (!virBitmapIsBitSet(list->vm[i].pcpus, 0)) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("SOS BSP is not pCPU0")); - ret = -EINVAL; - goto cleanup; - } - } - - while ((pos = virBitmapNextSetBit(list->vm[i].pcpus, pos)) >= 0 && - pos < totalCpus) - map[pos] += 1; - } - } - - if (acrnOfflineCpus(get_nprocs_conf(), postLaunchedPcpus, map) < 0) { + if (acrnOfflineCpus(nodeInfo->cpus) < 0) { ret = -EIO; goto cleanup; } @@ -2847,40 +2260,25 @@ acrnInitPlatform(acrnPlatformInfoPtr pi, virNodeInfoPtr nodeInfo, cleanup: if (map) VIR_FREE(map); - virBitmapFree(postLaunchedPcpus); return ret; } static int acrnPersistentDomainInit(virDomainObjPtr dom, void *opaque) { - unsigned char hvUUID[VIR_UUID_BUFLEN]; - char uuidstr[VIR_UUID_STRING_BUFLEN]; - - struct acrnVmList *vmList = opaque; - acrnDomainObjPrivatePtr priv = dom->privateData; virObjectEventPtr event = NULL; + acrnConnectPtr driver = opaque; if (!dom->persistent) return -1; - if (acrnAllocateVm(acrn_driver->domains, dom->def, &acrn_driver->pi, vmList, - hvUUID) < 0) - return -1; - - VIR_DEBUG("Adding ACRN %sdomain %s (%s)", - acrnIsRtvm(dom->def) ? "RT " : "", - virUUIDFormat(hvUUID, uuidstr), dom->def->name); - - uuid_copy(priv->hvUUID, hvUUID); - event = virDomainEventLifecycleNewFromObj(dom, VIR_DOMAIN_EVENT_DEFINED, VIR_DOMAIN_EVENT_DEFINED_ADDED); if (!event) return -1; - virObjectEventStateQueue(acrn_driver->domainEventState, event); + virObjectEventStateQueue(driver->domainEventState, event); return 0; } @@ -2891,7 +2289,6 @@ acrnStateInitialize(bool privileged, void *opaque G_GNUC_UNUSED) { int ret; - struct acrnVmList *list = NULL; bool autostart = true; if (root) { @@ -2917,22 +2314,9 @@ acrnStateInitialize(bool privileged, if (virCapabilitiesGetNodeInfo(&acrn_driver->nodeInfo) < 0) goto cleanup; - list = acrnVmListNew(); - if (!list) - goto cleanup; - - ret = acrnGetPlatform(&acrn_driver->pi, list); - if (ret == -ENODEV) { - /* we are not running on an ACRN enabled system */ - VIR_INFO("ACRN hypervisor not available, disabling driver"); - ret = VIR_DRV_STATE_INIT_SKIPPED; - goto cleanup_nofail; - } - if (ret < 0) - goto cleanup; - if (acrnInitPlatform(&acrn_driver->pi, &acrn_driver->nodeInfo, - &acrn_driver->vcpuAllocMap, list) < 0) + if (acrnInitPlatform(&acrn_driver->nodeInfo, + &acrn_driver->vcpuAllocMap) < 0) goto cleanup; if (!(acrn_driver->domains = virDomainObjListNew())) @@ -2972,23 +2356,20 @@ acrnStateInitialize(bool privileged, goto cleanup; if (virDomainObjListForEach(acrn_driver->domains, false, - acrnPersistentDomainInit, list) < 0) + acrnPersistentDomainInit, acrn_driver) < 0) goto cleanup; if (virDriverShouldAutostart(ACRN_STATE_DIR, &autostart) < 0) goto cleanup; if (autostart) - acrnAutostartDomains(acrn_driver, list); + acrnAutostartDomains(acrn_driver); - acrnVmListFree(list); return VIR_DRV_STATE_INIT_COMPLETE; cleanup: ret = VIR_DRV_STATE_INIT_ERROR; -cleanup_nofail: acrnStateCleanup(); - acrnVmListFree(list); return ret; } From 495d9e382aa250502ba1899db65871bb064860c8 Mon Sep 17 00:00:00 2001 From: Xiangyang Wu Date: Thu, 4 Nov 2021 13:34:48 +0800 Subject: [PATCH 18/32] acrn-driver: don't pass UUID to acrn-dm UUID parameter of acrn-dm will be removed in ACRN v2.7, so don't need to pass UUID to acrn-dm. Tracked-On: projectacrn/acrn-hypervisor#6685 Signed-off-by: Xiangyang Wu --- src/acrn/acrn_driver.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 1bf5b5018b..a46e7e2d0c 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -611,7 +611,6 @@ acrnBuildStartCmd(virDomainObjPtr vm) acrnDomainXmlNsDefPtr nsdef; struct acrnCmdDeviceData data = { 0 }; char *pcpus; - char uuidstr[VIR_UUID_STRING_BUFLEN]; size_t i; if (!vm || !(def = vm->def)) @@ -638,12 +637,6 @@ acrnBuildStartCmd(virDomainObjPtr vm) virCommandAddArgFormat(cmd, "%lluM", VIR_DIV_UP(virDomainDefGetMemoryInitial(def), 1024)); - /* UUID */ - if (virUUIDIsValid(def->uuid)) { - virCommandAddArg(cmd, "-U"); - virCommandAddArg(cmd, virUUIDFormat(def->uuid, uuidstr)); - } - /* RTVM */ if (acrnIsRtvm(def)) virCommandAddArgList(cmd, From b1623783f39675d74f880b0f8bd243b6ccb42a32 Mon Sep 17 00:00:00 2001 From: Xiangyang Wu Date: Sun, 21 Nov 2021 09:33:50 +0800 Subject: [PATCH 19/32] acrn-driver: fix a cpu offline bug Currently, in acrnOfflineCpus, only one charater is used to store CPU ID of the processor which will be offlined, if the CPU ID of a processor is more than 9, acrn driver will fail to offline this procesor. In this patch, use four charaters to store CPU ID, can support CPU ID (0 ~ 999). Signed-off-by: Xiangyang Wu --- src/acrn/acrn_driver.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index a46e7e2d0c..dac14acc05 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -2171,7 +2171,7 @@ acrnOfflineCpus(int nprocs) { ssize_t i = -1; int fd; - char path[128], chr, online; + char path[128], chr, online, cpu_id[4]; ssize_t rc; virBitmapPtr cpus; @@ -2181,7 +2181,8 @@ acrnOfflineCpus(int nprocs) if (i == 0) continue; - snprintf(path, sizeof(path), "%s/cpu%ld/online", SYSFS_CPU_PATH, i); + snprintf(cpu_id, sizeof(cpu_id), "%ld", i); + snprintf(path, sizeof(path), "%s/cpu%s/online", SYSFS_CPU_PATH, cpu_id); if ((fd = open(path, O_RDWR)) < 0) { virReportError(VIR_ERR_OPEN_FAILED, _("%s"), path); @@ -2211,9 +2212,7 @@ acrnOfflineCpus(int nprocs) return -1; } - chr += i; - - if (write(fd, &chr, sizeof(chr)) < sizeof(chr)) { + if (write(fd, cpu_id, strlen(cpu_id)) < strlen(cpu_id)) { close(fd); virReportError(VIR_ERR_WRITE_FAILED, _(ACRN_OFFLINE_PATH)); return -1; From aac27599a4526c74f5a6cef2066195d3931d587a Mon Sep 17 00:00:00 2001 From: Yuanyuan Zhao Date: Mon, 28 Feb 2022 02:36:32 +0000 Subject: [PATCH 20/32] acrn-driver: remove acrn-dm param `-A` Command parameter -A is used to make ACRN DM generate a ACPI table for a user VM. Currently, Command parameter -A have been removed in ACRN DM, ACRN DM generates ACPI table automatically, acrn-driver in Libvirt doesn't need to generate this parameter for user VM. Tracked-On: #17 Reviewed-by: Xiangyang Wu Signed-off-by: Yuanyuan Zhao --- src/acrn/acrn_driver.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index dac14acc05..1ef3062476 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -623,10 +623,6 @@ acrnBuildStartCmd(virDomainObjPtr vm) priv = vm->privateData; - /* ACPI */ - if (def->features[VIR_DOMAIN_FEATURE_ACPI] == VIR_TRISTATE_SWITCH_ON) - virCommandAddArg(cmd, "-A"); - /* CPU */ pcpus = virBitmapFormat(priv->cpuAffinitySet); virCommandAddArgList(cmd, "--cpu_affinity", pcpus, NULL); From aa0a75dcf9bee2170f2551d631991e0a303c02ae Mon Sep 17 00:00:00 2001 From: Xiangyang Wu Date: Tue, 25 Jan 2022 15:22:44 +0800 Subject: [PATCH 21/32] acrn-driver: support vm destroy and force shutdown The following updates are made in this patch to support vm desroy: - Create socket folder for acrn DM command monitor; - Pass socket path to DM through command arguments; - Implement a monitor to interact with DM command monitor; - Implement event loop to watch DM command monitor socket events, such as read event, disconnect event to receive reply and sync user VM state with acrn DM. Tracked-On: #5921 Signed-off-by: Xiangyang Wu --- src/acrn/Makefile.inc.am | 2 + src/acrn/acrn_domain.h | 5 + src/acrn/acrn_driver.c | 243 ++++++++++++---- src/acrn/acrn_monitor.c | 584 +++++++++++++++++++++++++++++++++++++++ src/acrn/acrn_monitor.h | 38 +++ 5 files changed, 817 insertions(+), 55 deletions(-) create mode 100644 src/acrn/acrn_monitor.c create mode 100644 src/acrn/acrn_monitor.h diff --git a/src/acrn/Makefile.inc.am b/src/acrn/Makefile.inc.am index 0cfcc3d3da..fbc67038c3 100644 --- a/src/acrn/Makefile.inc.am +++ b/src/acrn/Makefile.inc.am @@ -8,6 +8,8 @@ ACRN_DRIVER_SOURCES = \ acrn/acrn_domain.c \ acrn/acrn_device.h \ acrn/acrn_device.c \ + acrn/acrn_monitor.h \ + acrn/acrn_monitor.c \ $(NULL) DRIVER_SOURCE_FILES += $(addprefix $(srcdir)/,$(ACRN_DRIVER_SOURCES)) diff --git a/src/acrn/acrn_domain.h b/src/acrn/acrn_domain.h index 0e3c20b4cc..43d16d1048 100644 --- a/src/acrn/acrn_domain.h +++ b/src/acrn/acrn_domain.h @@ -2,6 +2,7 @@ #define __ACRN_DOMAIN_H__ #include "domain_conf.h" +#include "acrn_monitor.h" typedef struct _acrnDomainObjPrivate acrnDomainObjPrivate; typedef acrnDomainObjPrivate *acrnDomainObjPrivatePtr; @@ -13,6 +14,10 @@ struct _acrnDomainObjPrivate { char *slave; } ttys[4]; size_t nttys; + + virDomainChrSourceDefPtr monConfig; + char *libDir; /* base path for per-domain files */ + acrnMonitorPtr mon; }; typedef struct _acrnDomainXmlNsDef acrnDomainXmlNsDef; diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 1ef3062476..07303349e7 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -26,6 +26,7 @@ #include "viraccessapicheck.h" #include "acrn_driver.h" #include "acrn_domain.h" +#include "acrn_monitor.h" #define VIR_FROM_THIS VIR_FROM_ACRN #define ACRN_DM_PATH "/usr/bin/acrn-dm" @@ -35,6 +36,7 @@ #define ACRN_AUTOSTART_DIR SYSCONFDIR "/libvirt/acrn/autostart" #define ACRN_CONFIG_DIR SYSCONFDIR "/libvirt/acrn" #define ACRN_STATE_DIR RUNSTATEDIR "/libvirt/acrn" +#define ACRN_MONITOR_DIR "/var/lib/libvirt/acrn" #define ACRN_NET_GENERATED_TAP_PREFIX "tap" #define ACRN_PI_VERSION (0x100) @@ -612,6 +614,7 @@ acrnBuildStartCmd(virDomainObjPtr vm) struct acrnCmdDeviceData data = { 0 }; char *pcpus; size_t i; + char *monitor_path; if (!vm || !(def = vm->def)) return NULL; @@ -689,39 +692,115 @@ acrnBuildStartCmd(virDomainObjPtr vm) /* VM name */ virCommandAddArg(cmd, def->name); + /* Command monitor */ + monitor_path = g_strdup_printf("%s/domain-%s/monitor.sock", ACRN_MONITOR_DIR, vm->def->name); + virCommandAddArgList(cmd, "--cmd_monitor", monitor_path, NULL); + return cmd; } static int -acrnProcessStart(virDomainObjPtr vm) +acrnProcessPrepareMonitorChr(virDomainChrSourceDefPtr monConfig, + const char *domainDir) +{ + monConfig->type = VIR_DOMAIN_CHR_TYPE_UNIX; + monConfig->data.nix.listen = true; + + monConfig->data.nix.path = g_strdup_printf("%s/monitor.sock", domainDir); + return 0; +} + +static int +acrnProcessWaitForMonitor(virDomainObjPtr vm, acrnMonitorStopCallback stop) +{ + acrnDomainObjPrivatePtr priv = vm->privateData; + virDomainChrSourceDefPtr config = priv->monConfig; + acrnMonitorPtr mon = NULL; + + if (!priv->libDir) + priv->libDir = g_strdup_printf("%s/domain-%s", ACRN_MONITOR_DIR, vm->def->name); + if (virFileMakePath(priv->libDir) < 0) { + virReportSystemError(errno, + _("Failed to mkdir %s"), + priv->libDir); + return -1; + } + if (!(config = virDomainChrSourceDefNew(acrn_driver->xmlopt))) + return -1; + VIR_DEBUG("Preparing monitor state"); + if (acrnProcessPrepareMonitorChr(config, priv->libDir) < 0) + return -1; + VIR_DEBUG("Monitor UNIX socket path:%s", config->data.nix.path); + + mon = acrnMonitorOpen(vm, config, stop); + + priv->mon = mon; + + if (priv->mon == NULL) { + VIR_INFO("Failed to connect monitor for %s", vm->def->name); + return -1; + } + VIR_DEBUG("acrnProcessWaitForMonitor:end"); + return 0; +} + +static virCommandPtr +acrnRunStartCommand(virDomainObjPtr vm) { - virCommandPtr cmd; int ret = -1; + virCommandPtr cmd; - if (!(cmd = acrnBuildStartCmd(vm))) - goto cleanup; + cmd = acrnBuildStartCmd(vm); + if (!cmd) + return NULL; - virCommandDaemonize(cmd); + virCommandRawStatus(cmd); + ret = virCommandRunAsync(cmd, &vm->pid); + if (ret < 0) { + virCommandFree(cmd); + cmd = NULL; + virReportSystemError(errno, "%s", _("virCommandRunAsync failed")); + } + + return cmd; +} + +static void acrnProcessStopCallback(virDomainObjPtr vm); + +static int +acrnProcessStart(virDomainObjPtr vm) +{ + virCommandPtr cmd; + int ret = -1; VIR_DEBUG("Starting domain '%s'", vm->def->name); - if (virCommandRun(cmd, NULL) < 0) + cmd = acrnRunStartCommand(vm); + if (!cmd) goto cleanup; - /* XXX */ - if (sscanf(vm->def->name, "vm%d", &vm->def->id) != 1 && - sscanf(vm->def->name, "instance-%d", &vm->def->id) != 1) - vm->def->id = 0; + vm->def->id = vm->pid; virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_BOOTED); - ret = 0; + + VIR_DEBUG("Waiting for monitor to show up"); + if (acrnProcessWaitForMonitor(vm, &acrnProcessStopCallback) < 0) + goto cleanup; + + if (virDomainObjSave(vm, acrn_driver->xmlopt, + ACRN_STATE_DIR) < 0) + goto cleanup; + + return 0; cleanup: - virCommandFree(cmd); - if (ret < 0) { - acrnNetCleanup(vm); - acrnTtyCleanup(vm); + if (cmd) { + virCommandFree(cmd); } + + vm->pid = -1; + acrnNetCleanup(vm); + acrnTtyCleanup(vm); return ret; } static int @@ -767,36 +846,19 @@ acrnAutostartDomains(acrnConnectPtr driver) virObjectUnref(conn); } -static virCommandPtr -acrnBuildStopCmd(virDomainDefPtr def) -{ - virCommandPtr cmd; - - if (!def) - return NULL; - - if (!(cmd = virCommandNewArgList(ACRN_CTL_PATH, "stop", "-f", - def->name, NULL))) - virReportError(VIR_ERR_NO_MEMORY, NULL); - - return cmd; -} static int -acrnProcessStop(virDomainObjPtr vm, int reason, size_t *allocMap) +acrnDomainShutdownMonitor(virDomainObjPtr vm) { - virDomainDefPtr def = vm->def; - virCommandPtr cmd; acrnDomainObjPrivatePtr priv = vm->privateData; - int ret = -1; - if (!(cmd = acrnBuildStopCmd(def))) - goto cleanup; - - VIR_DEBUG("Stopping domain '%s'", def->name); + return acrnMonitorSystemPowerdown(priv->mon); +} - if (virCommandRun(cmd, NULL) < 0) - goto cleanup; +static void +acrnProcessCleanup(virDomainObjPtr vm, int reason, size_t *allocMap) +{ + acrnDomainObjPrivatePtr priv = vm->privateData; /* clean up network interfaces */ acrnNetCleanup(vm); @@ -804,17 +866,55 @@ acrnProcessStop(virDomainObjPtr vm, int reason, size_t *allocMap) /* clean up ttys */ acrnTtyCleanup(vm); + acrnMonitorClose(priv->mon); + if (priv->monConfig) { + if (priv->monConfig->type == VIR_DOMAIN_CHR_TYPE_UNIX) + unlink(priv->monConfig->data.nix.path); + virObjectUnref(priv->monConfig); + priv->monConfig = NULL; + } + virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, reason); - def->id = -1; - ret = 0; + vm->pid = -1; + vm->def->id = -1; -cleanup: - virCommandFree(cmd); acrnFreeVcpus(priv->cpuAffinitySet, allocMap); + virDomainDeleteConfig(ACRN_STATE_DIR, NULL, vm); +} + +static int +acrnProcessStop(virDomainObjPtr vm, int reason) +{ + virDomainDefPtr def = vm->def; + int ret = 0; + + VIR_DEBUG("Stopping domain '%s'", def->name); + + if (acrnDomainShutdownMonitor(vm) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("Fail to stop domain '%s'"), def->name); + ret = -1; + } + virDomainObjSetState(vm, VIR_DOMAIN_SHUTDOWN, reason); + return ret; } +static void +acrnProcessStopCallback(virDomainObjPtr vm) +{ + int reason; + acrnDomainObjPrivatePtr priv = vm->privateData; + acrnMonitorPtr mon = priv->mon; + + VIR_DEBUG("acrnProcessStopCallback '%s'", vm->def->name); + reason = acrnMonitorGetReason(mon); + acrnProcessCleanup(vm, reason, acrn_driver->vcpuAllocMap); + if (!vm->persistent) + virDomainObjListRemove(acrn_driver->domains, vm); +} + static virDomainPtr acrnDomainLookupByUUID(virConnectPtr conn, const unsigned char *uuid) @@ -882,9 +982,9 @@ acrnDomainShutdown(virDomainPtr dom) goto cleanup; } - if (acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, - privconn->vcpuAllocMap) < 0) + if (acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN) < 0) { goto cleanup; + } if (!(event = virDomainEventLifecycleNewFromObj( vm, @@ -894,6 +994,9 @@ acrnDomainShutdown(virDomainPtr dom) ret = 0; + if (!vm->persistent) + virDomainObjListRemove(privconn->domains, vm); + cleanup: if (vm) virObjectUnlock(vm); @@ -910,7 +1013,7 @@ acrnDomainDestroy(virDomainPtr dom) virDomainObjPtr vm; virDomainState state; virObjectEventPtr event = NULL; - int reason, ret = -1; + int reason, ret = -1, val = 0; acrnDriverLock(privconn); @@ -930,9 +1033,7 @@ acrnDomainDestroy(virDomainPtr dom) virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, VIR_DOMAIN_SHUTOFF_DESTROYED); } else { - if (acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_DESTROYED, - privconn->vcpuAllocMap) < 0) - goto cleanup; + val = acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_DESTROYED); } event = virDomainEventLifecycleNewFromObj( @@ -950,7 +1051,7 @@ acrnDomainDestroy(virDomainPtr dom) if (!event) goto cleanup; - ret = 0; + ret = val; cleanup: if (vm) @@ -1269,8 +1370,7 @@ acrnDomainCreateXML(virConnectPtr conn, vm, VIR_DOMAIN_EVENT_STARTED, VIR_DOMAIN_EVENT_STARTED_BOOTED))) { - acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_DESTROYED, - privconn->vcpuAllocMap); + acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_DESTROYED); goto cleanup; } @@ -1331,8 +1431,7 @@ acrnDomainCreateWithFlags(virDomainPtr domain, unsigned int flags) VIR_DOMAIN_EVENT_STARTED, VIR_DOMAIN_EVENT_STARTED_BOOTED))) { /* domain must be persistent */ - acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_DESTROYED, - privconn->vcpuAllocMap); + acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_DESTROYED); goto cleanup; } @@ -2258,7 +2357,7 @@ acrnPersistentDomainInit(virDomainObjPtr dom, void *opaque) acrnConnectPtr driver = opaque; if (!dom->persistent) - return -1; + return 0; event = virDomainEventLifecycleNewFromObj(dom, VIR_DOMAIN_EVENT_DEFINED, @@ -2269,7 +2368,33 @@ acrnPersistentDomainInit(virDomainObjPtr dom, void *opaque) virObjectEventStateQueue(driver->domainEventState, event); return 0; } +struct acrnProcessReconnectData { + acrnConnectPtr driver; +}; +static int +viracrnProcessReconnect(virDomainObjPtr vm, + void *opaque) +{ + int ret = -1; + + if (!virDomainObjIsActive(vm)) + return 0; + VIR_DEBUG("ACRN driver try to reconnect %s\n", vm->def->name); + if (acrnProcessWaitForMonitor(vm, &acrnProcessStopCallback) < 0) + goto cleanup; + ret = 0; +cleanup: + return ret; +} +static void +viracrnProcessReconnectAll(acrnConnectPtr driver) +{ + struct acrnProcessReconnectData data; + data.driver = driver; + virDomainObjListForEach(driver->domains, false, viracrnProcessReconnect, &data); + return; +} static virDrvStateInitResult acrnStateInitialize(bool privileged, const char *root, @@ -2328,6 +2453,12 @@ acrnStateInitialize(bool privileged, ACRN_STATE_DIR); goto cleanup; } + if (virFileMakePath(ACRN_MONITOR_DIR) < 0) { + virReportSystemError(errno, + _("Failed to mkdir %s"), + ACRN_MONITOR_DIR); + goto cleanup; + } if (virDomainObjListLoadAllConfigs(acrn_driver->domains, ACRN_STATE_DIR, @@ -2347,6 +2478,8 @@ acrnStateInitialize(bool privileged, acrnPersistentDomainInit, acrn_driver) < 0) goto cleanup; + viracrnProcessReconnectAll(acrn_driver); + if (virDriverShouldAutostart(ACRN_STATE_DIR, &autostart) < 0) goto cleanup; diff --git a/src/acrn/acrn_monitor.c b/src/acrn/acrn_monitor.c new file mode 100644 index 0000000000..72c030306f --- /dev/null +++ b/src/acrn/acrn_monitor.c @@ -0,0 +1,584 @@ +#include +#include +#include +#include +#include "datatypes.h" +#include "virfile.h" +#include "viralloc.h" +#include "virutil.h" +#include "vircommand.h" +#include "virthread.h" +#include "virstring.h" +#include "domain_event.h" +#include "virjson.h" +#include "acrn_monitor.h" +#include "virtime.h" +#include "virerror.h" +#include "virlog.h" +#include "acrn_domain.h" + +#define VIR_FROM_THIS VIR_FROM_ACRN +#define ACRN_DEFAULT_MONITOR_WAIT 30 + +#define LINE_ENDING "\n" + +VIR_LOG_INIT("acrn.acrn_monitor"); + +struct _acrnMonitor { + int fd; + int watch; + virDomainObjPtr vm; + virMutex lock; + + acrnMonitorMessagePtr msg; + /* Buffer incoming data ready for command monitor + * code to process & find message boundaries */ + char buffer[1024]; + + acrnMonitorStopCallback stop; + int shutdown_reason; +}; + +static int +acrnMonitorIOWriteWithFD(acrnMonitorPtr mon, + const char *data, + size_t len, + int fd) +{ + struct msghdr msg; + struct iovec iov[1]; + int ret; + char control[CMSG_SPACE(sizeof(int))]; + struct cmsghdr *cmsg; + + memset(&msg, 0, sizeof(msg)); + memset(control, 0, sizeof(control)); + + iov[0].iov_base = (void *)data; + iov[0].iov_len = len; + + msg.msg_iov = iov; + msg.msg_iovlen = 1; + + msg.msg_control = control; + msg.msg_controllen = sizeof(control); + + cmsg = CMSG_FIRSTHDR(&msg); + /* Some static analyzers, like clang 2.6-0.6.pre2, fail to see + that our use of CMSG_FIRSTHDR will not return NULL. */ + sa_assert(cmsg); + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + memcpy(CMSG_DATA(cmsg), &fd, sizeof(int)); + + do { + ret = sendmsg(mon->fd, &msg, 0); + } while (ret < 0 && errno == EINTR); + + return ret; +} + +static int +acrnMonitorIORead(acrnMonitorPtr mon) +{ + int ret = 0; + int got; + + memset(mon->buffer, 0x0, 1024); + got = read(mon->fd, + mon->buffer, + 1023); + if (got < 0) { + if (errno == EAGAIN) + virReportSystemError(errno, "%s", + _("Unable to read from monitor")); + ret = -1; + } + + ret += got; + mon->buffer[got] = '\0'; + + VIR_DEBUG("Now read %d bytes of data: %s.", got, mon->buffer); + + + return ret; +} + +static int +acrnMonitorJSONIOProcessLine(acrnMonitorPtr mon, + const char *line, + acrnMonitorMessagePtr msg) +{ + virJSONValuePtr obj = NULL; + int ret = -1; + + VIR_DEBUG("Line [%s]", line); + + if (!(obj = virJSONValueFromString(line))) + goto cleanup; + + if (virJSONValueGetType(obj) != VIR_JSON_TYPE_OBJECT) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Parsed JSON reply '%s' isn't an object"), line); + goto cleanup; + } + if (virJSONValueObjectHasKey(obj, "ack")) { + VIR_DEBUG("mon=%p reply=%s", mon, line); + if (msg) { + msg->rxObject = obj; + msg->finished = 1; + obj = NULL; + ret = 0; + } else { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unexpected JSON reply '%s'"), line); + } + } +cleanup: + virJSONValueFree(obj); + return ret; +} + +static int +acrnMonitorIOProcess(acrnMonitorPtr mon) +{ + int len; + acrnMonitorMessagePtr msg = NULL; + + if (mon->msg) + msg = mon->msg; + + len = acrnMonitorJSONIOProcessLine(mon, + mon->buffer, msg); + if (len < 0) + return -1; + + return len; +} + +static virJSONValuePtr +acrnMonitorMakeCommand(const char *cmdname) +{ + virJSONValuePtr obj; + + if (!(obj = virJSONValueNewObject())) + goto error; + + if (virJSONValueObjectAppendString(obj, "command", cmdname) < 0) + goto error; + VIR_DEBUG("Make command: %s", cmdname); + return obj; + + error: + virJSONValueFree(obj); + return NULL; +} + +static int +acrnMonitorCommand(acrnMonitorPtr mon, + virJSONValuePtr cmd, + virJSONValuePtr *reply, + int seconds) +{ + int ret = -1; + char *cmdstr = NULL; + char *txBuffer; + int txLength; + acrnMonitorMessage msg; + virTimeBackOffVar timebackoff; + + *reply = NULL; + msg.rxObject = NULL; + + VIR_DEBUG("acrnMonitorCommand: send destroy command"); + if (!(cmdstr = virJSONValueToString(cmd, false))) + goto cleanup; + txBuffer = g_strdup_printf("%s" LINE_ENDING, cmdstr); + txLength = strlen(txBuffer); + + mon->msg = &msg; + + VIR_DEBUG("Send command '%s' for write, seconds = %d", cmdstr, seconds); + + ret = acrnMonitorIOWriteWithFD(mon, txBuffer, txLength, mon->fd); + + VIR_DEBUG("Receive command reply ret=%d", ret); + + if (virTimeBackOffStart(&timebackoff, 1, ACRN_DEFAULT_MONITOR_WAIT * 1000) < 0) + goto cleanup; + + while (virTimeBackOffWait(&timebackoff)) { + virObjectLock(mon); + if (msg.rxObject) + break; + virObjectUnlock(mon); + } + + if (ret > 0) { + if (!msg.rxObject) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Missing monitor reply object")); + ret = -1; + } else { + VIR_DEBUG("Get command reply object."); + *reply = msg.rxObject; + } + } + + cleanup: + VIR_FREE(cmdstr); + VIR_FREE(txBuffer); + + return ret; +} +static int +acrnMonitorJSONCheckError(virJSONValuePtr reply) +{ + char *str; + virJSONValuePtr data; + virJSONType type; + const char *num_str; + int ret = -1; + + if ((reply != NULL) && virJSONValueObjectHasKey(reply, "ack")) { + str = virJSONValueToString(reply, false); + if (str != NULL) + VIR_DEBUG("Receive reply string=%s.", str); + + data = virJSONValueObjectGet(reply, "ack"); + if (data != NULL) { + type = virJSONValueGetType(data); + VIR_DEBUG("Receive reply type=%d.", type); + } + + num_str = virJSONValueGetNumberString(data); + if (num_str != NULL) { + VIR_DEBUG("Receive number string=%s.", num_str); + if (strcmp(num_str, "0") == 0) { + ret = 0; + } else { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Receive error: '%s'"), num_str); + } + } + } + return ret; +} +int +acrnMonitorSystemPowerdown(acrnMonitorPtr mon) +{ + int ret; + virJSONValuePtr cmd; + virJSONValuePtr reply = NULL; + + if (!mon) + return -1; + VIR_DEBUG("acrnMonitorSystemPowerdown: send destroy command"); + cmd = acrnMonitorMakeCommand("destroy"); + if (!cmd) + return -1; + + ret = acrnMonitorCommand(mon, cmd, &reply, 60); + ret = acrnMonitorJSONCheckError(reply); + + return ret; +} + +static void +acrnMonitorUpdateWatch(acrnMonitorPtr mon) +{ + int events = + VIR_EVENT_HANDLE_HANGUP | + VIR_EVENT_HANDLE_ERROR; + + if (!mon->watch) + return; + + events |= VIR_EVENT_HANDLE_READABLE; + virEventUpdateHandle(mon->watch, events); +} + +static void +acrnMonitorFreeCB(void *opaque) +{ + acrnMonitorPtr mon = opaque; + virDomainObjPtr domain = mon->vm; + acrnDomainObjPrivatePtr priv = domain->privateData; + + if (priv && priv->mon && priv->mon->stop) + priv->mon->stop(domain); +} + +static void +acrnMonitorIO(int watch, int fd, int events, void *opaque) +{ + acrnMonitorPtr mon = opaque; + bool error = false; + bool eof = false; + bool hangup = false; + + virObjectRef(mon); + /* lock access to the monitor and protect fd */ + virObjectLock(mon); + + if (mon->fd == -1 || mon->watch == 0) { + virObjectUnlock(mon); + virObjectUnref(mon); + return; + } + + VIR_DEBUG("Monitor %p I/O on watch %d fd %d events %d", mon, watch, fd, events); + if (mon->fd != fd || mon->watch != watch) { + if (events & (VIR_EVENT_HANDLE_HANGUP | VIR_EVENT_HANDLE_ERROR)) + eof = true; + virReportError(VIR_ERR_INTERNAL_ERROR, + _("event from unexpected fd %d!=%d / watch %d!=%d"), + mon->fd, fd, mon->watch, watch); + error = true; + } else { + if (!error && (events & VIR_EVENT_HANDLE_READABLE)) { + int got = acrnMonitorIORead(mon); + events &= ~VIR_EVENT_HANDLE_READABLE; + if (got < 0) { + error = true; + if (errno == ECONNRESET) + hangup = true; + } else if (got == 0) { + eof = true; + } else { + /* Ignore hangup/error events if we read some data, to + * give time for that data to be consumed */ + events = 0; + + if (acrnMonitorIOProcess(mon) < 0) + error = true; + } + } + + if (events & VIR_EVENT_HANDLE_HANGUP) { + hangup = true; + if (!error) { + VIR_DEBUG("End of file from acrn monitor"); + eof = true; + events &= ~VIR_EVENT_HANDLE_HANGUP; + } + } + + if (!error && !eof && + events & VIR_EVENT_HANDLE_ERROR) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Invalid file descriptor while waiting for monitor")); + eof = true; + events &= ~VIR_EVENT_HANDLE_ERROR; + } + + } + + acrnMonitorUpdateWatch(mon); + + if (eof) { + VIR_DEBUG("acrnMonitorIO: EOF."); + mon->shutdown_reason = VIR_DOMAIN_SHUTOFF_SHUTDOWN; + virObjectUnlock(mon); + acrnMonitorUnregister(mon); + acrnMonitorFreeCB(mon); + virObjectUnref(mon); + } else if (error) { + VIR_DEBUG("acrnMonitorIO: error."); + mon->shutdown_reason = VIR_DOMAIN_SHUTOFF_UNKNOWN; + virObjectUnlock(mon); + acrnMonitorUnregister(mon); + virObjectUnref(mon); + } else { + mon->shutdown_reason = VIR_DOMAIN_SHUTOFF_UNKNOWN; + virObjectUnlock(mon); + acrnMonitorUnregister(mon); + virObjectUnref(mon); + } +} + +bool +acrnMonitorRegister(acrnMonitorPtr mon) +{ + virObjectRef(mon); + if ((mon->watch = virEventAddHandle(mon->fd, + VIR_EVENT_HANDLE_HANGUP | + VIR_EVENT_HANDLE_ERROR | + VIR_EVENT_HANDLE_READABLE, + acrnMonitorIO, + mon, + virObjectFreeCallback)) < 0) { + virObjectUnref(mon); + return false; + } + + return true; +} + +void +acrnMonitorUnregister(acrnMonitorPtr mon) +{ + if (mon->watch) { + virEventRemoveHandle(mon->watch); + mon->watch = 0; + } +} + +int +acrnMonitorGetReason(acrnMonitorPtr mon) +{ + return mon->shutdown_reason; +} + +static int +acrnMonitorOpenUnix(const char *monitor) +{ + struct sockaddr_un addr; + int monfd; + int ret = -1; + virTimeBackOffVar timebackoff; + + if (monitor == NULL) { + VIR_DEBUG("Socket path is NULL"); + return -1; + } + if ((monfd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) { + virReportSystemError(errno, + "%s", _("failed to create socket")); + return -1; + } + VIR_DEBUG("acrnMonitorOpenUnix:create sock"); + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + if (virStrcpyStatic(addr.sun_path, monitor) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Monitor path %s too big for destination"), monitor); + goto error; + } + if (virTimeBackOffStart(&timebackoff, 1, ACRN_DEFAULT_MONITOR_WAIT * 1000) < 0) + goto error; + while (virTimeBackOffWait(&timebackoff)) { + ret = connect(monfd, (struct sockaddr *) &addr, sizeof(addr)); + if (ret == 0) + break; + + if (errno == ENOENT || errno == ECONNREFUSED) { + /* ENOENT : Socket may not have shown up yet + * ECONNREFUSED : Leftover socket hasn't been removed yet */ + continue; + } + virReportSystemError(errno, "%s", + _("failed to connect to monitor socket")); + goto error; + } + VIR_DEBUG("acrnMonitorOpenUnix:connect to monitor sock:fd=%d", monfd); + return monfd; + + error: + VIR_FORCE_CLOSE(monfd); + return -1; +} +void +acrnMonitorClose(acrnMonitorPtr mon) +{ + if (!mon) + return; + virMutexLock(&mon->lock); + if (mon->fd >= 0) { + VIR_FORCE_CLOSE(mon->fd); + } + virMutexUnlock(&mon->lock); +} + +static acrnMonitorPtr +acrnMonitorOpenInternal(virDomainObjPtr vm, + int fd, + acrnMonitorStopCallback cb) +{ + acrnMonitorPtr mon = NULL; + + if (!cb) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Callback must be supplied")); + return NULL; + } + + if (VIR_ALLOC(mon) < 0) + return NULL; + if (virMutexInit(&mon->lock) < 0) { + VIR_FREE(mon); + return NULL; + } + + mon->fd = fd; + mon->vm = virObjectRef(vm); + mon->stop = cb; + + if (virSetCloseExec(mon->fd) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("Unable to set monitor close-on-exec flag")); + goto cleanup; + } + if (virSetNonBlock(mon->fd) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("Unable to put monitor into non-blocking mode")); + goto cleanup; + } + + virObjectLock(mon); + if (!acrnMonitorRegister(mon)) { + virObjectUnlock(mon); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("unable to register monitor events")); + goto cleanup; + } + virObjectUnlock(mon); + + return mon; + +cleanup: + mon->stop = NULL; + /* The caller owns 'fd' on failure */ + mon->fd = -1; + acrnMonitorClose(mon); + return NULL; +} + +acrnMonitorPtr +acrnMonitorOpen(virDomainObjPtr vm, + virDomainChrSourceDefPtr config, + acrnMonitorStopCallback cb) +{ + int fd = -1; + acrnMonitorPtr ret = NULL; + + virObjectRef(vm); + + if (config->type != VIR_DOMAIN_CHR_TYPE_UNIX) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("unable to handle monitor type: %s"), + virDomainChrTypeToString(config->type)); + goto cleanup; + } + + fd = acrnMonitorOpenUnix(config->data.nix.path); + + if (fd < 0) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("domain is not running")); + goto cleanup; + } + + ret = acrnMonitorOpenInternal(vm, fd, cb); + +cleanup: + if (!ret) + VIR_FORCE_CLOSE(fd); + virObjectUnref(vm); + return ret; + +} \ No newline at end of file diff --git a/src/acrn/acrn_monitor.h b/src/acrn/acrn_monitor.h new file mode 100644 index 0000000000..274c54074a --- /dev/null +++ b/src/acrn/acrn_monitor.h @@ -0,0 +1,38 @@ +#ifndef __ACRN_DEVICE_H__ +#define __ACRN_DEVICE_H__ + +#include "domain_conf.h" + +typedef struct _acrnMonitor acrnMonitor; +typedef acrnMonitor *acrnMonitorPtr; + +typedef void (*acrnMonitorStopCallback)(virDomainObjPtr vm); +typedef struct _acrnMonitorMessage acrnMonitorMessage; +typedef acrnMonitorMessage *acrnMonitorMessagePtr; + +struct _acrnMonitorMessage { + int txFD; + + char *txBuffer; + int txOffset; + int txLength; + + /* Used by the text monitor reply / error */ + char *rxBuffer; + int rxLength; + /* Used by the JSON monitor to hold reply / error */ + void *rxObject; + + /* True if rxBuffer / rxObject are ready, or a + * fatal error occurred on the monitor channel + */ + bool finished; +}; + +int acrnMonitorSystemPowerdown(acrnMonitorPtr mon); +acrnMonitorPtr acrnMonitorOpen(virDomainObjPtr vm, virDomainChrSourceDefPtr config, acrnMonitorStopCallback cb); +void acrnMonitorClose(acrnMonitorPtr mon); +bool acrnMonitorRegister(acrnMonitorPtr mon); +void acrnMonitorUnregister(acrnMonitorPtr mon); +int acrnMonitorGetReason(acrnMonitorPtr mon); +#endif /* __ACRN_DEVICE_H__ */ From ecb37adc9db774de352d93ab79c892cf00d7ba52 Mon Sep 17 00:00:00 2001 From: Xiangyang Wu Date: Sun, 27 Feb 2022 12:56:37 +0800 Subject: [PATCH 22/32] acrn-driver: support shutdown user VM gracefully The following updates are maded to support shutdown user VM gracefully: - Implement a acrn manager to interact with ACRN lifecycle manager. - Send user_vm_shutdown command to ACRN lifecycle manager to shutdown a specified user VM. Tracked-On: #5921 Signed-off-by: Xiangyang Wu --- src/acrn/Makefile.inc.am | 2 + src/acrn/acrn_domain.h | 5 + src/acrn/acrn_driver.c | 72 ++++++- src/acrn/acrn_manager.c | 438 +++++++++++++++++++++++++++++++++++++++ src/acrn/acrn_manager.h | 38 ++++ 5 files changed, 550 insertions(+), 5 deletions(-) create mode 100644 src/acrn/acrn_manager.c create mode 100644 src/acrn/acrn_manager.h diff --git a/src/acrn/Makefile.inc.am b/src/acrn/Makefile.inc.am index fbc67038c3..5d03ba3797 100644 --- a/src/acrn/Makefile.inc.am +++ b/src/acrn/Makefile.inc.am @@ -10,6 +10,8 @@ ACRN_DRIVER_SOURCES = \ acrn/acrn_device.c \ acrn/acrn_monitor.h \ acrn/acrn_monitor.c \ + acrn/acrn_manager.h \ + acrn/acrn_manager.c \ $(NULL) DRIVER_SOURCE_FILES += $(addprefix $(srcdir)/,$(ACRN_DRIVER_SOURCES)) diff --git a/src/acrn/acrn_domain.h b/src/acrn/acrn_domain.h index 43d16d1048..b9f75243dd 100644 --- a/src/acrn/acrn_domain.h +++ b/src/acrn/acrn_domain.h @@ -3,6 +3,7 @@ #include "domain_conf.h" #include "acrn_monitor.h" +#include "acrn_manager.h" typedef struct _acrnDomainObjPrivate acrnDomainObjPrivate; typedef acrnDomainObjPrivate *acrnDomainObjPrivatePtr; @@ -18,6 +19,10 @@ struct _acrnDomainObjPrivate { virDomainChrSourceDefPtr monConfig; char *libDir; /* base path for per-domain files */ acrnMonitorPtr mon; + + virDomainChrSourceDefPtr mgrConfig; + char *mgrDir; + acrnManagerPtr mgr; }; typedef struct _acrnDomainXmlNsDef acrnDomainXmlNsDef; diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 07303349e7..bbc18e3a0b 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -27,6 +27,7 @@ #include "acrn_driver.h" #include "acrn_domain.h" #include "acrn_monitor.h" +#include "acrn_manager.h" #define VIR_FROM_THIS VIR_FROM_ACRN #define ACRN_DM_PATH "/usr/bin/acrn-dm" @@ -37,6 +38,7 @@ #define ACRN_CONFIG_DIR SYSCONFDIR "/libvirt/acrn" #define ACRN_STATE_DIR RUNSTATEDIR "/libvirt/acrn" #define ACRN_MONITOR_DIR "/var/lib/libvirt/acrn" +#define ACRN_MANAGER_DIR "/var/lib/life_mngr" #define ACRN_NET_GENERATED_TAP_PREFIX "tap" #define ACRN_PI_VERSION (0x100) @@ -743,7 +745,34 @@ acrnProcessWaitForMonitor(virDomainObjPtr vm, acrnMonitorStopCallback stop) VIR_DEBUG("acrnProcessWaitForMonitor:end"); return 0; } +static int +acrnProcessWaitForManager(virDomainObjPtr vm, acrnManagerStopCallback stop) +{ + acrnDomainObjPrivatePtr priv = vm->privateData; + virDomainChrSourceDefPtr config = priv->mgrConfig; + acrnManagerPtr mon = NULL; + + if (!priv->mgrDir) + priv->mgrDir = g_strdup_printf("%s", ACRN_MANAGER_DIR); + + if (!(config = virDomainChrSourceDefNew(acrn_driver->xmlopt))) + return -1; + + if (acrnProcessPrepareMonitorChr(config, priv->mgrDir) < 0) + return -1; + VIR_DEBUG("Manager UNIX socket path:%s", config->data.nix.path); + + mon = acrnManagerOpen(vm, config, stop); + + priv->mgr = mon; + if (priv->mgr == NULL) { + VIR_INFO("Failed to connect acrn manager for %s", vm->def->name); + return -1; + } + VIR_DEBUG("acrnProcessWaitForManager:end"); + return 0; +} static virCommandPtr acrnRunStartCommand(virDomainObjPtr vm) { @@ -854,6 +883,13 @@ acrnDomainShutdownMonitor(virDomainObjPtr vm) return acrnMonitorSystemPowerdown(priv->mon); } +static int +acrnDomainShutdownManager(virDomainObjPtr vm) +{ + acrnDomainObjPrivatePtr priv = vm->privateData; + + return acrnManagerSystemPowerdown(priv->mgr); +} static void acrnProcessCleanup(virDomainObjPtr vm, int reason, size_t *allocMap) @@ -866,6 +902,14 @@ acrnProcessCleanup(virDomainObjPtr vm, int reason, size_t *allocMap) /* clean up ttys */ acrnTtyCleanup(vm); + acrnManagerClose(priv->mgr); + if (priv->mgrConfig) { + if (priv->mgrConfig->type == VIR_DOMAIN_CHR_TYPE_UNIX) + unlink(priv->mgrConfig->data.nix.path); + virObjectUnref(priv->mgrConfig); + priv->mgrConfig = NULL; + } + acrnMonitorClose(priv->mon); if (priv->monConfig) { if (priv->monConfig->type == VIR_DOMAIN_CHR_TYPE_UNIX) @@ -883,6 +927,27 @@ acrnProcessCleanup(virDomainObjPtr vm, int reason, size_t *allocMap) virDomainDeleteConfig(ACRN_STATE_DIR, NULL, vm); } +static int +acrnProcessShutdown(virDomainObjPtr vm, int reason) +{ + virDomainDefPtr def = vm->def; + int ret = 0; + + VIR_DEBUG("Waiting for acrn manager"); + if (acrnProcessWaitForManager(vm, NULL) < 0) + return -1; + + VIR_DEBUG("Stopping domain '%s'", def->name); + if (acrnDomainShutdownManager(vm) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("Fail to stop domain '%s'"), def->name); + ret = -1; + } + virDomainObjSetState(vm, VIR_DOMAIN_SHUTDOWN, reason); + + return ret; +} + static int acrnProcessStop(virDomainObjPtr vm, int reason) { @@ -893,7 +958,7 @@ acrnProcessStop(virDomainObjPtr vm, int reason) if (acrnDomainShutdownMonitor(vm) < 0) { virReportError(VIR_ERR_OPERATION_INVALID, - _("Fail to stop domain '%s'"), def->name); + _("Fail to stop domain '%s'"), def->name); ret = -1; } virDomainObjSetState(vm, VIR_DOMAIN_SHUTDOWN, reason); @@ -982,7 +1047,7 @@ acrnDomainShutdown(virDomainPtr dom) goto cleanup; } - if (acrnProcessStop(vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN) < 0) { + if (acrnProcessShutdown(vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN) < 0) { goto cleanup; } @@ -994,9 +1059,6 @@ acrnDomainShutdown(virDomainPtr dom) ret = 0; - if (!vm->persistent) - virDomainObjListRemove(privconn->domains, vm); - cleanup: if (vm) virObjectUnlock(vm); diff --git a/src/acrn/acrn_manager.c b/src/acrn/acrn_manager.c new file mode 100644 index 0000000000..dbe9a840be --- /dev/null +++ b/src/acrn/acrn_manager.c @@ -0,0 +1,438 @@ +#include +#include +#include +#include +#include "datatypes.h" +#include "virfile.h" +#include "viralloc.h" +#include "virutil.h" +#include "vircommand.h" +#include "virthread.h" +#include "virstring.h" +#include "domain_event.h" +#include "virjson.h" +#include "acrn_manager.h" +#include "virtime.h" +#include "virerror.h" +#include "virlog.h" +#include "acrn_domain.h" + +#define VIR_FROM_THIS VIR_FROM_ACRN +#define ACRN_DEFAULT_MONITOR_WAIT 30 + +#define LINE_ENDING "\n" + +VIR_LOG_INIT("acrn.acrn_manager"); + +struct _acrnManager { + int fd; + int watch; + virDomainObjPtr vm; + virMutex lock; + + acrnManagerMessagePtr msg; + /* Buffer incoming data ready for acrn lifecycle manager + * code to process & find message boundaries */ + char buffer[1024]; + + acrnManagerStopCallback stop; + int shutdown_reason; +}; + +static int +acrnManagerIOWriteWithFD(acrnManagerPtr mon, + const char *data, + size_t len, + int fd) +{ + struct msghdr msg; + struct iovec iov[1]; + int ret; + char control[CMSG_SPACE(sizeof(int))]; + struct cmsghdr *cmsg; + + memset(&msg, 0, sizeof(msg)); + memset(control, 0, sizeof(control)); + + iov[0].iov_base = (void *)data; + iov[0].iov_len = len; + + msg.msg_iov = iov; + msg.msg_iovlen = 1; + + msg.msg_control = control; + msg.msg_controllen = sizeof(control); + + cmsg = CMSG_FIRSTHDR(&msg); + /* Some static analyzers, like clang 2.6-0.6.pre2, fail to see + that our use of CMSG_FIRSTHDR will not return NULL. */ + sa_assert(cmsg); + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + memcpy(CMSG_DATA(cmsg), &fd, sizeof(int)); + + do { + ret = sendmsg(mon->fd, &msg, 0); + } while (ret < 0 && errno == EINTR); + + return ret; +} + +static int +acrnManagerIORead(acrnManagerPtr mon) +{ + int ret = 0; + int got; + + memset(mon->buffer, 0x0, 1024); + got = read(mon->fd, + mon->buffer, + 1023); + if (got < 0) { + if (errno == EAGAIN) + virReportSystemError(errno, "%s", + _("Unable to read from monitor")); + ret = -1; + } + + ret += got; + mon->buffer[got] = '\0'; + + VIR_DEBUG("Now read %d bytes of data: %s.", got, mon->buffer); + + + return ret; +} + +static int +acrnManagerIOProcess(acrnManagerPtr mon) +{ + return 0; +} + + +static int +acrnManagerCommand(acrnManagerPtr mon, + const char *cmd, + int seconds) +{ + int ret = -1; + char *txBuffer; + int txLength; + acrnManagerMessage msg; + virDomainObjPtr vm = mon->vm; + + msg.rxObject = NULL; + + VIR_DEBUG("acrnManagerCommand: $s", cmd); + txBuffer = g_strdup_printf("%s:%s", cmd, vm->def->name); + txLength = strlen(txBuffer); + + mon->msg = &msg; + + VIR_DEBUG("Send command '%s' for write, seconds = %d", txBuffer, seconds); + + ret = acrnManagerIOWriteWithFD(mon, txBuffer, txLength, mon->fd); + + VIR_DEBUG("Receive command reply ret=%d", ret); + + cleanup: + VIR_FREE(txBuffer); + + return 0; +} + +int +acrnManagerSystemPowerdown(acrnManagerPtr mon) +{ + int ret; + const char *cmd = "user_vm_shutdown"; + + if (!mon) + return -1; + VIR_DEBUG("acrnManagerSystemPowerdown: send shutdown command"); + + ret = acrnManagerCommand(mon, cmd, 60); + + return ret; +} + +static void +acrnManagerUpdateWatch(acrnManagerPtr mon) +{ + int events = + VIR_EVENT_HANDLE_HANGUP | + VIR_EVENT_HANDLE_ERROR; + + if (!mon->watch) + return; + + events |= VIR_EVENT_HANDLE_READABLE; + virEventUpdateHandle(mon->watch, events); +} + +static void +acrnManagerIO(int watch, int fd, int events, void *opaque) +{ + acrnManagerPtr mon = opaque; + bool error = false; + bool eof = false; + bool hangup = false; + + virObjectRef(mon); + /* lock access to the monitor and protect fd */ + virObjectLock(mon); + + if (mon->fd == -1 || mon->watch == 0) { + virObjectUnlock(mon); + virObjectUnref(mon); + return; + } + + VIR_DEBUG("Manager %p I/O on watch %d fd %d events %d", mon, watch, fd, events); + if (mon->fd != fd || mon->watch != watch) { + if (events & (VIR_EVENT_HANDLE_HANGUP | VIR_EVENT_HANDLE_ERROR)) + eof = true; + virReportError(VIR_ERR_INTERNAL_ERROR, + _("event from unexpected fd %d!=%d / watch %d!=%d"), + mon->fd, fd, mon->watch, watch); + error = true; + } else { + if (!error && (events & VIR_EVENT_HANDLE_READABLE)) { + int got = acrnManagerIORead(mon); + events &= ~VIR_EVENT_HANDLE_READABLE; + if (got < 0) { + error = true; + if (errno == ECONNRESET) + hangup = true; + } else if (got == 0) { + eof = true; + } else { + /* Ignore hangup/error events if we read some data, to + * give time for that data to be consumed */ + events = 0; + + if (acrnManagerIOProcess(mon) < 0) + error = true; + } + } + + if (events & VIR_EVENT_HANDLE_HANGUP) { + hangup = true; + if (!error) { + VIR_DEBUG("End of file from acrn manager"); + eof = true; + events &= ~VIR_EVENT_HANDLE_HANGUP; + } + } + + if (!error && !eof && + events & VIR_EVENT_HANDLE_ERROR) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Invalid file descriptor while waiting for manager")); + eof = true; + events &= ~VIR_EVENT_HANDLE_ERROR; + } + + } + + acrnManagerUpdateWatch(mon); + + if (eof) { + VIR_DEBUG("acrnManagerIO: EOF."); + mon->shutdown_reason = VIR_DOMAIN_SHUTOFF_SHUTDOWN; + virObjectUnlock(mon); + acrnManagerUnregister(mon); + virObjectUnref(mon); + } else if (error) { + VIR_DEBUG("acrnManagerIO: error."); + virObjectUnlock(mon); + acrnManagerUnregister(mon); + virObjectUnref(mon); + } else { + mon->shutdown_reason = VIR_DOMAIN_SHUTOFF_UNKNOWN; + virObjectUnlock(mon); + acrnManagerUnregister(mon); + virObjectUnref(mon); + } +} + +bool +acrnManagerRegister(acrnManagerPtr mon) +{ + virObjectRef(mon); + if ((mon->watch = virEventAddHandle(mon->fd, + VIR_EVENT_HANDLE_HANGUP | + VIR_EVENT_HANDLE_ERROR | + VIR_EVENT_HANDLE_READABLE, + acrnManagerIO, + mon, + virObjectFreeCallback)) < 0) { + virObjectUnref(mon); + return false; + } + + return true; +} + +void +acrnManagerUnregister(acrnManagerPtr mon) +{ + if (mon->watch) { + virEventRemoveHandle(mon->watch); + mon->watch = 0; + } +} + +int +acrnManagerGetReason(acrnManagerPtr mon) +{ + return mon->shutdown_reason; +} + +static int +acrnManagerOpenUnix(const char *monitor) +{ + struct sockaddr_un addr; + int monfd; + int ret = -1; + virTimeBackOffVar timebackoff; + + if (monitor == NULL) { + VIR_DEBUG("Socket path is NULL"); + return -1; + } + if ((monfd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) { + virReportSystemError(errno, + "%s", _("failed to create socket")); + return -1; + } + VIR_DEBUG("acrnManagerOpenUnix:create sock"); + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + if (virStrcpyStatic(addr.sun_path, monitor) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Monitor path %s too big for destination"), monitor); + goto error; + } + if (virTimeBackOffStart(&timebackoff, 1, ACRN_DEFAULT_MONITOR_WAIT * 1000) < 0) + goto error; + while (virTimeBackOffWait(&timebackoff)) { + ret = connect(monfd, (struct sockaddr *) &addr, sizeof(addr)); + if (ret == 0) + break; + + if (errno == ENOENT || errno == ECONNREFUSED) { + /* ENOENT : Socket may not have shown up yet + * ECONNREFUSED : Leftover socket hasn't been removed yet */ + continue; + } + virReportSystemError(errno, "%s", + _("failed to connect to monitor socket")); + goto error; + } + VIR_DEBUG("acrnManagerOpenUnix:connect to monitor sock:fd=%d", monfd); + return monfd; + + error: + VIR_FORCE_CLOSE(monfd); + return -1; +} +void +acrnManagerClose(acrnManagerPtr mon) +{ + if (!mon) + return; + virMutexLock(&mon->lock); + if (mon->fd >= 0) { + VIR_FORCE_CLOSE(mon->fd); + } + virMutexUnlock(&mon->lock); +} + +static acrnManagerPtr +acrnManagerOpenInternal(virDomainObjPtr vm, + int fd, + acrnManagerStopCallback cb) +{ + acrnManagerPtr mon = NULL; + + if (VIR_ALLOC(mon) < 0) + return NULL; + if (virMutexInit(&mon->lock) < 0) { + VIR_FREE(mon); + return NULL; + } + + mon->fd = fd; + mon->vm = virObjectRef(vm); + mon->stop = cb; + + if (virSetCloseExec(mon->fd) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("Unable to set monitor close-on-exec flag")); + goto cleanup; + } + if (virSetNonBlock(mon->fd) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("Unable to put monitor into non-blocking mode")); + goto cleanup; + } + + virObjectLock(mon); + if (!acrnManagerRegister(mon)) { + virObjectUnlock(mon); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("unable to register monitor events")); + goto cleanup; + } + virObjectUnlock(mon); + + return mon; + +cleanup: + mon->stop = NULL; + /* The caller owns 'fd' on failure */ + mon->fd = -1; + acrnManagerClose(mon); + return NULL; +} + +acrnManagerPtr +acrnManagerOpen(virDomainObjPtr vm, + virDomainChrSourceDefPtr config, + acrnManagerStopCallback cb) +{ + int fd = -1; + acrnManagerPtr ret = NULL; + + virObjectRef(vm); + + if (config->type != VIR_DOMAIN_CHR_TYPE_UNIX) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("unable to handle monitor type: %s"), + virDomainChrTypeToString(config->type)); + goto cleanup; + } + + fd = acrnManagerOpenUnix(config->data.nix.path); + + if (fd < 0) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("domain is not running")); + goto cleanup; + } + + ret = acrnManagerOpenInternal(vm, fd, cb); + +cleanup: + if (!ret) + VIR_FORCE_CLOSE(fd); + virObjectUnref(vm); + return ret; + +} \ No newline at end of file diff --git a/src/acrn/acrn_manager.h b/src/acrn/acrn_manager.h new file mode 100644 index 0000000000..d93c65dff6 --- /dev/null +++ b/src/acrn/acrn_manager.h @@ -0,0 +1,38 @@ +#ifndef __ACRN_MANAGER_H__ +#define __ACRN_MANAGER_H__ + +#include "domain_conf.h" + +typedef struct _acrnManager acrnManager; +typedef acrnManager *acrnManagerPtr; + +typedef void (*acrnManagerStopCallback)(virDomainObjPtr vm); +typedef struct _acrnManagerMessage acrnManagerMessage; +typedef acrnManagerMessage *acrnManagerMessagePtr; + +struct _acrnManagerMessage { + int txFD; + + char *txBuffer; + int txOffset; + int txLength; + + /* Used by the text monitor reply / error */ + char *rxBuffer; + int rxLength; + /* Used by the JSON monitor to hold reply / error */ + void *rxObject; + + /* True if rxBuffer / rxObject are ready, or a + * fatal error occurred on the monitor channel + */ + bool finished; +}; + +int acrnManagerSystemPowerdown(acrnManagerPtr mon); +acrnManagerPtr acrnManagerOpen(virDomainObjPtr vm, virDomainChrSourceDefPtr config, acrnManagerStopCallback cb); +void acrnManagerClose(acrnManagerPtr mon); +bool acrnManagerRegister(acrnManagerPtr mon); +void acrnManagerUnregister(acrnManagerPtr mon); +int acrnManagerGetReason(acrnManagerPtr mon); +#endif /* __ACRN_MANAGER_H__ */ From bc8c077ebdca2ace9b8071e49cfab4c271710827 Mon Sep 17 00:00:00 2001 From: Xiangyang Wu Date: Tue, 1 Mar 2022 09:36:05 +0800 Subject: [PATCH 23/32] acrn-driver: support user VM reboot The follwing updates are made to support user VM reboot: - Implement acrnDomainReboot in acrn driver; - Send user_vm_reboot command to ACRN lifecycle manager. Tracked-On: #5921 Signed-off-by: Xiangyang Wu --- src/acrn/acrn_driver.c | 62 +++++++++++++++++++++++++++++++++++------ src/acrn/acrn_manager.c | 15 ++++++++++ src/acrn/acrn_manager.h | 1 + 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index bbc18e3a0b..92329daf21 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -890,6 +890,13 @@ acrnDomainShutdownManager(virDomainObjPtr vm) return acrnManagerSystemPowerdown(priv->mgr); } +static int +acrnDomainRebootManager(virDomainObjPtr vm) +{ + acrnDomainObjPrivatePtr priv = vm->privateData; + + return acrnManagerSystemReboot(priv->mgr); +} static void acrnProcessCleanup(virDomainObjPtr vm, int reason, size_t *allocMap) @@ -928,7 +935,7 @@ acrnProcessCleanup(virDomainObjPtr vm, int reason, size_t *allocMap) } static int -acrnProcessShutdown(virDomainObjPtr vm, int reason) +acrnProcessShutdown(virDomainObjPtr vm, int reason, bool reboot) { virDomainDefPtr def = vm->def; int ret = 0; @@ -937,13 +944,22 @@ acrnProcessShutdown(virDomainObjPtr vm, int reason) if (acrnProcessWaitForManager(vm, NULL) < 0) return -1; - VIR_DEBUG("Stopping domain '%s'", def->name); - if (acrnDomainShutdownManager(vm) < 0) { - virReportError(VIR_ERR_OPERATION_INVALID, - _("Fail to stop domain '%s'"), def->name); - ret = -1; + if (reboot) { + VIR_DEBUG("Rebooting domain '%s'", def->name); + if (acrnDomainRebootManager(vm) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("Fail to reboot domain '%s'"), def->name); + ret = -1; + } + } else { + VIR_DEBUG("Stopping domain '%s'", def->name); + if (acrnDomainShutdownManager(vm) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("Fail to stop domain '%s'"), def->name); + ret = -1; + } + virDomainObjSetState(vm, VIR_DOMAIN_SHUTDOWN, reason); } - virDomainObjSetState(vm, VIR_DOMAIN_SHUTDOWN, reason); return ret; } @@ -1047,7 +1063,7 @@ acrnDomainShutdown(virDomainPtr dom) goto cleanup; } - if (acrnProcessShutdown(vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN) < 0) { + if (acrnProcessShutdown(vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, false) < 0) { goto cleanup; } @@ -1068,6 +1084,35 @@ acrnDomainShutdown(virDomainPtr dom) return ret; } +static int +acrnDomainReboot(virDomainPtr dom, unsigned int flags) +{ + acrnConnectPtr privconn = dom->conn->privateData; + virDomainObjPtr vm; + int ret = -1; + + acrnDriverLock(privconn); + + if (!(vm = acrnDomObjFromDomain(dom))) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("domain is not running")); + goto cleanup; + } + + if (acrnProcessShutdown(vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, true) < 0) { + goto cleanup; + } + + ret = 0; + +cleanup: + acrnDriverUnlock(privconn); + return ret; +} + static int acrnDomainDestroy(virDomainPtr dom) { @@ -2571,6 +2616,7 @@ static virHypervisorDriver acrnHypervisorDriver = { .domainLookupByUUID = acrnDomainLookupByUUID, /* 0.0.1 */ .domainLookupByName = acrnDomainLookupByName, /* 0.0.1 */ .domainShutdown = acrnDomainShutdown, /* 0.0.1 */ + .domainReboot = acrnDomainReboot, /* 0.0.1 */ .domainDestroy = acrnDomainDestroy, /* 0.0.1 */ .domainIsPersistent = acrnDomainIsPersistent, /* 0.0.1 */ .domainGetAutostart = acrnDomainGetAutostart, /* 0.0.1 */ diff --git a/src/acrn/acrn_manager.c b/src/acrn/acrn_manager.c index dbe9a840be..510c499f72 100644 --- a/src/acrn/acrn_manager.c +++ b/src/acrn/acrn_manager.c @@ -158,6 +158,21 @@ acrnManagerSystemPowerdown(acrnManagerPtr mon) return ret; } +int +acrnManagerSystemReboot(acrnManagerPtr mon) +{ + int ret; + const char *cmd = "user_vm_reboot"; + + if (!mon) + return -1; + VIR_DEBUG("acrnManagerSystemPowerdown: send reboot command"); + + ret = acrnManagerCommand(mon, cmd, 60); + + return ret; +} + static void acrnManagerUpdateWatch(acrnManagerPtr mon) { diff --git a/src/acrn/acrn_manager.h b/src/acrn/acrn_manager.h index d93c65dff6..8613a32589 100644 --- a/src/acrn/acrn_manager.h +++ b/src/acrn/acrn_manager.h @@ -30,6 +30,7 @@ struct _acrnManagerMessage { }; int acrnManagerSystemPowerdown(acrnManagerPtr mon); +int acrnManagerSystemReboot(acrnManagerPtr mon); acrnManagerPtr acrnManagerOpen(virDomainObjPtr vm, virDomainChrSourceDefPtr config, acrnManagerStopCallback cb); void acrnManagerClose(acrnManagerPtr mon); bool acrnManagerRegister(acrnManagerPtr mon); From 96e0e88152c97e73f2feaa911eb8b460472de2e5 Mon Sep 17 00:00:00 2001 From: Yuanyuan Zhao Date: Tue, 1 Mar 2022 08:54:17 +0000 Subject: [PATCH 24/32] acrn-driver: add CPU affinity support CPU affinity support is added using XML namespace "acrn": - : indicate CPU affinity via The value of CPU affinity should be target pCPU's local APIC id. The 'cpuset' attribute of CPU allocation will be invalid but will not report warning. Tracked-On: #19 Signed-off-by: Yuanyuan Zhao --- src/acrn/acrn_domain.c | 23 +++++++++++++++++++---- src/acrn/acrn_domain.h | 1 + src/acrn/acrn_driver.c | 21 ++++++++++++++++++--- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/acrn/acrn_domain.c b/src/acrn/acrn_domain.c index fa32c088b8..04a7af60a6 100644 --- a/src/acrn/acrn_domain.c +++ b/src/acrn/acrn_domain.c @@ -349,6 +349,8 @@ acrnDomainDefNamespaceFree(void *nsdata) if (!nsdef) return; + if (nsdef->cpu_affinity) + VIR_FREE(nsdef->cpu_affinity); virStringListFreeCount(nsdef->args, nsdef->nargs); VIR_FREE(nsdef); } @@ -371,9 +373,9 @@ acrnDomainDefNamespaceParseConfig(acrnDomainXmlNsDefPtr nsdef, if (nnodes == 0) return 0; - if (nnodes > 1) { + if (nnodes > 2) { virReportError(VIR_ERR_XML_ERROR, - _("More than 1 acrn:config nodes")); + _("More than 2 acrn:config nodes")); return -1; } @@ -381,6 +383,14 @@ acrnDomainDefNamespaceParseConfig(acrnDomainXmlNsDefPtr nsdef, if (node->type == XML_ELEMENT_NODE) { if (virXMLNodeNameEqual(node, "rtvm")) nsdef->rtvm = true; + if (virXMLNodeNameEqual(node, "cpu_affinity")) { + nsdef->cpu_affinity = virXMLPropString(node, "value"); + if (nsdef->cpu_affinity == NULL) { + virReportError(VIR_ERR_XML_ERROR, + _("No CPU specified in acrn:cpu_affinity")); + return -1; + } + } } } @@ -449,13 +459,18 @@ static void acrnDomainDefNamespaceFormatXMLConfig(virBufferPtr buf, acrnDomainXmlNsDefPtr xmlns) { - if (!xmlns->rtvm) + if (!xmlns->rtvm && !xmlns->cpu_affinity) return; virBufferAddLit(buf, "\n"); virBufferAdjustIndent(buf, 2); - virBufferAddLit(buf, "\n"); + if (xmlns->rtvm) + virBufferAddLit(buf, "\n"); + + if (xmlns->cpu_affinity) + virBufferEscapeString(buf, "\n", + xmlns->cpu_affinity); virBufferAdjustIndent(buf, -2); virBufferAddLit(buf, "\n"); diff --git a/src/acrn/acrn_domain.h b/src/acrn/acrn_domain.h index b9f75243dd..4d35113aa0 100644 --- a/src/acrn/acrn_domain.h +++ b/src/acrn/acrn_domain.h @@ -29,6 +29,7 @@ typedef struct _acrnDomainXmlNsDef acrnDomainXmlNsDef; typedef acrnDomainXmlNsDef *acrnDomainXmlNsDefPtr; struct _acrnDomainXmlNsDef { bool rtvm; + char *cpu_affinity; size_t nargs; char **args; }; diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index 92329daf21..d9051b631a 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -126,6 +126,17 @@ acrnIsRtvm(virDomainDefPtr def) return (nsdef && nsdef->rtvm); } +static char* +acrnGetCpuAffinity(virDomainDefPtr def) +{ + acrnDomainXmlNsDefPtr nsdef = def->namespaceData; + + if (nsdef) + return nsdef->cpu_affinity; + + return NULL; +} + static int acrnAllocateVcpus(virBitmapPtr pcpus, size_t maxvcpus, size_t *allocMap, virBitmapPtr vcpus) @@ -203,6 +214,10 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, size_t *allocMap) if (!vm || !(def = vm->def)) return -1; + if (acrnGetCpuAffinity(def)) { + return 0; + } + priv = vm->privateData; if (def->cpumask == NULL || virBitmapIsAllClear(def->cpumask)) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cpuset is empty")); @@ -629,9 +644,9 @@ acrnBuildStartCmd(virDomainObjPtr vm) priv = vm->privateData; /* CPU */ - pcpus = virBitmapFormat(priv->cpuAffinitySet); - virCommandAddArgList(cmd, "--cpu_affinity", pcpus, NULL); - VIR_FREE(pcpus); + pcpus = acrnGetCpuAffinity(def); + if(pcpus) + virCommandAddArgList(cmd, "--cpu_affinity", pcpus, NULL); /* Memory */ virCommandAddArg(cmd, "-m"); From bf101b6e7c58321948de3ded8ea5a9fdf0253ef4 Mon Sep 17 00:00:00 2001 From: Yuanyuan Zhao Date: Wed, 9 Mar 2022 11:01:30 +0000 Subject: [PATCH 25/32] acrn-driver: pass `--rtvm` to acrn-dm instead of `--lapic_pt` For acrn-dm enable `lapic_pt` automatically for rtvm for better performance and reserve `--lapic_pt` for future use, translate `` to `--rtvm` instead of `--lapic_pt`. Signed-off-by: Yuanyuan Zhao --- src/acrn/acrn_driver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index d9051b631a..b620ea08b3 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -656,7 +656,7 @@ acrnBuildStartCmd(virDomainObjPtr vm) /* RTVM */ if (acrnIsRtvm(def)) virCommandAddArgList(cmd, - "--lapic_pt", + "--rtvm", "--virtio_poll", "1000000", NULL); From 660804297744084b9d39e1e2f134f096f7f73017 Mon Sep 17 00:00:00 2001 From: Yuanyuan Zhao Date: Mon, 18 Apr 2022 06:00:29 +0000 Subject: [PATCH 26/32] acrn_domain : add cpu_affinity to check list Add cpu_affinity to check list when parse acrn name space. Tracked-On: #19 Signed-off-by: Yuanyuan Zhao --- src/acrn/acrn_domain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acrn/acrn_domain.c b/src/acrn/acrn_domain.c index 04a7af60a6..69c65df202 100644 --- a/src/acrn/acrn_domain.c +++ b/src/acrn/acrn_domain.c @@ -445,7 +445,7 @@ acrnDomainDefNamespaceParse(xmlXPathContextPtr ctxt, acrnDomainDefNamespaceParseCommandlineArgs(nsdata, ctxt) < 0) goto cleanup; - if (nsdata->rtvm || nsdata->nargs) + if (nsdata->rtvm || nsdata->nargs || nsdata->cpu_affinity) *data = g_steal_pointer(&nsdata); ret = 0; From e87633c09e477dc3f83d23e7c8705d8462e0091c Mon Sep 17 00:00:00 2001 From: Yuanyuan Zhao Date: Fri, 29 Apr 2022 15:50:20 +0800 Subject: [PATCH 27/32] acrn-driver: mapping processor and apicid Analysis '/proc/cpuinfo' and create a mapping between processor and apicid before offline CPUs. When libvit allocate cpus for VM, this map can be used to generate '--cpu_affinity' parameter. Tracked-On: #19 Signed-off-by: Yuanyuan Zhao --- src/acrn/acrn_driver.c | 114 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index b620ea08b3..a2af2a2e15 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -55,6 +55,7 @@ struct _acrnConnect { virObjectEventStatePtr domainEventState; virHostdevManagerPtr hostdevMgr; size_t *vcpuAllocMap; + unsigned int *apicidMap; }; typedef struct _acrnDomainNamespaceDef acrnDomainNamespaceDef; @@ -72,6 +73,8 @@ struct acrnAutostartData { }; static acrnConnectPtr acrn_driver = NULL; +#define CPUINFO_PATH "/proc/cpuinfo" + static void acrnDriverLock(acrnConnectPtr driver) { @@ -2323,6 +2326,8 @@ acrnStateCleanup(void) if (acrn_driver->vcpuAllocMap) VIR_FREE(acrn_driver->vcpuAllocMap); + if (acrn_driver->apicidMap) + VIR_FREE(acrn_driver->apicidMap); virMutexDestroy(&acrn_driver->lock); VIR_FREE(acrn_driver); @@ -2379,6 +2384,110 @@ virAcrnCapsBuild(void) virObjectUnref(caps); return NULL; } +static int +acrnGetPorcessor(char *str) { + int processor; + char *start = strstr(str, "processor"); + if (start != NULL) { + sscanf(start, "processor\t: %u", &processor); + return processor; + } + return -1; +} +static int +acrnGetApiced(char *str) { + int apicid; + char *start = strstr(str, "apicid"); + if (start != NULL) { + sscanf(start, "apicid\t\t: %d", &apicid); + return apicid; + } + return -1; +} + +static int +acrnGetLapicMap(int nprocs, unsigned int **apicidMap) +{ +#define READ_LEN (128) + int fd = -1, ret = 0, i; + int pos = 0, len = READ_LEN, total = 0; + int processor = -1, apicid = -1; + char str[READ_LEN], *token, *next, s[2] = {0xa, 0x0}; + ssize_t rc; + unsigned int *map; + + if (VIR_ALLOC_N(map, nprocs) < 0) { + ret = -ENOMEM; + goto cleanup; + } + + if ((fd = open(CPUINFO_PATH, O_RDONLY)) < 0) { + virReportError(VIR_ERR_OPEN_FAILED, _("%s"), CPUINFO_PATH); + ret = -1; + goto cleanup; + } + + rc = pread(fd, &str[pos], len, 0); + do { + total += rc; + token = strtok(str, s); + + if(processor != -1) { + apicid = acrnGetApiced(str); + } else { + processor = acrnGetPorcessor(str); + if (processor >= nprocs) + processor = -1; + } + + if(processor != -1 && apicid != -1) { + map[processor] = apicid; + processor = -1; + apicid = -1; + } + + next = strtok(NULL, s); + do { + if(processor != -1) { + apicid = acrnGetApiced(token); + } else { + processor = acrnGetPorcessor(token); + if (processor >= nprocs) + processor = -1; + } + + if(processor != -1 && apicid != -1) { + map[processor] = apicid; + processor = -1; + apicid = -1; + } + token = next; + } while ((next = strtok(NULL, s)) != NULL); + + pos = 0; + for (i = len - 1; i > 1; i--) { + if (str[i - 1] == 0x0 || str[i-1] == 0x20) { + pos = len - i - 1; + if (pos != 0) { + memmove(str, &str[i], pos); + } + len = READ_LEN - pos; + break; + } + } + } while ((rc = pread(fd, &str[pos], len, total)) > 0); + + *apicidMap = map; + +cleanup: + if (fd > 0) + close(fd); + + if (ret != 0 && map != NULL) { + VIR_FREE(map); + } + return ret; +} /* * Vacate SOS CPUs for UOS vCPU allocation. @@ -2457,6 +2566,11 @@ acrnInitPlatform(virNodeInfoPtr nodeInfo, size_t **allocMap) nodeInfo->cpus = totalCpus; + if (acrnGetLapicMap(nodeInfo->cpus, &acrn_driver->apicidMap) < 0) { + ret = -EIO; + goto cleanup; + } + if (acrnOfflineCpus(nodeInfo->cpus) < 0) { ret = -EIO; goto cleanup; From b28fad178d135e63be50f473432b06279b786d57 Mon Sep 17 00:00:00 2001 From: Yuanyuan Zhao Date: Fri, 29 Apr 2022 16:09:35 +0800 Subject: [PATCH 28/32] acrn-driver: allocate vCPU for Guest VM If a VM only set the max CPU number and not set the CPUs to use, this patch will allocate vCPUs to the VM. The CPUs that have been used the least times will be select. The selection will start from the last CPU. Tracked-On: #19 Signed-off-by: Yuanyuan Zhao --- src/acrn/acrn_driver.c | 96 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index a2af2a2e15..d8a785b1f5 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -129,6 +129,22 @@ acrnIsRtvm(virDomainDefPtr def) return (nsdef && nsdef->rtvm); } +static void +acrnProcessor2Apicid(virBitmapPtr vcpus, virBufferPtr buf) +{ + ssize_t pos = -1; + bool first = true; + + while ((pos = virBitmapNextSetBit(vcpus, pos)) >= 0) { + if(!first) { + virBufferAddLit(buf, ","); + } else { + first = false; + } + virBufferAsprintf(buf, "%d", acrn_driver->apicidMap[pos]); + } +} + static char* acrnGetCpuAffinity(virDomainDefPtr def) { @@ -207,6 +223,70 @@ acrnSetOnlineVcpus(virDomainDefPtr def, virBitmapPtr vcpus) return virDomainDefSetVcpus(def, virBitmapCountBits(vcpus)); } +static int +acrnSetCpumask(virDomainDefPtr def, size_t *allocMap) +{ + int ret = 0, i, j, refresh = 1; + size_t *pos = NULL, *used = NULL, max_index, max_used; + uint16_t total_cpus = acrn_driver->nodeInfo.cpus; + + if (def->maxvcpus > total_cpus) { + def->maxvcpus = total_cpus; + } + + if (def->cpumask == NULL && + !(def->cpumask = virBitmapNew(total_cpus))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + ret = -ENOMEM; + goto cleanup; + } + + if (VIR_ALLOC_N(pos, def->maxvcpus) < 0 || VIR_ALLOC_N(used, def->maxvcpus) < 0) { + ret = -ENOMEM; + goto cleanup; + } + + for (i = 0; i < def->maxvcpus; i++) { + pos[i] = total_cpus - i - 1; + used[i] = allocMap[total_cpus - i -1]; + } + + for (i = total_cpus - def->maxvcpus; i >= 0 ; i--) { + if (refresh) { + max_index = 0; + max_used = used[0]; + for (j = 1; j < def->maxvcpus; j++) { + if (used[j] > max_used) { + max_index = j; + max_used = used[j]; + } + } + refresh = 0; + } + if (allocMap[i] < max_used) { + pos[max_index] = i; + used[max_index] = allocMap[i]; + refresh = 1; + } + } + + for (i = 0; i < def->maxvcpus; i++) { + if (virBitmapSetBit(def->cpumask, pos[i]) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("failed to set bit %ld in cpu mask"), pos[i]); + ret = -EINVAL; + goto cleanup; + } + } + +cleanup: + if (pos != NULL) + VIR_FREE(pos); + if (used != NULL) + VIR_FREE(used); + return ret; +} + static int acrnProcessPrepareDomain(virDomainObjPtr vm, size_t *allocMap) { @@ -223,8 +303,10 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, size_t *allocMap) priv = vm->privateData; if (def->cpumask == NULL || virBitmapIsAllClear(def->cpumask)) { - virReportError(VIR_ERR_INTERNAL_ERROR, _("cpuset is empty")); - goto cleanup; + if (acrnSetCpumask(def, allocMap) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, _("cpuset is empty")); + goto cleanup; + } } virBitmapShrink(def->cpumask, acrn_driver->nodeInfo.cpus); if (priv->cpuAffinitySet) @@ -648,8 +730,16 @@ acrnBuildStartCmd(virDomainObjPtr vm) /* CPU */ pcpus = acrnGetCpuAffinity(def); - if(pcpus) + if (pcpus) { virCommandAddArgList(cmd, "--cpu_affinity", pcpus, NULL); + } else { + virBuffer buf = VIR_BUFFER_INITIALIZER; + + virBufferAddLit(&buf, "--cpu_affinity="); + acrnProcessor2Apicid(priv->cpuAffinitySet, &buf); + virCommandAddArgBuffer(cmd, &buf); + virBufferFreeAndReset(&buf); + } /* Memory */ virCommandAddArg(cmd, "-m"); From 3001288d1ab347b49bdcb10cfd3bce3e005c4d3b Mon Sep 17 00:00:00 2001 From: Yuanyuan Zhao Date: Fri, 29 Apr 2022 14:25:14 +0800 Subject: [PATCH 29/32] acrn-driver: add vCPU info for `acrn:cpu_affinity` vCPU allocated by `acrn:cpu_affinity` is delivered to acrn-dm directly, and vCPU preparation processs is bypassed. So add vCPU info which used to record durring vCPU preparation in this patch: 1. Set vCPU online state of the util configuration (struct virDomainDef). 2. Add the vCPU to allocMap. Tracked-On: #19 Signed-off-by: Yuanyuan Zhao --- src/acrn/acrn_driver.c | 60 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index d8a785b1f5..cbd99df653 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -223,6 +223,65 @@ acrnSetOnlineVcpus(virDomainDefPtr def, virBitmapPtr vcpus) return virDomainDefSetVcpus(def, virBitmapCountBits(vcpus)); } +static int +acrnSetVcpuAffinityInfo(virDomainObjPtr vm, size_t *allocMap) +{ + virDomainDefPtr def; + acrnDomainObjPrivatePtr priv; + char *src, *token, *str; + int ret = 0, i; + int apicid; + + if (!vm || !(def = vm->def)) + return -1; + + src = acrnGetCpuAffinity(def); + + if (VIR_ALLOC_N(str, strlen(src)) < 0) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + ret = -ENOMEM; + goto cleanup; + } + strcpy(str, src); + + priv = vm->privateData; + if (priv->cpuAffinitySet) + virBitmapFree(priv->cpuAffinitySet); + + + if (!(priv->cpuAffinitySet = virBitmapNew(acrn_driver->nodeInfo.cpus))) { + virReportError(VIR_ERR_NO_MEMORY, NULL); + ret = -ENOMEM; + goto cleanup; + } + + token = strtok(str, ","); + if (token == NULL) { + ret = -1; + goto cleanup; + } + + do { + apicid = atoi(token); + for (i = 0; i < acrn_driver->nodeInfo.cpus; i++) { + if (apicid == acrn_driver->apicidMap[i]) { + allocMap[i] += 1; + virBitmapSetBit(priv->cpuAffinitySet, i); + break; + } + } + } while ((token = strtok(NULL, ",")) != NULL); + + if (!virBitmapIsAllClear(priv->cpuAffinitySet)) { + acrnSetOnlineVcpus(def, priv->cpuAffinitySet); + } +cleanup: + if (str) { + free(str); + } + return ret; +} + static int acrnSetCpumask(virDomainDefPtr def, size_t *allocMap) { @@ -298,6 +357,7 @@ acrnProcessPrepareDomain(virDomainObjPtr vm, size_t *allocMap) return -1; if (acrnGetCpuAffinity(def)) { + acrnSetVcpuAffinityInfo(vm, allocMap); return 0; } From 8f18930d5f33fcc7df137cb6f3c7b73c43b7905e Mon Sep 17 00:00:00 2001 From: Yuanyuan Zhao Date: Mon, 6 Jun 2022 13:51:08 +0800 Subject: [PATCH 30/32] acrn-driver: ethernet parameter modify Modify ethernet parameter to keep consistent with acrn-dm. Tracked-On: #24 Signed-off-by: Yuanyuan Zhao --- src/acrn/acrn_driver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index cbd99df653..f84eb051e2 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -599,7 +599,7 @@ acrnCommandAddDeviceArg(virDomainDefPtr def, return -1; virCommandAddArg(cmd, "-s"); - virCommandAddArgFormat(cmd, "%u:%u:%u,virtio-net,%s,mac=%s", + virCommandAddArgFormat(cmd, "%u:%u:%u,virtio-net,tap=%s,mac=%s", info->addr.pci.bus, info->addr.pci.slot, info->addr.pci.function, From d937d429142c2f16090c3a3e521c210b21e8f489 Mon Sep 17 00:00:00 2001 From: Fei Li Date: Fri, 12 Aug 2022 13:57:37 +0800 Subject: [PATCH 31/32] acrn_driver: offline CPUs in Service VM first Offline CPUs in Service VM before offlining them in ACRN Hypervisor. Tracked-On: #26 Signed-off-by: Fei Li --- src/acrn/acrn_driver.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/acrn/acrn_driver.c b/src/acrn/acrn_driver.c index f84eb051e2..de7593451b 100644 --- a/src/acrn/acrn_driver.c +++ b/src/acrn/acrn_driver.c @@ -2649,9 +2649,9 @@ acrnOfflineCpus(int nprocs) int fd; char path[128], chr, online, cpu_id[4]; ssize_t rc; - virBitmapPtr cpus; + virBitmapPtr cpus, cpus_bak; - cpus = virHostCPUGetOnlineBitmap(); + cpus_bak = cpus = virHostCPUGetOnlineBitmap(); while ((i = virBitmapNextSetBit(cpus, i)) >= 0 && i < nprocs) { /* cpu0 can't be offlined */ if (i == 0) @@ -2682,21 +2682,28 @@ acrnOfflineCpus(int nprocs) virReportError(VIR_ERR_READ_FAILED, _("%s"), path); return -1; } + } - if ((fd = open(ACRN_OFFLINE_PATH, O_WRONLY)) < 0) { - virReportError(VIR_ERR_OPEN_FAILED, _(ACRN_OFFLINE_PATH)); - return -1; - } + if ((fd = open(ACRN_OFFLINE_PATH, O_WRONLY)) < 0) { + virReportError(VIR_ERR_OPEN_FAILED, _(ACRN_OFFLINE_PATH)); + return -1; + } + while ((i = virBitmapNextSetBit(cpus_bak, i)) >= 0 && i < nprocs) { + /* cpu0 can't be offlined */ + if (i == 0) + continue; + + snprintf(cpu_id, sizeof(cpu_id), "%ld", i); if (write(fd, cpu_id, strlen(cpu_id)) < strlen(cpu_id)) { close(fd); virReportError(VIR_ERR_WRITE_FAILED, _(ACRN_OFFLINE_PATH)); return -1; } - - close(fd); } + close(fd); + return 0; } From 1ae2cada6f11457e0e60934abf14ef89cf3f788a Mon Sep 17 00:00:00 2001 From: Chenli Wei Date: Wed, 30 Jul 2025 13:40:23 +0800 Subject: [PATCH 32/32] acrn: fix the compilation issues This patch resolved the compilation issues with Meson and Ninja Signed-off-by: Chenli Wei --- include/libvirt/virterror.h | 2 +- src/conf/schemas/domaincommon.rng | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h index ad466f6a80..28c63b86aa 100644 --- a/include/libvirt/virterror.h +++ b/include/libvirt/virterror.h @@ -141,7 +141,7 @@ typedef enum { VIR_FROM_TPM = 70, /* Error from TPM (Since: 5.6.0) */ VIR_FROM_BPF = 71, /* Error from BPF code (Since: 5.10.0) */ VIR_FROM_CH = 72, /* Error from Cloud-Hypervisor driver (Since: 7.5.0) */ - VIR_FROM_ACRN = 73, /* Error from acrn driver */ + VIR_FROM_ACRN = 73, /* Error from acrn driver (Since: 10.0.0)*/ # ifdef VIR_ENUM_SENTINELS VIR_ERR_DOMAIN_LAST /* (Since: 0.9.13) */ diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index a34427c330..0f06daa7d5 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -232,6 +232,7 @@ vz bhyve hvf + acrn